lunar-toolkit 0.1.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lunar_toolkit/__init__.py +6 -0
- lunar_toolkit/asp.py +432 -0
- lunar_toolkit/cli/__init__.py +88 -0
- lunar_toolkit/cli/_jobs.py +437 -0
- lunar_toolkit/cli/_status.py +368 -0
- lunar_toolkit/cli/_utils.py +128 -0
- lunar_toolkit/cli/app.py +205 -0
- lunar_toolkit/cli/build_db.py +130 -0
- lunar_toolkit/cli/download.py +111 -0
- lunar_toolkit/cli/kaguya.py +253 -0
- lunar_toolkit/cli/ode_search.py +144 -0
- lunar_toolkit/cli/pairs.py +236 -0
- lunar_toolkit/cli/process.py +138 -0
- lunar_toolkit/cli/query.py +70 -0
- lunar_toolkit/cli/reference.py +233 -0
- lunar_toolkit/cli/search.py +90 -0
- lunar_toolkit/cli/serve.py +706 -0
- lunar_toolkit/cli/sfs.py +344 -0
- lunar_toolkit/cli/sfs_report.py +148 -0
- lunar_toolkit/cli/shadowcam.py +246 -0
- lunar_toolkit/cli/stereo.py +85 -0
- lunar_toolkit/cli/workspace.py +84 -0
- lunar_toolkit/config.py +345 -0
- lunar_toolkit/constants.py +217 -0
- lunar_toolkit/csm.py +340 -0
- lunar_toolkit/downloader.py +447 -0
- lunar_toolkit/geometry.py +556 -0
- lunar_toolkit/isis.py +222 -0
- lunar_toolkit/kaguya/__init__.py +45 -0
- lunar_toolkit/kaguya/mi.py +274 -0
- lunar_toolkit/kaguya/tc.py +370 -0
- lunar_toolkit/lroc/__init__.py +6 -0
- lunar_toolkit/lroc/cumindex.py +1197 -0
- lunar_toolkit/lroc/nac.py +702 -0
- lunar_toolkit/observation.py +127 -0
- lunar_toolkit/ode.py +485 -0
- lunar_toolkit/provenance.py +312 -0
- lunar_toolkit/reference.py +1111 -0
- lunar_toolkit/selection.py +714 -0
- lunar_toolkit/sfs.py +1534 -0
- lunar_toolkit/sfs_products.py +676 -0
- lunar_toolkit/sfs_report.py +1328 -0
- lunar_toolkit/shadowcam/__init__.py +29 -0
- lunar_toolkit/shadowcam/sc.py +392 -0
- lunar_toolkit/static/index.html +3259 -0
- lunar_toolkit/stereo.py +519 -0
- lunar_toolkit/stereo_dem.py +145 -0
- lunar_toolkit/tools.py +264 -0
- lunar_toolkit/workspace.py +1173 -0
- lunar_toolkit-0.1.0.dev2.dist-info/METADATA +177 -0
- lunar_toolkit-0.1.0.dev2.dist-info/RECORD +55 -0
- lunar_toolkit-0.1.0.dev2.dist-info/WHEEL +5 -0
- lunar_toolkit-0.1.0.dev2.dist-info/entry_points.txt +2 -0
- lunar_toolkit-0.1.0.dev2.dist-info/licenses/LICENSE +28 -0
- lunar_toolkit-0.1.0.dev2.dist-info/top_level.txt +1 -0
lunar_toolkit/sfs.py
ADDED
|
@@ -0,0 +1,1534 @@
|
|
|
1
|
+
"""Shape-from-shading DEMs from LRO NAC images with ISIS and ASP (Bertone et al. 2026 workflow)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import dataclasses
|
|
7
|
+
import importlib
|
|
8
|
+
import logging
|
|
9
|
+
import math
|
|
10
|
+
import re
|
|
11
|
+
import shutil
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import TYPE_CHECKING, Any
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
|
|
19
|
+
from . import asp, isis
|
|
20
|
+
from .constants import (
|
|
21
|
+
CAMERA_SOURCES,
|
|
22
|
+
MOON_RADIUS_KM,
|
|
23
|
+
REGISTRATION_MODES,
|
|
24
|
+
SELECTION_METHODS,
|
|
25
|
+
SFS_CAMERA_MODELS,
|
|
26
|
+
)
|
|
27
|
+
from .geometry import load_aoi, ps_xy
|
|
28
|
+
from .selection import select_kmedoids
|
|
29
|
+
from .stereo import azimuth_difference
|
|
30
|
+
from .stereo_dem import stereo_dem
|
|
31
|
+
from .tools import ToolError, is_dry_run, reuse_existing
|
|
32
|
+
|
|
33
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
34
|
+
import pandas as pd
|
|
35
|
+
|
|
36
|
+
from .config import RunConfig
|
|
37
|
+
from .provenance import Provenance
|
|
38
|
+
from .workspace import Workspace
|
|
39
|
+
|
|
40
|
+
log = logging.getLogger(__name__)
|
|
41
|
+
|
|
42
|
+
# ASP "Bundle adjustment" recipe for many LRO NAC images under varied illumination
|
|
43
|
+
# (docs/sfs_usage.rst; --match-first-to-last: near the poles the Sun loops the sky).
|
|
44
|
+
ASP_BUNDLE_ADJUST: dict[str, Any] = {
|
|
45
|
+
"ip_per_image": 30000,
|
|
46
|
+
"overlap_limit": 100,
|
|
47
|
+
"num_iterations": 100,
|
|
48
|
+
"num_passes": 2,
|
|
49
|
+
"min_matches": 1,
|
|
50
|
+
"max_pairwise_matches": 2000,
|
|
51
|
+
"camera_weight": 0,
|
|
52
|
+
"robust_threshold": 2,
|
|
53
|
+
"tri_weight": 0.05,
|
|
54
|
+
"tri_robust_threshold": 0.05,
|
|
55
|
+
"remove_outliers_params": "75.0 3.0 100 100",
|
|
56
|
+
"save_intermediate_cameras": True,
|
|
57
|
+
"match_first_to_last": True,
|
|
58
|
+
"forced_triangulation_distance": 100000,
|
|
59
|
+
"min_triangulation_angle": 1e-10,
|
|
60
|
+
"datum": "D_MOON",
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
# ASP "Registration refinement" recipe (bundle_adjust --heights-from-dem, reusing the matches).
|
|
64
|
+
ASP_REFINE: dict[str, Any] = {
|
|
65
|
+
"max_pairwise_matches": 2000,
|
|
66
|
+
"match_first_to_last": True,
|
|
67
|
+
"min_matches": 1,
|
|
68
|
+
"num_iterations": 100,
|
|
69
|
+
"num_passes": 2,
|
|
70
|
+
"camera_weight": 0,
|
|
71
|
+
"save_intermediate_cameras": True,
|
|
72
|
+
"heights_from_dem_uncertainty": 20.0,
|
|
73
|
+
"forced_triangulation_distance": 100000,
|
|
74
|
+
"min_triangulation_angle": 1e-10,
|
|
75
|
+
"remove_outliers_params": "75.0 3.0 100 100",
|
|
76
|
+
"parameter_tolerance": 1e-20,
|
|
77
|
+
"datum": "D_MOON",
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
# ASP "Running SfS in parallel" recipe.
|
|
81
|
+
ASP_SFS: dict[str, Any] = {
|
|
82
|
+
"tile_size": 200,
|
|
83
|
+
"padding": 50,
|
|
84
|
+
"shadow_threshold": 0.005,
|
|
85
|
+
"crop_input_images": True,
|
|
86
|
+
"blending_dist": 10,
|
|
87
|
+
"min_blend_size": 50,
|
|
88
|
+
"allow_borderline_data": True,
|
|
89
|
+
"low_light_threshold": 0.03,
|
|
90
|
+
"smoothness_weight": 0.08,
|
|
91
|
+
"initial_dem_constraint_weight": 0.0025,
|
|
92
|
+
"reflectance_type": 1,
|
|
93
|
+
"max_iterations": 5,
|
|
94
|
+
"save_sparingly": True,
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
# Bertone et al. (2026): match each image against the next 200 in Sun-azimuth order whose
|
|
98
|
+
# footprints overlap (``auto_overlap_percent`` → ``--auto-overlap-params "<dem> 15"``), and
|
|
99
|
+
# soft-anchor cameras at 100 m horizontal / vertical (App. A); intrinsics stay fixed
|
|
100
|
+
# (ASP's default without --solve-intrinsics).
|
|
101
|
+
_UNCERTAINTY = {"camera_position_uncertainty": "100,100"}
|
|
102
|
+
BERTONE_BUNDLE_ADJUST: dict[str, Any] = {
|
|
103
|
+
**ASP_BUNDLE_ADJUST,
|
|
104
|
+
# The next 200 images in Sun-azimuth order (Bertone et al. 2026, §2). Footprint-based
|
|
105
|
+
# pairs (`auto_overlap_percent`) need cameras that fall mostly on the prior DEM, which
|
|
106
|
+
# a crop does not satisfy; set it as an override for full-region runs.
|
|
107
|
+
"overlap_limit": 200,
|
|
108
|
+
**_UNCERTAINTY,
|
|
109
|
+
}
|
|
110
|
+
BERTONE_REFINE: dict[str, Any] = {**ASP_REFINE, **_UNCERTAINTY}
|
|
111
|
+
# §2: mu = 0.05–0.1, lambda = 1e-3, five iterations; App. A: shadow threshold 0.002, albedo fixed.
|
|
112
|
+
BERTONE_SFS: dict[str, Any] = {
|
|
113
|
+
**ASP_SFS,
|
|
114
|
+
"shadow_threshold": 0.002,
|
|
115
|
+
"smoothness_weight": 0.08,
|
|
116
|
+
"initial_dem_constraint_weight": 1e-3,
|
|
117
|
+
"max_iterations": 5,
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
PRESETS: dict[str, dict[str, dict[str, Any]]] = {
|
|
121
|
+
"bertone": {"bundle_adjust": BERTONE_BUNDLE_ADJUST, "refine": BERTONE_REFINE, "sfs": BERTONE_SFS},
|
|
122
|
+
"asp": {"bundle_adjust": ASP_BUNDLE_ADJUST, "refine": ASP_REFINE, "sfs": ASP_SFS},
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
_OVERLAP_OPTIONS = ("overlap_list", "auto_overlap_params", "auto_overlap_buffer", "overlap_limit")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@dataclass
|
|
130
|
+
class SfsConfig:
|
|
131
|
+
"""Settings for :class:`SfsPipeline`.
|
|
132
|
+
|
|
133
|
+
Parameters
|
|
134
|
+
----------
|
|
135
|
+
resolution : float
|
|
136
|
+
SfS DEM grid spacing (m/px).
|
|
137
|
+
projwin : tuple, optional
|
|
138
|
+
``(ulx, uly, lrx, lry)`` crop of the prior DEM, in its projected units.
|
|
139
|
+
preset : {"bertone", "asp"}
|
|
140
|
+
Option defaults: Bertone et al. (2026) or the ASP SfS recipe (:data:`PRESETS`).
|
|
141
|
+
ba_resolution : float, optional
|
|
142
|
+
Grid spacing of the images mapprojected for bundle-adjustment matching and
|
|
143
|
+
stereo registration (default 1 m, NAC native; None = *resolution*).
|
|
144
|
+
reduce : int, "auto" or None
|
|
145
|
+
ISIS ``reduce`` factor bringing the cubes to *resolution* before SfS.
|
|
146
|
+
``"auto"``: ``round(resolution / native)`` per image (native = observation
|
|
147
|
+
``pixel_scale``, else *native_resolution*). Reduced cubes use ISIS cameras
|
|
148
|
+
(with ``--use-approx-camera-models``); bundle adjustment still uses CSM.
|
|
149
|
+
native_resolution : float
|
|
150
|
+
Assumed image m/px when the observation table does not give it.
|
|
151
|
+
csm : bool
|
|
152
|
+
CSM cameras (7–15× faster than ISIS) for mapproject, bundle_adjust and stereo.
|
|
153
|
+
web_spice : bool
|
|
154
|
+
``spiceinit web=true`` (USGS SPICE web service) instead of local kernels.
|
|
155
|
+
registration : {"stereo", "residuals", "none"}
|
|
156
|
+
Registering the adjusted cameras to the prior DEM (translation only).
|
|
157
|
+
``"stereo"``: DEMs from up to *stereo_pairs* pairs, mosaicked and
|
|
158
|
+
``pc_align``-ed (falls back to ``"residuals"`` without pairs);
|
|
159
|
+
``"residuals"``: the bundle-adjustment triangulated points.
|
|
160
|
+
stereo_pairs : int
|
|
161
|
+
Maximum stereo pairs for ``"stereo"`` registration.
|
|
162
|
+
stereo_convergence : (min, max)
|
|
163
|
+
Convergence-angle limits (deg) for those pairs.
|
|
164
|
+
max_displacement : float
|
|
165
|
+
``pc_align --max-displacement`` (m).
|
|
166
|
+
refine : bool
|
|
167
|
+
Re-run bundle adjustment on the kept, registered cameras with
|
|
168
|
+
``--heights-from-dem`` (reusing the matches).
|
|
169
|
+
max_offset_p85 : float, optional
|
|
170
|
+
Drop images whose 85th-percentile mapprojected match offset (m) is larger.
|
|
171
|
+
max_large_residual_fraction : float, optional
|
|
172
|
+
Drop images with more than this fraction of reprojection errors over
|
|
173
|
+
*large_residual_px* pixels.
|
|
174
|
+
min_matches : int
|
|
175
|
+
Drop images with fewer inlier matches.
|
|
176
|
+
selection : {"coverage", "image_subset", "all"}
|
|
177
|
+
SfS image selection: greedy per-pixel illumination coverage
|
|
178
|
+
(:func:`select_by_illumination`), ASP ``image_subset`` per Sun-azimuth bin,
|
|
179
|
+
or all kept images.
|
|
180
|
+
max_shadow_fraction : float, optional
|
|
181
|
+
Images at least this shadowed (below the SfS shadow threshold) are dropped.
|
|
182
|
+
selection_views, selection_azimuth_sep : int, float
|
|
183
|
+
Illumination directions wanted per pixel and their minimum separation (deg).
|
|
184
|
+
max_sfs_images : int, optional
|
|
185
|
+
Cap on the selected images (``"coverage"`` / ``"kmedoids"`` selection).
|
|
186
|
+
selection_clusters : int
|
|
187
|
+
Medoids seeding ``"kmedoids"`` selection (Bertone et al. 2026, §2: ~30).
|
|
188
|
+
sfs_cameras : {"csm", "isis"}
|
|
189
|
+
Camera models for SfS on reduced cubes: CSM cameras re-mapped to the reduced
|
|
190
|
+
image (:func:`lunar_toolkit.csm.isis_reduce_with_csm`, as in Bertone et al.
|
|
191
|
+
2026, App. A) or ISIS cameras with ``--use-approx-camera-models``.
|
|
192
|
+
cameras_from : {"ba", "published"}
|
|
193
|
+
``"published"`` uses the bundle-adjusted cameras of Bertone et al. (2026) for
|
|
194
|
+
*published_region* (:func:`lunar_toolkit.reference.published_cameras`) and skips
|
|
195
|
+
bundle adjustment, registration and refinement; images without one are removed.
|
|
196
|
+
published_region : str or int, optional
|
|
197
|
+
Region of the published cameras (default: from ``region_id`` ``A3NN``).
|
|
198
|
+
max_sfs_slew : float, optional
|
|
199
|
+
Only images with ``|slew| <`` this (deg) are selected for SfS (Bertone et al.
|
|
200
|
+
2026, §2: nadir images); slewed images still join bundle adjustment and stereo
|
|
201
|
+
registration. Unknown slews count as nadir; ``None`` disables the cut.
|
|
202
|
+
selection_grid : int
|
|
203
|
+
Cells along the longer side of the selection grid.
|
|
204
|
+
region_id : str, optional
|
|
205
|
+
``ROIID`` for product file names (default: the work directory name).
|
|
206
|
+
ensemble_members : int
|
|
207
|
+
SfS runs on random subsets of the kept images (sharing at most 40% of their
|
|
208
|
+
images) over an *ensemble_crop* m square at the DEM centre, for the ensemble
|
|
209
|
+
uncertainty maps (Bertone et al. 2026, §3.2); 0 skips this.
|
|
210
|
+
ensemble_size : int, optional
|
|
211
|
+
Images per ensemble member (default: as many as selected).
|
|
212
|
+
ensemble_crop : float
|
|
213
|
+
Side of the ensemble crop (m).
|
|
214
|
+
seed : int
|
|
215
|
+
Random seed of the ensemble image subsets.
|
|
216
|
+
processes, threads : int, optional
|
|
217
|
+
Passed to ``parallel_bundle_adjust``, ``parallel_stereo``, ``parallel_sfs`` and ``mapproject``.
|
|
218
|
+
bundle_adjust_options, refine_options, sfs_options : dict
|
|
219
|
+
Overrides merged onto the preset; a value of ``None`` drops a default.
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
resolution: float = 5.0
|
|
223
|
+
projwin: tuple[float, float, float, float] | None = None
|
|
224
|
+
preset: str = "bertone"
|
|
225
|
+
ba_resolution: float | None = 1.0
|
|
226
|
+
reduce: int | str | None = "auto"
|
|
227
|
+
native_resolution: float = 1.0
|
|
228
|
+
csm: bool = True
|
|
229
|
+
web_spice: bool = True
|
|
230
|
+
registration: str = "stereo"
|
|
231
|
+
stereo_pairs: int = 30
|
|
232
|
+
stereo_convergence: tuple[float, float] = (5.0, 60.0)
|
|
233
|
+
max_displacement: float = 400.0
|
|
234
|
+
refine: bool = True
|
|
235
|
+
max_offset_p85: float | None = 1.8
|
|
236
|
+
max_large_residual_fraction: float | None = 0.85
|
|
237
|
+
large_residual_px: float = 1.0
|
|
238
|
+
min_matches: int = 1
|
|
239
|
+
selection: str = "coverage"
|
|
240
|
+
max_shadow_fraction: float | None = 0.95
|
|
241
|
+
selection_views: int = 3
|
|
242
|
+
selection_azimuth_sep: float = 30.0
|
|
243
|
+
max_sfs_images: int | None = None
|
|
244
|
+
selection_clusters: int = 30
|
|
245
|
+
sfs_cameras: str = "csm"
|
|
246
|
+
cameras_from: str = "ba"
|
|
247
|
+
published_region: str | int | None = None
|
|
248
|
+
max_sfs_slew: float | None = 5.0
|
|
249
|
+
selection_grid: int = 250
|
|
250
|
+
region_id: str | None = None
|
|
251
|
+
ensemble_members: int = 0
|
|
252
|
+
ensemble_size: int | None = None
|
|
253
|
+
ensemble_crop: float = 2500.0
|
|
254
|
+
seed: int = 0
|
|
255
|
+
processes: int | None = None
|
|
256
|
+
threads: int | None = None
|
|
257
|
+
bundle_adjust_options: dict[str, Any] = field(default_factory=dict)
|
|
258
|
+
refine_options: dict[str, Any] = field(default_factory=dict)
|
|
259
|
+
sfs_options: dict[str, Any] = field(default_factory=dict)
|
|
260
|
+
|
|
261
|
+
def __post_init__(self) -> None:
|
|
262
|
+
for name, value, allowed in (
|
|
263
|
+
("preset", self.preset, tuple(PRESETS)),
|
|
264
|
+
("registration", self.registration, REGISTRATION_MODES),
|
|
265
|
+
("selection", self.selection, SELECTION_METHODS),
|
|
266
|
+
("sfs_cameras", self.sfs_cameras, SFS_CAMERA_MODELS),
|
|
267
|
+
("cameras_from", self.cameras_from, CAMERA_SOURCES),
|
|
268
|
+
):
|
|
269
|
+
if value not in allowed:
|
|
270
|
+
raise ValueError(f"{name} must be one of {', '.join(allowed)}, not {value!r}")
|
|
271
|
+
if isinstance(self.reduce, str) and self.reduce != "auto":
|
|
272
|
+
raise ValueError(f"reduce must be an integer, 'auto' or None, not {self.reduce!r}")
|
|
273
|
+
|
|
274
|
+
def options(self, step: str) -> dict[str, Any]:
|
|
275
|
+
"""Preset options for ``"bundle_adjust"``, ``"refine"`` or ``"sfs"`` with overrides applied."""
|
|
276
|
+
extra = {
|
|
277
|
+
"bundle_adjust": self.bundle_adjust_options,
|
|
278
|
+
"refine": self.refine_options,
|
|
279
|
+
"sfs": self.sfs_options,
|
|
280
|
+
}[step]
|
|
281
|
+
return {**PRESETS[self.preset][step], **extra}
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
@dataclass
|
|
285
|
+
class SfsImage:
|
|
286
|
+
"""One input image: cube, optional CSM camera, Sun angles and derived files."""
|
|
287
|
+
|
|
288
|
+
product_id: str
|
|
289
|
+
cube: Path
|
|
290
|
+
camera: Path | None = None
|
|
291
|
+
sun_azimuth: float | None = None
|
|
292
|
+
sun_elevation: float | None = None
|
|
293
|
+
pixel_scale: float | None = None
|
|
294
|
+
mapprojected: Path | None = None
|
|
295
|
+
shadow_fraction: float | None = None
|
|
296
|
+
slew_angle: float | None = None
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
@dataclass(frozen=True)
|
|
300
|
+
class RemovedImage:
|
|
301
|
+
"""An image dropped by a quality filter: *stage* (``ba``, ``refine``, ``selection``) and why."""
|
|
302
|
+
|
|
303
|
+
image: SfsImage
|
|
304
|
+
stage: str
|
|
305
|
+
reason: str
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
class SfsPipeline:
|
|
309
|
+
"""Build an SfS DEM over a region from NAC images and a prior DEM.
|
|
310
|
+
|
|
311
|
+
:meth:`run` follows Bertone et al. (2026, PSJ 7:118) and the ASP large-area SfS recipe:
|
|
312
|
+
|
|
313
|
+
1. calibrate the images and write CSM cameras (``images/``);
|
|
314
|
+
2. crop the prior DEM and resample it to the target grid (``dem/``);
|
|
315
|
+
3. sort by Sun azimuth (``sfs --query``);
|
|
316
|
+
4. mapproject at native resolution (``prj/``) and bundle-adjust with matches between
|
|
317
|
+
the mapprojected images (``ba/``);
|
|
318
|
+
5. drop images with no matches, large mapprojected match offsets or mostly large
|
|
319
|
+
residuals (``selection/removed.csv``);
|
|
320
|
+
6. register the cameras to the prior DEM, translation only, from stereo DEMs
|
|
321
|
+
(``stereo/``) or the BA points; apply to all cameras (``align/``, ``ba_align/``);
|
|
322
|
+
7. refine with ``--heights-from-dem`` and filter again (``ba_refine/``);
|
|
323
|
+
8. mapproject at the target resolution (``prj_sfs/``), write max-lit QA mosaics,
|
|
324
|
+
all images and per 30° Sun-azimuth bin (``qa/``);
|
|
325
|
+
9. select an illumination-diverse subset (``selection/``);
|
|
326
|
+
10. reduce the selected cubes to the target resolution and run ``parallel_sfs`` (``sfs/``);
|
|
327
|
+
11. products (``products/`` via :mod:`lunar_toolkit.sfs_products`, else hillshade + diff).
|
|
328
|
+
|
|
329
|
+
Steps whose outputs exist are skipped, so a rerun resumes.
|
|
330
|
+
|
|
331
|
+
Parameters
|
|
332
|
+
----------
|
|
333
|
+
work_dir : Path
|
|
334
|
+
Directory for all intermediate and final files.
|
|
335
|
+
prior_dem : Path
|
|
336
|
+
Reference terrain, e.g. a LOLA polar stereographic LDEM GeoTIFF.
|
|
337
|
+
config : SfsConfig, optional
|
|
338
|
+
force : bool
|
|
339
|
+
Recompute steps even when their outputs exist.
|
|
340
|
+
observations : DataFrame, optional
|
|
341
|
+
Observation table (:mod:`lunar_toolkit.observation` schema) with the input
|
|
342
|
+
images, for native pixel scales and stereo pairs; looked up in the local
|
|
343
|
+
EDR ``CumIndex`` when needed and not given.
|
|
344
|
+
aoi : optional
|
|
345
|
+
Region the stereo pairs must overlap in: anything
|
|
346
|
+
:func:`lunar_toolkit.stereo.find_stereo_pairs` takes (dict, GeoJSON / WKT).
|
|
347
|
+
data_dir : Path, optional
|
|
348
|
+
``CumIndex`` data directory.
|
|
349
|
+
run_config : RunConfig, optional
|
|
350
|
+
The full run settings (:mod:`lunar_toolkit.config`) written to ``config.yaml``
|
|
351
|
+
and ``provenance.json``; default: *work_dir*, *prior_dem*, the sources and *config*.
|
|
352
|
+
provenance : bool
|
|
353
|
+
Write ``config.yaml`` and append this run to ``provenance.json`` (never in a dry run).
|
|
354
|
+
|
|
355
|
+
Attributes
|
|
356
|
+
----------
|
|
357
|
+
all_images, images : list of SfsImage
|
|
358
|
+
Every input / those passing the quality filters (Sun-azimuth order).
|
|
359
|
+
removed : list of RemovedImage
|
|
360
|
+
selected, unselected : list of SfsImage
|
|
361
|
+
The SfS inputs and the remaining good images (e.g. for an uncertainty ensemble);
|
|
362
|
+
both carry ``mapprojected`` (``prj_sfs/``) and ``sun_azimuth``.
|
|
363
|
+
dem : Path
|
|
364
|
+
The cropped prior DEM on the target grid.
|
|
365
|
+
|
|
366
|
+
Example
|
|
367
|
+
-------
|
|
368
|
+
::
|
|
369
|
+
|
|
370
|
+
cfg = SfsConfig(resolution=5, projwin=(-10000, 50000, 0, 40000))
|
|
371
|
+
p = SfsPipeline("work/malapert", "LDEM_87S_5MPP_ADJ.TIF", cfg)
|
|
372
|
+
dem = p.run(["raw/M1234567890LE.IMG", ...])
|
|
373
|
+
"""
|
|
374
|
+
|
|
375
|
+
def __init__(
|
|
376
|
+
self,
|
|
377
|
+
work_dir: Path,
|
|
378
|
+
prior_dem: Path,
|
|
379
|
+
config: SfsConfig | None = None,
|
|
380
|
+
force: bool = False,
|
|
381
|
+
*,
|
|
382
|
+
observations: pd.DataFrame | None = None,
|
|
383
|
+
aoi: Any = None,
|
|
384
|
+
data_dir: Path | None = None,
|
|
385
|
+
run_config: RunConfig | None = None,
|
|
386
|
+
provenance: bool = True,
|
|
387
|
+
) -> None:
|
|
388
|
+
self.work_dir = Path(work_dir)
|
|
389
|
+
self.prior_dem = Path(prior_dem)
|
|
390
|
+
self.config = config or SfsConfig()
|
|
391
|
+
self.force = force
|
|
392
|
+
self.observations = observations
|
|
393
|
+
self.aoi = aoi
|
|
394
|
+
self.data_dir = data_dir
|
|
395
|
+
self.all_images: list[SfsImage] = []
|
|
396
|
+
self.images: list[SfsImage] = []
|
|
397
|
+
self.removed: list[RemovedImage] = []
|
|
398
|
+
self.selected: list[SfsImage] = []
|
|
399
|
+
self.unselected: list[SfsImage] = []
|
|
400
|
+
self.dem = self.work_dir / "dem" / f"prior-{self.config.resolution:g}m.tif"
|
|
401
|
+
self._registered = False
|
|
402
|
+
self.run_config = run_config
|
|
403
|
+
self.provenance = provenance
|
|
404
|
+
self._prov: Provenance | None = None
|
|
405
|
+
|
|
406
|
+
def run(self, sources: Sequence[Path]) -> Path:
|
|
407
|
+
"""All steps; returns the final SfS DEM.
|
|
408
|
+
|
|
409
|
+
Writes ``config.yaml`` and appends a run to ``provenance.json`` first (see
|
|
410
|
+
:meth:`start_provenance`), then records each step there.
|
|
411
|
+
"""
|
|
412
|
+
sources = [Path(s) for s in sources]
|
|
413
|
+
self.start_provenance(sources)
|
|
414
|
+
step = self._step
|
|
415
|
+
try:
|
|
416
|
+
step("prepare_images", self.prepare_images, sources)
|
|
417
|
+
step("prepare_dem", self.prepare_dem)
|
|
418
|
+
step("sort_by_sun", self.sort_by_sun)
|
|
419
|
+
step(
|
|
420
|
+
"mapproject",
|
|
421
|
+
self.mapproject,
|
|
422
|
+
self.images,
|
|
423
|
+
"prj",
|
|
424
|
+
self.config.ba_resolution or self.config.resolution,
|
|
425
|
+
)
|
|
426
|
+
if self.config.cameras_from == "published":
|
|
427
|
+
step("published_cameras", self.use_published_cameras)
|
|
428
|
+
else:
|
|
429
|
+
step("bundle_adjust", self.bundle_adjust)
|
|
430
|
+
step("filter_ba", self.filter_images, self.ba_prefix, "ba")
|
|
431
|
+
step("register", self.register)
|
|
432
|
+
if self.config.refine:
|
|
433
|
+
step("refine", self.refine)
|
|
434
|
+
step("filter_refine", self.filter_images, self.refine_prefix, "refine")
|
|
435
|
+
step("mapproject_for_sfs", self.mapproject_for_sfs)
|
|
436
|
+
step("qa_mosaics", self.qa_mosaics)
|
|
437
|
+
step("select_images", self.select_images)
|
|
438
|
+
dem = step("shape_from_shading", self.shape_from_shading)
|
|
439
|
+
step("products", self.products, dem)
|
|
440
|
+
if self.config.ensemble_members:
|
|
441
|
+
step("ensemble", self.ensemble)
|
|
442
|
+
except BaseException:
|
|
443
|
+
if self._prov:
|
|
444
|
+
self._prov.finish("error")
|
|
445
|
+
raise
|
|
446
|
+
if self._prov:
|
|
447
|
+
self._prov.finish("ok")
|
|
448
|
+
return dem
|
|
449
|
+
|
|
450
|
+
def effective_config(self, sources: Sequence[Path] = ()) -> RunConfig:
|
|
451
|
+
""":attr:`run_config` with this pipeline's work dir, prior DEM and :class:`SfsConfig`."""
|
|
452
|
+
from .config import RunConfig
|
|
453
|
+
|
|
454
|
+
if self.run_config is None:
|
|
455
|
+
return RunConfig(
|
|
456
|
+
work_dir=self.work_dir,
|
|
457
|
+
dem=str(self.prior_dem),
|
|
458
|
+
sources=[str(s) for s in sources],
|
|
459
|
+
sfs=self.config,
|
|
460
|
+
)
|
|
461
|
+
return dataclasses.replace(
|
|
462
|
+
self.run_config, work_dir=self.work_dir, dem=str(self.prior_dem), sfs=self.config
|
|
463
|
+
)
|
|
464
|
+
|
|
465
|
+
def start_provenance(self, sources: Sequence[Path]) -> Path | None:
|
|
466
|
+
"""Write ``config.yaml`` and start a ``provenance.json`` run; returns the config path.
|
|
467
|
+
|
|
468
|
+
Inputs are *sources* and the prior DEM. Nothing is written in a dry run
|
|
469
|
+
or with ``provenance=False``.
|
|
470
|
+
"""
|
|
471
|
+
if is_dry_run() or not self.provenance:
|
|
472
|
+
return None
|
|
473
|
+
from .config import dump_config
|
|
474
|
+
from .provenance import Provenance
|
|
475
|
+
|
|
476
|
+
cfg = self.effective_config(sources)
|
|
477
|
+
path = self.work_dir / "config.yaml"
|
|
478
|
+
dump_config(cfg, path)
|
|
479
|
+
self._prov = Provenance(self.work_dir, cfg, [*sources, self.prior_dem], checksums=cfg.checksums)
|
|
480
|
+
return path
|
|
481
|
+
|
|
482
|
+
def _step(self, name: str, func: Any, *args: Any) -> Any:
|
|
483
|
+
if self._prov is None:
|
|
484
|
+
return func(*args)
|
|
485
|
+
with self._prov.step(name) as entry:
|
|
486
|
+
entry["result"] = result = func(*args)
|
|
487
|
+
return result
|
|
488
|
+
|
|
489
|
+
@property
|
|
490
|
+
def ba_prefix(self) -> Path:
|
|
491
|
+
return self.work_dir / "ba" / "run"
|
|
492
|
+
|
|
493
|
+
@property
|
|
494
|
+
def align_prefix(self) -> Path:
|
|
495
|
+
return self.work_dir / "align" / "run"
|
|
496
|
+
|
|
497
|
+
@property
|
|
498
|
+
def ba_align_prefix(self) -> Path:
|
|
499
|
+
return self.work_dir / "ba_align" / "run"
|
|
500
|
+
|
|
501
|
+
@property
|
|
502
|
+
def refine_prefix(self) -> Path:
|
|
503
|
+
return self.work_dir / "ba_refine" / "run"
|
|
504
|
+
|
|
505
|
+
@property
|
|
506
|
+
def sfs_camera_prefix(self) -> Path:
|
|
507
|
+
"""Adjustments renamed for the reduced ISIS cubes."""
|
|
508
|
+
return self.work_dir / "sfs_cameras" / "run"
|
|
509
|
+
|
|
510
|
+
@property
|
|
511
|
+
def sfs_prefix(self) -> Path:
|
|
512
|
+
return self.work_dir / "sfs" / "run"
|
|
513
|
+
|
|
514
|
+
@property
|
|
515
|
+
def sfs_dem(self) -> Path:
|
|
516
|
+
return Path(f"{self.sfs_prefix}-DEM-final.tif")
|
|
517
|
+
|
|
518
|
+
@property
|
|
519
|
+
def camera_prefix(self) -> Path | None:
|
|
520
|
+
"""Adjustments of the final (registered, refined) cameras; None for published cameras."""
|
|
521
|
+
if self.config.cameras_from == "published":
|
|
522
|
+
return None
|
|
523
|
+
if self.config.refine:
|
|
524
|
+
return self.refine_prefix
|
|
525
|
+
return self.ba_align_prefix if self.config.registration != "none" else self.ba_prefix
|
|
526
|
+
|
|
527
|
+
@property
|
|
528
|
+
def shadow_threshold(self) -> float:
|
|
529
|
+
return float(self.config.options("sfs").get("shadow_threshold") or 0.0)
|
|
530
|
+
|
|
531
|
+
def _skip(self, path: Path) -> bool:
|
|
532
|
+
return reuse_existing(path, self.force)
|
|
533
|
+
|
|
534
|
+
def _parallel_opts(self) -> dict[str, Any]:
|
|
535
|
+
return {"processes": self.config.processes, "threads": self.config.threads}
|
|
536
|
+
|
|
537
|
+
def _cameras(self, images: Sequence[SfsImage]) -> list[Path]:
|
|
538
|
+
return [im.camera for im in images if im.camera] if self.config.csm else []
|
|
539
|
+
|
|
540
|
+
def _map_path(self, subdir: str, im: SfsImage) -> Path:
|
|
541
|
+
return self.work_dir / subdir / f"{im.product_id}.map.tif"
|
|
542
|
+
|
|
543
|
+
def prepare_images(self, sources: Sequence[Path]) -> list[SfsImage]:
|
|
544
|
+
"""Calibrate EDRs (``.IMG``) or take ready cubes; write CSM cameras."""
|
|
545
|
+
cfg = self.config
|
|
546
|
+
scales = _by_id(self.observations, "pixel_scale")
|
|
547
|
+
slews = _by_id(self.observations, "slew_angle")
|
|
548
|
+
self.all_images = []
|
|
549
|
+
for src in map(Path, sources):
|
|
550
|
+
cube, camera = isis.prepare_lronac(
|
|
551
|
+
src, self.work_dir / "images", csm=cfg.csm, web=cfg.web_spice, force=self.force
|
|
552
|
+
)
|
|
553
|
+
pid = src.name.split(".")[0]
|
|
554
|
+
self.all_images.append(
|
|
555
|
+
SfsImage(
|
|
556
|
+
pid, cube, camera, pixel_scale=scales.get(pid.upper()), slew_angle=slews.get(pid.upper())
|
|
557
|
+
)
|
|
558
|
+
)
|
|
559
|
+
self.images = list(self.all_images)
|
|
560
|
+
self._record_observations()
|
|
561
|
+
return self.images
|
|
562
|
+
|
|
563
|
+
def _record_observations(self) -> None:
|
|
564
|
+
"""Write the given observation rows to the workspace ``images`` table (not in a dry run)."""
|
|
565
|
+
if is_dry_run() or self.observations is None or not len(self.observations):
|
|
566
|
+
return
|
|
567
|
+
from .workspace import WorkspaceError
|
|
568
|
+
|
|
569
|
+
try:
|
|
570
|
+
with self.workspace() as ws:
|
|
571
|
+
ws.add_observations(self.observations)
|
|
572
|
+
except WorkspaceError as exc:
|
|
573
|
+
log.warning("Workspace images not written: %s", exc)
|
|
574
|
+
|
|
575
|
+
def prepare_dem(self) -> Path:
|
|
576
|
+
"""Crop the prior DEM to ``projwin`` and resample to the target grid."""
|
|
577
|
+
cfg = self.config
|
|
578
|
+
crop = self.work_dir / "dem" / "prior.tif"
|
|
579
|
+
if not self._skip(crop):
|
|
580
|
+
asp.gdal_translate(self.prior_dem, crop, projwin=cfg.projwin)
|
|
581
|
+
if not self._skip(self.dem):
|
|
582
|
+
asp.gdal_translate(crop, self.dem, tr=cfg.resolution, r="cubicspline")
|
|
583
|
+
return self.dem
|
|
584
|
+
|
|
585
|
+
def sort_by_sun(self) -> list[SfsImage]:
|
|
586
|
+
"""Fill Sun azimuth / elevation and sort the images by azimuth.
|
|
587
|
+
|
|
588
|
+
Bundle adjustment matches neighbours in this order, and ``parallel_sfs``
|
|
589
|
+
expects it.
|
|
590
|
+
"""
|
|
591
|
+
angles = asp.sun_angles(
|
|
592
|
+
self.dem,
|
|
593
|
+
[im.cube for im in self.images],
|
|
594
|
+
self._cameras(self.images),
|
|
595
|
+
out_prefix=self.work_dir / "query" / "run",
|
|
596
|
+
)
|
|
597
|
+
by_image = {Path(a.image).name: a for a in angles}
|
|
598
|
+
for im in self.images:
|
|
599
|
+
a = by_image.get(im.cube.name)
|
|
600
|
+
if a:
|
|
601
|
+
im.sun_azimuth, im.sun_elevation = a.azimuth, a.elevation
|
|
602
|
+
if angles:
|
|
603
|
+
self.images.sort(key=lambda im: (im.sun_azimuth is None, im.sun_azimuth or 0.0))
|
|
604
|
+
elif not is_dry_run():
|
|
605
|
+
log.warning("sfs --query reported no Sun angles; keeping input order")
|
|
606
|
+
return self.images
|
|
607
|
+
|
|
608
|
+
def mapproject(
|
|
609
|
+
self, images: Sequence[SfsImage], subdir: str, resolution: float, **options: Any
|
|
610
|
+
) -> list[Path]:
|
|
611
|
+
"""Mapproject *images* onto the prior DEM (cropped to ``projwin``) into ``<subdir>/``."""
|
|
612
|
+
cfg = self.config
|
|
613
|
+
win = cfg.projwin
|
|
614
|
+
outs = []
|
|
615
|
+
for im in images:
|
|
616
|
+
out = self._map_path(subdir, im)
|
|
617
|
+
if not self._skip(out):
|
|
618
|
+
asp.mapproject(
|
|
619
|
+
self.dem,
|
|
620
|
+
im.cube,
|
|
621
|
+
out,
|
|
622
|
+
im.camera if cfg.csm else None,
|
|
623
|
+
tr=resolution,
|
|
624
|
+
t_projwin=(win[0], win[3], win[2], win[1]) if win else None,
|
|
625
|
+
threads=cfg.threads,
|
|
626
|
+
**options,
|
|
627
|
+
)
|
|
628
|
+
outs.append(out)
|
|
629
|
+
return outs
|
|
630
|
+
|
|
631
|
+
def bundle_adjust(self) -> Path:
|
|
632
|
+
"""``parallel_bundle_adjust`` with matches from the mapprojected images.
|
|
633
|
+
|
|
634
|
+
``--mapproj-dem`` writes the per-image match-offset report used by
|
|
635
|
+
:meth:`filter_images`.
|
|
636
|
+
"""
|
|
637
|
+
done = Path(f"{self.ba_prefix}-final_residuals_stats.txt")
|
|
638
|
+
if self._skip(done):
|
|
639
|
+
return self.ba_prefix
|
|
640
|
+
opts = self._ba_options(self.config.options("bundle_adjust"))
|
|
641
|
+
return asp.bundle_adjust(
|
|
642
|
+
[im.cube for im in self.images],
|
|
643
|
+
self.ba_prefix,
|
|
644
|
+
self._cameras(self.images),
|
|
645
|
+
mapprojected=[self._map_path("prj", im) for im in self.images],
|
|
646
|
+
dem=self.dem,
|
|
647
|
+
parallel=True,
|
|
648
|
+
mapproj_dem=self.dem,
|
|
649
|
+
**{**self._parallel_opts(), **opts},
|
|
650
|
+
)
|
|
651
|
+
|
|
652
|
+
def _ba_options(self, opts: dict[str, Any]) -> dict[str, Any]:
|
|
653
|
+
"""Preset bundle-adjust options made valid for ASP.
|
|
654
|
+
|
|
655
|
+
``auto_overlap_percent`` becomes ``--auto-overlap-params "<dem> <pct>"``, and of
|
|
656
|
+
ASP's mutually exclusive pair-selection options only the most specific is kept
|
|
657
|
+
(overlap list, footprint overlap, overlap buffer, then consecutive-image limit).
|
|
658
|
+
"""
|
|
659
|
+
opts = dict(opts)
|
|
660
|
+
pct = opts.pop("auto_overlap_percent", None)
|
|
661
|
+
if pct is not None and "auto_overlap_params" not in opts:
|
|
662
|
+
opts["auto_overlap_params"] = f"{self.dem} {pct:g}"
|
|
663
|
+
given = [k for k in _OVERLAP_OPTIONS if opts.get(k) not in (None, False)]
|
|
664
|
+
for key in given[1:]:
|
|
665
|
+
log.info(
|
|
666
|
+
"bundle_adjust: --%s overrides --%s; dropping the latter",
|
|
667
|
+
given[0].replace("_", "-"),
|
|
668
|
+
key.replace("_", "-"),
|
|
669
|
+
)
|
|
670
|
+
del opts[key]
|
|
671
|
+
return opts
|
|
672
|
+
|
|
673
|
+
def workspace(self, read_only: bool = False) -> Workspace:
|
|
674
|
+
"""The work dir's DuckDB workspace (``lunar.duckdb``, :mod:`lunar_toolkit.workspace`)."""
|
|
675
|
+
from .workspace import Workspace
|
|
676
|
+
|
|
677
|
+
return Workspace(self.work_dir, threads=self.config.threads, read_only=read_only)
|
|
678
|
+
|
|
679
|
+
def filter_images(self, prefix: Path, stage: str) -> list[RemovedImage]:
|
|
680
|
+
"""Drop images that ASP's bundle-adjustment reports flag (see :func:`flag_images`).
|
|
681
|
+
|
|
682
|
+
Loads ``<prefix>-*`` reports (residual stats and raw pixels, match offsets, point
|
|
683
|
+
maps, convergence angles) into the workspace as run ``<prefix>``'s directory name
|
|
684
|
+
(``ba``, ``ba_refine``) and removes the images of its ``image_rejections`` view,
|
|
685
|
+
with this config's thresholds. Removed images are logged and appended to
|
|
686
|
+
``selection/removed.csv``.
|
|
687
|
+
"""
|
|
688
|
+
cfg = self.config
|
|
689
|
+
files = {
|
|
690
|
+
k: Path(f"{prefix}-{name}")
|
|
691
|
+
for k, name in (
|
|
692
|
+
("stats", "final_residuals_stats.txt"),
|
|
693
|
+
("raw", "final_residuals_raw_pixels.txt"),
|
|
694
|
+
("offsets", "mapproj_match_offset_stats.txt"),
|
|
695
|
+
)
|
|
696
|
+
}
|
|
697
|
+
if not any(p.exists() for p in files.values()):
|
|
698
|
+
if not is_dry_run():
|
|
699
|
+
log.warning("No bundle-adjustment reports under %s; keeping all images", prefix)
|
|
700
|
+
return []
|
|
701
|
+
with self.workspace() as ws:
|
|
702
|
+
ws.set_rejection_params(cfg)
|
|
703
|
+
ws.load_bundle_adjust(prefix, run=Path(prefix).parent.name)
|
|
704
|
+
flags = ws.rejections(Path(prefix).parent.name)
|
|
705
|
+
removed = [
|
|
706
|
+
RemovedImage(im, stage, flags[im.cube.name]) for im in self.images if im.cube.name in flags
|
|
707
|
+
]
|
|
708
|
+
return self._remove(removed)
|
|
709
|
+
|
|
710
|
+
def _remove(self, removed: list[RemovedImage]) -> list[RemovedImage]:
|
|
711
|
+
if not removed:
|
|
712
|
+
return []
|
|
713
|
+
gone = {id(r.image) for r in removed}
|
|
714
|
+
self.images = [im for im in self.images if id(im) not in gone]
|
|
715
|
+
self.removed += removed
|
|
716
|
+
for r in removed:
|
|
717
|
+
log.info("Removed %s (%s): %s", r.image.product_id, r.stage, r.reason)
|
|
718
|
+
log.info("%d images removed at %s, %d remain", len(removed), removed[0].stage, len(self.images))
|
|
719
|
+
_write_csv(
|
|
720
|
+
self.work_dir / "selection" / "removed.csv",
|
|
721
|
+
["product_id", "stage", "reason"],
|
|
722
|
+
[[r.image.product_id, r.stage, r.reason] for r in self.removed],
|
|
723
|
+
)
|
|
724
|
+
return removed
|
|
725
|
+
|
|
726
|
+
def register(self) -> Path:
|
|
727
|
+
"""Register the adjusted cameras to the prior DEM (translation only); returns their prefix."""
|
|
728
|
+
mode = self.config.registration
|
|
729
|
+
if mode == "none":
|
|
730
|
+
return self.ba_prefix
|
|
731
|
+
transform = Path(f"{self.align_prefix}-transform.txt")
|
|
732
|
+
if not self._skip(transform):
|
|
733
|
+
if mode != "stereo" or self._stereo_mosaic() is None:
|
|
734
|
+
self._align_residuals()
|
|
735
|
+
else:
|
|
736
|
+
self._align_stereo()
|
|
737
|
+
marker = Path(f"{self.ba_align_prefix}-applied-transform.txt")
|
|
738
|
+
if not self._skip(marker):
|
|
739
|
+
asp.apply_transform(
|
|
740
|
+
[im.cube for im in self.images],
|
|
741
|
+
transform,
|
|
742
|
+
self.ba_align_prefix,
|
|
743
|
+
self._cameras(self.images),
|
|
744
|
+
input_adjustments_prefix=self.ba_prefix,
|
|
745
|
+
)
|
|
746
|
+
if not is_dry_run():
|
|
747
|
+
marker.write_text(transform.read_text())
|
|
748
|
+
return self.ba_align_prefix
|
|
749
|
+
|
|
750
|
+
def _align_residuals(self) -> Path:
|
|
751
|
+
return asp.pc_align(
|
|
752
|
+
self.dem,
|
|
753
|
+
Path(f"{self.ba_prefix}-final_residuals_pointmap.csv"),
|
|
754
|
+
self.align_prefix,
|
|
755
|
+
max_displacement=self.config.max_displacement,
|
|
756
|
+
compute_translation_only=True,
|
|
757
|
+
csv_format="1:lon 2:lat 3:height_above_datum",
|
|
758
|
+
save_transformed_source_points=True,
|
|
759
|
+
)
|
|
760
|
+
|
|
761
|
+
def _align_stereo(self) -> Path:
|
|
762
|
+
return asp.pc_align(
|
|
763
|
+
self.dem,
|
|
764
|
+
self.work_dir / "stereo" / "mosaic.tif",
|
|
765
|
+
self.align_prefix,
|
|
766
|
+
max_displacement=self.config.max_displacement,
|
|
767
|
+
compute_translation_only=True,
|
|
768
|
+
)
|
|
769
|
+
|
|
770
|
+
def stereo_pairs(self) -> pd.DataFrame:
|
|
771
|
+
"""Up to ``stereo_pairs`` kept-image pairs ranked by :func:`~lunar_toolkit.stereo.find_stereo_pairs`."""
|
|
772
|
+
from .stereo import find_stereo_pairs
|
|
773
|
+
|
|
774
|
+
obs = self._observations()
|
|
775
|
+
kept = {im.product_id.upper() for im in self.images}
|
|
776
|
+
obs = obs[obs["product_id"].astype(str).str.upper().isin(kept)]
|
|
777
|
+
pairs = find_stereo_pairs(obs, aoi=self.aoi, convergence=self.config.stereo_convergence)
|
|
778
|
+
return pairs.head(self.config.stereo_pairs)
|
|
779
|
+
|
|
780
|
+
def _stereo_mosaic(self) -> Path | None:
|
|
781
|
+
"""Stereo DEMs (triangulated with the ``ba/`` cameras) mosaicked; None without any."""
|
|
782
|
+
cfg = self.config
|
|
783
|
+
mosaic = self.work_dir / "stereo" / "mosaic.tif"
|
|
784
|
+
if self._skip(mosaic):
|
|
785
|
+
return mosaic
|
|
786
|
+
try:
|
|
787
|
+
pairs = self.stereo_pairs()
|
|
788
|
+
except (OSError, KeyError, ValueError) as exc:
|
|
789
|
+
log.warning("No observations for stereo registration (%s); using BA points", exc)
|
|
790
|
+
return None
|
|
791
|
+
by_id = {im.product_id.upper(): im for im in self.images}
|
|
792
|
+
dems = []
|
|
793
|
+
for _, pair in pairs.iterrows():
|
|
794
|
+
a, b = by_id[str(pair["product_id_1"]).upper()], by_id[str(pair["product_id_2"]).upper()]
|
|
795
|
+
try:
|
|
796
|
+
dem = stereo_dem(
|
|
797
|
+
a.cube,
|
|
798
|
+
b.cube,
|
|
799
|
+
self.work_dir / "stereo" / f"{a.product_id}__{b.product_id}",
|
|
800
|
+
resolution=cfg.resolution,
|
|
801
|
+
prior_dem=self.dem,
|
|
802
|
+
align=False,
|
|
803
|
+
csm=cfg.csm,
|
|
804
|
+
cameras=(a.camera, b.camera) if cfg.csm and a.camera and b.camera else None,
|
|
805
|
+
mapprojected=(self._map_path("prj", a), self._map_path("prj", b)),
|
|
806
|
+
bundle_adjust_prefix=self.ba_prefix,
|
|
807
|
+
processes=cfg.processes,
|
|
808
|
+
threads=cfg.threads,
|
|
809
|
+
force=self.force,
|
|
810
|
+
)
|
|
811
|
+
except ToolError as exc:
|
|
812
|
+
log.warning("Stereo %s / %s failed: %s", a.product_id, b.product_id, exc)
|
|
813
|
+
continue
|
|
814
|
+
if is_dry_run() or dem.exists():
|
|
815
|
+
dems.append(dem)
|
|
816
|
+
if not dems:
|
|
817
|
+
log.warning("No stereo DEMs for registration; using BA points")
|
|
818
|
+
return None
|
|
819
|
+
return asp.dem_mosaic(dems, mosaic)
|
|
820
|
+
|
|
821
|
+
def _observations(self) -> pd.DataFrame:
|
|
822
|
+
if self.observations is None:
|
|
823
|
+
from .lroc.cumindex import CumIndex
|
|
824
|
+
|
|
825
|
+
ci = CumIndex("edr", data_dir=self.data_dir)
|
|
826
|
+
self.observations = ci.to_observations(ci.lookup([im.product_id for im in self.all_images]))
|
|
827
|
+
return self.observations
|
|
828
|
+
|
|
829
|
+
def refine(self) -> Path:
|
|
830
|
+
"""Bundle-adjust the kept, registered cameras again with ``--heights-from-dem``.
|
|
831
|
+
|
|
832
|
+
Reuses the ``ba/`` matches (``--match-files-prefix``); ``--mapproj-dem`` gives
|
|
833
|
+
the match-offset report for a second :meth:`filter_images`.
|
|
834
|
+
"""
|
|
835
|
+
done = Path(f"{self.refine_prefix}-final_residuals_stats.txt")
|
|
836
|
+
if self._skip(done):
|
|
837
|
+
return self.refine_prefix
|
|
838
|
+
start = self.ba_align_prefix if self.config.registration != "none" else self.ba_prefix
|
|
839
|
+
opts = self._ba_options(self.config.options("refine"))
|
|
840
|
+
return asp.bundle_adjust(
|
|
841
|
+
[im.cube for im in self.images],
|
|
842
|
+
self.refine_prefix,
|
|
843
|
+
self._cameras(self.images),
|
|
844
|
+
match_files_prefix=self.ba_prefix,
|
|
845
|
+
input_adjustments_prefix=start,
|
|
846
|
+
heights_from_dem=self.dem,
|
|
847
|
+
mapproj_dem=self.dem,
|
|
848
|
+
threads=self.config.threads,
|
|
849
|
+
**opts,
|
|
850
|
+
)
|
|
851
|
+
|
|
852
|
+
def mapproject_for_sfs(self) -> list[Path]:
|
|
853
|
+
"""Mapproject the kept images at the target resolution with the final cameras (``prj_sfs/``)."""
|
|
854
|
+
outs = self.mapproject(
|
|
855
|
+
self.images, "prj_sfs", self.config.resolution, bundle_adjust_prefix=self.camera_prefix
|
|
856
|
+
)
|
|
857
|
+
for im, out in zip(self.images, outs):
|
|
858
|
+
im.mapprojected = out
|
|
859
|
+
return outs
|
|
860
|
+
|
|
861
|
+
def qa_mosaics(self, bin_width: float = 30.0) -> dict[str, Path]:
|
|
862
|
+
"""Max-lit mosaics of all kept images and per Sun-azimuth bin (``qa/``).
|
|
863
|
+
|
|
864
|
+
Ghosting in these shows misregistered images (Bertone et al. 2026, App. A).
|
|
865
|
+
Bins are named by their start, e.g. ``max_lit_az030.tif`` for 30–60°.
|
|
866
|
+
"""
|
|
867
|
+
qa = self.work_dir / "qa"
|
|
868
|
+
maps = [im.mapprojected or self._map_path("prj_sfs", im) for im in self.images]
|
|
869
|
+
groups: dict[str, list[Path]] = {"max_lit": maps}
|
|
870
|
+
for im, m in zip(self.images, maps):
|
|
871
|
+
if im.sun_azimuth is not None:
|
|
872
|
+
start = int((im.sun_azimuth % 360.0) // bin_width * bin_width)
|
|
873
|
+
groups.setdefault(f"max_lit_az{start:03d}", []).append(m)
|
|
874
|
+
out = {}
|
|
875
|
+
for name, paths in groups.items():
|
|
876
|
+
out[name] = qa / f"{name}.tif"
|
|
877
|
+
if paths and not self._skip(out[name]):
|
|
878
|
+
asp.dem_mosaic(paths, out[name], max=True)
|
|
879
|
+
return out
|
|
880
|
+
|
|
881
|
+
def select_images(self) -> list[SfsImage]:
|
|
882
|
+
"""Split the kept images into SfS inputs and the rest (``selection/``).
|
|
883
|
+
|
|
884
|
+
Images at least ``max_shadow_fraction`` shadowed are removed first. Writes
|
|
885
|
+
``selection.csv`` (every kept image: azimuth, shadow fraction, mapprojected path,
|
|
886
|
+
selected) plus ``selected.txt`` / ``unselected.txt`` (one product ID per line).
|
|
887
|
+
"""
|
|
888
|
+
cfg = self.config
|
|
889
|
+
out_dir = self.work_dir / "selection"
|
|
890
|
+
lit = self._lit_masks()
|
|
891
|
+
if lit is not None and cfg.max_shadow_fraction is not None:
|
|
892
|
+
shadowed = [
|
|
893
|
+
RemovedImage(im, "selection", f"{im.shadow_fraction:.1%} shadowed")
|
|
894
|
+
for im in self.images
|
|
895
|
+
if im.shadow_fraction is not None and im.shadow_fraction >= cfg.max_shadow_fraction
|
|
896
|
+
]
|
|
897
|
+
gone = {id(r.image) for r in shadowed}
|
|
898
|
+
lit = lit[[k for k, im in enumerate(self.images) if id(im) not in gone]]
|
|
899
|
+
self._remove(shadowed)
|
|
900
|
+
eligible = self._sfs_eligible()
|
|
901
|
+
if cfg.selection == "all" or lit is None:
|
|
902
|
+
chosen = eligible
|
|
903
|
+
elif cfg.selection == "coverage":
|
|
904
|
+
az = np.array([np.nan if im.sun_azimuth is None else im.sun_azimuth for im in self.images])
|
|
905
|
+
picked, views = select_by_illumination(
|
|
906
|
+
lit[eligible],
|
|
907
|
+
az[eligible],
|
|
908
|
+
views=cfg.selection_views,
|
|
909
|
+
azimuth_sep=cfg.selection_azimuth_sep,
|
|
910
|
+
max_images=cfg.max_sfs_images,
|
|
911
|
+
)
|
|
912
|
+
chosen = [eligible[k] for k in picked]
|
|
913
|
+
covered = lit[eligible].any(axis=0)
|
|
914
|
+
if covered.any():
|
|
915
|
+
log.info(
|
|
916
|
+
"Selected %d / %d nadir images; lit cells with >=%d directions: %.1f%%",
|
|
917
|
+
len(chosen),
|
|
918
|
+
len(eligible),
|
|
919
|
+
cfg.selection_views,
|
|
920
|
+
100 * np.mean(views[covered] >= cfg.selection_views),
|
|
921
|
+
)
|
|
922
|
+
elif cfg.selection == "kmedoids":
|
|
923
|
+
az = np.array([np.nan if im.sun_azimuth is None else im.sun_azimuth for im in self.images])
|
|
924
|
+
res = select_kmedoids(
|
|
925
|
+
lit[eligible],
|
|
926
|
+
az[eligible],
|
|
927
|
+
cfg.selection_clusters,
|
|
928
|
+
views=cfg.selection_views,
|
|
929
|
+
azimuth_sep=cfg.selection_azimuth_sep,
|
|
930
|
+
max_images=cfg.max_sfs_images,
|
|
931
|
+
seed=cfg.seed,
|
|
932
|
+
)
|
|
933
|
+
chosen = sorted(eligible[k] for k in res.chosen)
|
|
934
|
+
log.info(
|
|
935
|
+
"k-medoids selected %d / %d nadir images (%d medoids); uncovered %.2f%%, poorly lit %.2f%%",
|
|
936
|
+
len(chosen),
|
|
937
|
+
len(eligible),
|
|
938
|
+
len(res.medoids),
|
|
939
|
+
100 * res.uncovered,
|
|
940
|
+
100 * res.poorly_lit,
|
|
941
|
+
)
|
|
942
|
+
else:
|
|
943
|
+
chosen = sorted(set(self._image_subset()) & set(eligible))
|
|
944
|
+
chosen_set = set(chosen)
|
|
945
|
+
self.selected = [im for k, im in enumerate(self.images) if k in chosen_set]
|
|
946
|
+
self.unselected = [im for k, im in enumerate(self.images) if k not in chosen_set]
|
|
947
|
+
_write_csv(
|
|
948
|
+
out_dir / "selection.csv",
|
|
949
|
+
["product_id", "sun_azimuth", "sun_elevation", "shadow_fraction", "mapprojected", "selected"],
|
|
950
|
+
[
|
|
951
|
+
[
|
|
952
|
+
im.product_id,
|
|
953
|
+
im.sun_azimuth,
|
|
954
|
+
im.sun_elevation,
|
|
955
|
+
im.shadow_fraction,
|
|
956
|
+
im.mapprojected,
|
|
957
|
+
k in chosen_set,
|
|
958
|
+
]
|
|
959
|
+
for k, im in enumerate(self.images)
|
|
960
|
+
],
|
|
961
|
+
)
|
|
962
|
+
for name, ims in (("selected", self.selected), ("unselected", self.unselected)):
|
|
963
|
+
_write_lines(out_dir / f"{name}.txt", [im.product_id for im in ims])
|
|
964
|
+
return self.selected
|
|
965
|
+
|
|
966
|
+
def _sfs_eligible(self) -> list[int]:
|
|
967
|
+
"""Indices of the kept images allowed into SfS (``|slew| < max_sfs_slew``)."""
|
|
968
|
+
limit = self.config.max_sfs_slew
|
|
969
|
+
if limit is None:
|
|
970
|
+
return list(range(len(self.images)))
|
|
971
|
+
if any(im.slew_angle is None for im in self.images):
|
|
972
|
+
try:
|
|
973
|
+
obs = self._observations()
|
|
974
|
+
except (OSError, KeyError, ValueError) as exc:
|
|
975
|
+
log.warning("No slew angles (%s); all images count as nadir", exc)
|
|
976
|
+
obs = None
|
|
977
|
+
if obs is not None and "slew_angle" in obs:
|
|
978
|
+
slews = dict(zip(obs["product_id"].astype(str).str.upper(), obs["slew_angle"]))
|
|
979
|
+
for im in self.images:
|
|
980
|
+
v = slews.get(im.product_id.upper())
|
|
981
|
+
if im.slew_angle is None and v is not None and np.isfinite(v):
|
|
982
|
+
im.slew_angle = float(v)
|
|
983
|
+
eligible = [
|
|
984
|
+
k for k, im in enumerate(self.images) if im.slew_angle is None or abs(im.slew_angle) < limit
|
|
985
|
+
]
|
|
986
|
+
if len(eligible) < len(self.images):
|
|
987
|
+
log.info(
|
|
988
|
+
"%d slewed images (|slew| >= %g deg) kept out of SfS", len(self.images) - len(eligible), limit
|
|
989
|
+
)
|
|
990
|
+
return eligible
|
|
991
|
+
|
|
992
|
+
def _lit_masks(self) -> np.ndarray | None:
|
|
993
|
+
"""``(images, cells)`` lit mask on a common coarse grid; fills ``shadow_fraction``."""
|
|
994
|
+
bounds = asp.raster_bounds(self.dem)
|
|
995
|
+
if bounds is None or not self.images:
|
|
996
|
+
return None
|
|
997
|
+
ulx, uly, lrx, lry = bounds
|
|
998
|
+
n = self.config.selection_grid
|
|
999
|
+
w, h = abs(lrx - ulx), abs(uly - lry)
|
|
1000
|
+
size = (n, max(1, round(n * h / w))) if w >= h else (max(1, round(n * w / h)), n)
|
|
1001
|
+
thr = self.shadow_threshold
|
|
1002
|
+
masks = []
|
|
1003
|
+
for im in self.images:
|
|
1004
|
+
r = asp.read_raster(
|
|
1005
|
+
im.mapprojected or self._map_path("prj_sfs", im), projwin=bounds, outsize=size
|
|
1006
|
+
)
|
|
1007
|
+
v = r.data.ravel() if r is not None else np.full(size[0] * size[1], np.nan)
|
|
1008
|
+
valid = np.isfinite(v) & (v > -1e30)
|
|
1009
|
+
lit = valid & (v > thr)
|
|
1010
|
+
im.shadow_fraction = 1.0 - float(lit.sum()) / float(valid.sum()) if valid.any() else 1.0
|
|
1011
|
+
masks.append(lit)
|
|
1012
|
+
return np.array(masks)
|
|
1013
|
+
|
|
1014
|
+
def _image_subset(self, bin_width: float = 30.0) -> list[int]:
|
|
1015
|
+
"""ASP ``image_subset`` in each Sun-azimuth bin; returns indices of the union."""
|
|
1016
|
+
bounds = asp.raster_bounds(self.dem)
|
|
1017
|
+
win = (bounds[0], bounds[3], bounds[2], bounds[1]) if bounds else None
|
|
1018
|
+
bins: dict[int, list[int]] = {}
|
|
1019
|
+
for k, im in enumerate(self.images):
|
|
1020
|
+
bins.setdefault(int(((im.sun_azimuth or 0.0) % 360.0) // bin_width), []).append(k)
|
|
1021
|
+
by_path = {str(im.mapprojected): k for k, im in enumerate(self.images)}
|
|
1022
|
+
chosen: list[int] = []
|
|
1023
|
+
for b, idx in sorted(bins.items()):
|
|
1024
|
+
subset = asp.image_subset(
|
|
1025
|
+
[self.images[k].mapprojected or self._map_path("prj_sfs", self.images[k]) for k in idx],
|
|
1026
|
+
self.work_dir / "selection" / f"image_subset_az{int(b * bin_width):03d}.txt",
|
|
1027
|
+
threshold=self.shadow_threshold,
|
|
1028
|
+
t_projwin=win,
|
|
1029
|
+
threads=self.config.threads,
|
|
1030
|
+
)
|
|
1031
|
+
chosen += [by_path[p] for p, _ in subset if p in by_path]
|
|
1032
|
+
return sorted(set(chosen))
|
|
1033
|
+
|
|
1034
|
+
def reduce_factor(self, im: SfsImage) -> int:
|
|
1035
|
+
"""ISIS ``reduce`` factor for *im* (1 = keep native resolution)."""
|
|
1036
|
+
cfg = self.config
|
|
1037
|
+
if cfg.reduce is None:
|
|
1038
|
+
return 1
|
|
1039
|
+
if cfg.reduce == "auto":
|
|
1040
|
+
native = im.pixel_scale or cfg.native_resolution
|
|
1041
|
+
return max(1, int(round(cfg.resolution / native)))
|
|
1042
|
+
return max(1, int(cfg.reduce))
|
|
1043
|
+
|
|
1044
|
+
def sfs_inputs(self, images: Sequence[SfsImage]) -> tuple[list[Path], list[Path], Path | None, bool]:
|
|
1045
|
+
"""Cubes, cameras and adjustment prefix for SfS on *images*.
|
|
1046
|
+
|
|
1047
|
+
Cubes needing it are ``reduce``-d to the target resolution (Bertone et al. 2026,
|
|
1048
|
+
App. A: SfS's own resampling leaves artifacts). With ``sfs_cameras="csm"`` each
|
|
1049
|
+
CSM camera is re-mapped to its reduced cube (:func:`csm.isis_reduce_with_csm`);
|
|
1050
|
+
otherwise ISIS cameras are used. The final ``.adjust`` files (resolution
|
|
1051
|
+
independent) are copied under :attr:`sfs_camera_prefix` with the new names;
|
|
1052
|
+
published cameras need none. Returns ``(cubes, cameras, prefix, isis_cameras)``.
|
|
1053
|
+
"""
|
|
1054
|
+
from . import csm
|
|
1055
|
+
|
|
1056
|
+
cfg = self.config
|
|
1057
|
+
factors = [self.reduce_factor(im) for im in images]
|
|
1058
|
+
prefix = self.camera_prefix
|
|
1059
|
+
if cfg.csm and all(f == 1 for f in factors):
|
|
1060
|
+
return [im.cube for im in images], self._cameras(images), prefix, False
|
|
1061
|
+
use_csm = cfg.csm and cfg.sfs_cameras == "csm"
|
|
1062
|
+
cubes: list[Path] = []
|
|
1063
|
+
cameras: list[Path] = []
|
|
1064
|
+
for im, f in zip(images, factors):
|
|
1065
|
+
cube, camera = im.cube, im.camera if use_csm else None
|
|
1066
|
+
if f > 1 and use_csm and im.camera is not None:
|
|
1067
|
+
cube, camera = csm.isis_reduce_with_csm(
|
|
1068
|
+
im.cube, im.camera, f, self.work_dir / "images", force=self.force
|
|
1069
|
+
)
|
|
1070
|
+
elif f > 1:
|
|
1071
|
+
cube = self.work_dir / "images" / f"{im.product_id}.red{f}.cub"
|
|
1072
|
+
if not self._skip(cube):
|
|
1073
|
+
isis.reduce(im.cube, cube, f)
|
|
1074
|
+
if prefix is not None:
|
|
1075
|
+
src = asp.bundle_adjust_file_name(prefix, im.cube, im.camera if cfg.csm else None)
|
|
1076
|
+
dst = asp.bundle_adjust_file_name(self.sfs_camera_prefix, cube, camera)
|
|
1077
|
+
if not is_dry_run() and not self._skip(dst):
|
|
1078
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
1079
|
+
shutil.copyfile(src, dst)
|
|
1080
|
+
cubes.append(cube)
|
|
1081
|
+
if camera is not None:
|
|
1082
|
+
cameras.append(camera)
|
|
1083
|
+
out_prefix = self.sfs_camera_prefix if prefix is not None else None
|
|
1084
|
+
return cubes, cameras, out_prefix, not use_csm
|
|
1085
|
+
|
|
1086
|
+
def use_published_cameras(self) -> list[SfsImage]:
|
|
1087
|
+
"""Replace the cameras with the published Bertone et al. (2026) adjusted ones.
|
|
1088
|
+
|
|
1089
|
+
Each published camera (a CSM state of the reduced full image) is re-mapped to
|
|
1090
|
+
the image of its cube (:func:`reference.published_camera_native`) and written to
|
|
1091
|
+
``images/<id>.published.json``; images without one are removed.
|
|
1092
|
+
"""
|
|
1093
|
+
from . import csm
|
|
1094
|
+
from . import reference as ref
|
|
1095
|
+
|
|
1096
|
+
region = self.config.published_region
|
|
1097
|
+
if region is None:
|
|
1098
|
+
rid = self.config.region_id or ""
|
|
1099
|
+
if not (rid.startswith("A3") and rid[2:].isdigit()):
|
|
1100
|
+
raise ValueError("cameras_from: published needs published_region (or region_id A3NN)")
|
|
1101
|
+
region = int(rid[2:])
|
|
1102
|
+
out_dir = self.work_dir / "images"
|
|
1103
|
+
if is_dry_run():
|
|
1104
|
+
for im in self.images:
|
|
1105
|
+
im.camera = out_dir / f"{im.product_id}.published.json"
|
|
1106
|
+
return self.images
|
|
1107
|
+
published = ref.published_cameras(
|
|
1108
|
+
region, self.work_dir / "published", images=[im.product_id for im in self.images]
|
|
1109
|
+
)
|
|
1110
|
+
missing = [
|
|
1111
|
+
RemovedImage(im, "published", "no published camera")
|
|
1112
|
+
for im in self.images
|
|
1113
|
+
if im.product_id not in published
|
|
1114
|
+
]
|
|
1115
|
+
self._remove(missing)
|
|
1116
|
+
for im in self.images:
|
|
1117
|
+
camera = out_dir / f"{im.product_id}.published.json"
|
|
1118
|
+
if not self._skip(camera):
|
|
1119
|
+
geom = csm.CubeGeometry.from_cube(im.cube)
|
|
1120
|
+
native = (geom.alpha_samples, geom.alpha_lines)
|
|
1121
|
+
csm.write_camera(ref.published_camera_native(published[im.product_id], native), camera)
|
|
1122
|
+
im.camera = camera
|
|
1123
|
+
return self.images
|
|
1124
|
+
|
|
1125
|
+
def shape_from_shading(
|
|
1126
|
+
self,
|
|
1127
|
+
images: Sequence[SfsImage] | None = None,
|
|
1128
|
+
prefix: Path | None = None,
|
|
1129
|
+
dem: Path | None = None,
|
|
1130
|
+
**options: Any,
|
|
1131
|
+
) -> Path:
|
|
1132
|
+
"""``parallel_sfs`` on the target-grid prior DEM; returns ``<prefix>-DEM-final.tif``.
|
|
1133
|
+
|
|
1134
|
+
*images* default to :attr:`selected` (else all kept images), *prefix* to
|
|
1135
|
+
``sfs/run`` and *dem* (input DEM, e.g. a crop) to :attr:`dem`. *options*
|
|
1136
|
+
override the configured SfS options.
|
|
1137
|
+
"""
|
|
1138
|
+
images = list(images) if images is not None else (self.selected or self.images)
|
|
1139
|
+
prefix = Path(prefix) if prefix else self.sfs_prefix
|
|
1140
|
+
out = Path(f"{prefix}-DEM-final.tif")
|
|
1141
|
+
if self._skip(out):
|
|
1142
|
+
return out
|
|
1143
|
+
cubes, cameras, ba_prefix, isis_cameras = self.sfs_inputs(images)
|
|
1144
|
+
opts: dict[str, Any] = {**self.config.options("sfs"), **self._parallel_opts()}
|
|
1145
|
+
if isis_cameras:
|
|
1146
|
+
opts["use_approx_camera_models"] = True
|
|
1147
|
+
opts.update(options)
|
|
1148
|
+
return asp.sfs(
|
|
1149
|
+
dem or self.dem, cubes, prefix, cameras, parallel=True, bundle_adjust_prefix=ba_prefix, **opts
|
|
1150
|
+
)
|
|
1151
|
+
|
|
1152
|
+
def products(self, dem: Path) -> dict[str, Path]:
|
|
1153
|
+
"""Standard products via :func:`lunar_toolkit.sfs_products.make_products` when available.
|
|
1154
|
+
|
|
1155
|
+
Otherwise (module missing or its dependencies not installed) a hillshade of the
|
|
1156
|
+
SfS DEM and its difference from the prior.
|
|
1157
|
+
"""
|
|
1158
|
+
try:
|
|
1159
|
+
products = importlib.import_module(f"{__package__}.sfs_products")
|
|
1160
|
+
except ImportError as exc:
|
|
1161
|
+
log.info("sfs_products unavailable (%s); writing hillshade and diff only", exc)
|
|
1162
|
+
else:
|
|
1163
|
+
height_error = Path(f"{self.sfs_prefix}-height-error.tif")
|
|
1164
|
+
return products.make_products(
|
|
1165
|
+
dem,
|
|
1166
|
+
self.dem,
|
|
1167
|
+
self.work_dir / "products",
|
|
1168
|
+
[
|
|
1169
|
+
(im.mapprojected, im.sun_azimuth, im.pixel_scale)
|
|
1170
|
+
for im in self.selected
|
|
1171
|
+
if im.mapprojected and im.sun_azimuth is not None
|
|
1172
|
+
],
|
|
1173
|
+
region_id=self.config.region_id or self.work_dir.name,
|
|
1174
|
+
resolution=self.config.resolution,
|
|
1175
|
+
height_error=height_error if height_error.exists() else None,
|
|
1176
|
+
shadow_threshold=self.shadow_threshold,
|
|
1177
|
+
force=self.force,
|
|
1178
|
+
)
|
|
1179
|
+
out = {"dem": dem}
|
|
1180
|
+
shade = Path(f"{self.sfs_prefix}-DEM-final_hillshade.tif")
|
|
1181
|
+
out["hillshade"] = shade if self._skip(shade) else asp.hillshade(dem, shade)
|
|
1182
|
+
diff_prefix = Path(f"{self.sfs_prefix}-prior")
|
|
1183
|
+
diff = Path(f"{diff_prefix}-diff.tif")
|
|
1184
|
+
out["diff"] = diff if self._skip(diff) else asp.geodiff(dem, self.dem, diff_prefix)
|
|
1185
|
+
return out
|
|
1186
|
+
|
|
1187
|
+
def ensemble(self) -> dict[str, Path]:
|
|
1188
|
+
"""SfS ensemble over a central crop and its spread (``ensemble/``, ``products/``).
|
|
1189
|
+
|
|
1190
|
+
Members are :func:`~lunar_toolkit.sfs_products.ensemble_subsets` of the kept
|
|
1191
|
+
images; returns :func:`~lunar_toolkit.sfs_products.ensemble_uncertainty`'s maps.
|
|
1192
|
+
"""
|
|
1193
|
+
from . import sfs_products
|
|
1194
|
+
|
|
1195
|
+
cfg = self.config
|
|
1196
|
+
bounds = asp.raster_bounds(self.dem) or cfg.projwin
|
|
1197
|
+
if bounds is None:
|
|
1198
|
+
log.warning("Ensemble skipped: DEM extent unknown (dry run without projwin)")
|
|
1199
|
+
return {}
|
|
1200
|
+
ens = self.work_dir / "ensemble"
|
|
1201
|
+
crop = ens / "prior.tif"
|
|
1202
|
+
if not self._skip(crop):
|
|
1203
|
+
centre = ((bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2)
|
|
1204
|
+
asp.gdal_translate(self.dem, crop, projwin=sfs_products.crop_window(*centre, cfg.ensemble_crop))
|
|
1205
|
+
size = min(cfg.ensemble_size or len(self.selected) or len(self.images), len(self.images))
|
|
1206
|
+
subsets = sfs_products.ensemble_subsets(self.images, cfg.ensemble_members, size, seed=cfg.seed)
|
|
1207
|
+
members = [
|
|
1208
|
+
self.shape_from_shading(subset, ens / f"m{k:02d}" / "run", dem=crop)
|
|
1209
|
+
for k, subset in enumerate(subsets)
|
|
1210
|
+
]
|
|
1211
|
+
_write_lines(ens / "members.txt", [" ".join(im.product_id for im in subset) for subset in subsets])
|
|
1212
|
+
if is_dry_run():
|
|
1213
|
+
return {}
|
|
1214
|
+
return sfs_products.ensemble_uncertainty(
|
|
1215
|
+
members,
|
|
1216
|
+
self.work_dir / "products",
|
|
1217
|
+
region_id=cfg.region_id or self.work_dir.name,
|
|
1218
|
+
resolution=cfg.resolution,
|
|
1219
|
+
reference=crop,
|
|
1220
|
+
force=self.force,
|
|
1221
|
+
)
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
def flag_images(
|
|
1225
|
+
residual_stats: dict[str, asp.ResidualStats],
|
|
1226
|
+
residual_norms: dict[str, np.ndarray],
|
|
1227
|
+
offsets: dict[str, asp.MatchOffsetStats],
|
|
1228
|
+
*,
|
|
1229
|
+
max_offset_p85: float | None = 1.8,
|
|
1230
|
+
max_large_residual_fraction: float | None = 0.85,
|
|
1231
|
+
large_residual_px: float = 1.0,
|
|
1232
|
+
min_matches: int = 1,
|
|
1233
|
+
) -> dict[str, str]:
|
|
1234
|
+
"""Images to drop after bundle adjustment (Bertone et al. 2026, §2), keyed by file name.
|
|
1235
|
+
|
|
1236
|
+
An image is flagged when it has fewer than *min_matches* residuals or match offsets,
|
|
1237
|
+
an 85th-percentile mapprojected match offset over *max_offset_p85* m, or more than
|
|
1238
|
+
*max_large_residual_fraction* of its reprojection errors over *large_residual_px*.
|
|
1239
|
+
The value is the reason. The rules are the workspace's ``image_rejections`` view
|
|
1240
|
+
(:mod:`lunar_toolkit.workspace`), evaluated in an in-memory database.
|
|
1241
|
+
"""
|
|
1242
|
+
from .workspace import flag_images_sql
|
|
1243
|
+
|
|
1244
|
+
return flag_images_sql(
|
|
1245
|
+
residual_stats,
|
|
1246
|
+
residual_norms,
|
|
1247
|
+
offsets,
|
|
1248
|
+
max_offset_p85=max_offset_p85,
|
|
1249
|
+
max_large_residual_fraction=max_large_residual_fraction,
|
|
1250
|
+
large_residual_px=large_residual_px,
|
|
1251
|
+
min_matches=min_matches,
|
|
1252
|
+
)
|
|
1253
|
+
|
|
1254
|
+
|
|
1255
|
+
def select_by_illumination(
|
|
1256
|
+
lit: np.ndarray,
|
|
1257
|
+
azimuth: np.ndarray,
|
|
1258
|
+
*,
|
|
1259
|
+
views: int = 3,
|
|
1260
|
+
azimuth_sep: float = 30.0,
|
|
1261
|
+
max_images: int | None = None,
|
|
1262
|
+
min_gain: int = 1,
|
|
1263
|
+
) -> tuple[list[int], np.ndarray]:
|
|
1264
|
+
"""Greedy per-pixel illumination-diverse image selection on a raster grid.
|
|
1265
|
+
|
|
1266
|
+
The raster analogue of :func:`lunar_toolkit.selection.select_for_coverage`, after
|
|
1267
|
+
Bertone et al. (2026, §2): a lit cell is *helped* by an image lighting it if it has
|
|
1268
|
+
fewer than *views* illumination directions so far and the image's Sun azimuth is at
|
|
1269
|
+
least *azimuth_sep* from each of them. Each step takes the image helping most
|
|
1270
|
+
cells, until none helps *min_gain* cells or *max_images* are chosen.
|
|
1271
|
+
|
|
1272
|
+
Parameters
|
|
1273
|
+
----------
|
|
1274
|
+
lit : (n_images, n_cells) bool array
|
|
1275
|
+
azimuth : (n_images,) array
|
|
1276
|
+
Sun azimuths (deg); NaN never helps.
|
|
1277
|
+
|
|
1278
|
+
Returns
|
|
1279
|
+
-------
|
|
1280
|
+
(chosen, count)
|
|
1281
|
+
Image indices in pick order, and the directions reached per cell.
|
|
1282
|
+
"""
|
|
1283
|
+
lit = np.asarray(lit, bool)
|
|
1284
|
+
az = np.asarray(azimuth, float)
|
|
1285
|
+
n, cells = lit.shape
|
|
1286
|
+
dirs = np.full((cells, views), np.nan)
|
|
1287
|
+
count = np.zeros(cells, int)
|
|
1288
|
+
available = np.isfinite(az)
|
|
1289
|
+
chosen: list[int] = []
|
|
1290
|
+
while available.any() and (max_images is None or len(chosen) < max_images):
|
|
1291
|
+
open_cells = count < views
|
|
1292
|
+
best, best_cells = -1, np.empty(0, int)
|
|
1293
|
+
for k in np.flatnonzero(available):
|
|
1294
|
+
idx = np.flatnonzero(lit[k] & open_cells)
|
|
1295
|
+
if len(idx) <= len(best_cells):
|
|
1296
|
+
continue
|
|
1297
|
+
d = azimuth_difference(dirs[idx], az[k])
|
|
1298
|
+
idx = idx[np.all(np.isnan(d) | (d >= azimuth_sep), axis=1)]
|
|
1299
|
+
if len(idx) > len(best_cells):
|
|
1300
|
+
best, best_cells = int(k), idx
|
|
1301
|
+
if best < 0 or len(best_cells) < min_gain:
|
|
1302
|
+
break
|
|
1303
|
+
dirs[best_cells, count[best_cells]] = az[best]
|
|
1304
|
+
count[best_cells] += 1
|
|
1305
|
+
available[best] = False
|
|
1306
|
+
chosen.append(best)
|
|
1307
|
+
return chosen, count
|
|
1308
|
+
|
|
1309
|
+
|
|
1310
|
+
def prefilter_candidates(
|
|
1311
|
+
df: pd.DataFrame, max_slew: float | None = None, max_resolution: float | None = 1.5
|
|
1312
|
+
) -> pd.DataFrame:
|
|
1313
|
+
"""Images finer than *max_resolution* m/px (and, with *max_slew*, ``|slew| <`` it).
|
|
1314
|
+
|
|
1315
|
+
Works on ``CumIndex`` rows (``SLEW_ANGLE``, ``RESOLUTION``) or observation rows
|
|
1316
|
+
(``slew_angle``, ``pixel_scale``). Rows with unknown values are dropped. The
|
|
1317
|
+
nadir cut of Bertone et al. (2026, §2) belongs to SfS selection
|
|
1318
|
+
(:attr:`SfsConfig.max_sfs_slew`), so searches keep slewed images by default: they
|
|
1319
|
+
give the stereo convergence the registration step needs.
|
|
1320
|
+
"""
|
|
1321
|
+
import pandas as pd
|
|
1322
|
+
|
|
1323
|
+
keep = pd.Series(True, index=df.index)
|
|
1324
|
+
slew_col = next((c for c in ("SLEW_ANGLE", "slew_angle") if c in df), None)
|
|
1325
|
+
if max_slew is not None:
|
|
1326
|
+
if slew_col:
|
|
1327
|
+
keep &= pd.to_numeric(df[slew_col], errors="coerce").abs() < max_slew
|
|
1328
|
+
else:
|
|
1329
|
+
log.warning("No slew column; slew filter skipped")
|
|
1330
|
+
res_col = next((c for c in ("RESOLUTION", "pixel_scale", "PIXEL_SCALE") if c in df), None)
|
|
1331
|
+
if max_resolution is not None and res_col:
|
|
1332
|
+
keep &= pd.to_numeric(df[res_col], errors="coerce") < max_resolution
|
|
1333
|
+
return df[keep]
|
|
1334
|
+
|
|
1335
|
+
|
|
1336
|
+
def cap_candidates(
|
|
1337
|
+
obs: pd.DataFrame,
|
|
1338
|
+
aoi: Any,
|
|
1339
|
+
max_images: int,
|
|
1340
|
+
*,
|
|
1341
|
+
max_sfs_slew: float | None = 5.0,
|
|
1342
|
+
stereo_pairs: int = 30,
|
|
1343
|
+
stereo_convergence: tuple[float, float] = (5.0, 60.0),
|
|
1344
|
+
views: int = 3,
|
|
1345
|
+
azimuth_sep: float = 30.0,
|
|
1346
|
+
) -> pd.DataFrame:
|
|
1347
|
+
"""At most *max_images* candidate images, chosen before anything is downloaded.
|
|
1348
|
+
|
|
1349
|
+
Keeps both images of the best stereo pairs (up to half the budget; they feed the
|
|
1350
|
+
registration step), then nadir images (``|slew| < max_sfs_slew``) chosen for
|
|
1351
|
+
illumination coverage over *aoi* (:func:`~lunar_toolkit.selection.select_for_coverage`:
|
|
1352
|
+
*views* directions at least *azimuth_sep* apart per cell), then the finest remaining
|
|
1353
|
+
nadir images. Returns the chosen rows in their original order.
|
|
1354
|
+
"""
|
|
1355
|
+
from .selection import select_for_coverage
|
|
1356
|
+
|
|
1357
|
+
if len(obs) <= max_images:
|
|
1358
|
+
return obs
|
|
1359
|
+
ids = obs["product_id"].astype(str).str.upper()
|
|
1360
|
+
chosen: list[str] = []
|
|
1361
|
+
|
|
1362
|
+
def take(new: Sequence[str], limit: int) -> None:
|
|
1363
|
+
for pid in new:
|
|
1364
|
+
if len(chosen) >= limit:
|
|
1365
|
+
return
|
|
1366
|
+
if pid not in chosen:
|
|
1367
|
+
chosen.append(pid)
|
|
1368
|
+
|
|
1369
|
+
if stereo_pairs:
|
|
1370
|
+
pairs = _candidate_pairs(obs, aoi, stereo_convergence)
|
|
1371
|
+
stereo_ids = [
|
|
1372
|
+
str(v).upper()
|
|
1373
|
+
for pair in pairs.head(stereo_pairs).itertuples()
|
|
1374
|
+
for v in (pair.product_id_1, pair.product_id_2)
|
|
1375
|
+
]
|
|
1376
|
+
take(stereo_ids, max_images // 2)
|
|
1377
|
+
slew = _numeric(obs.get("slew_angle"))
|
|
1378
|
+
nadir = obs[(slew.abs() < max_sfs_slew) | slew.isna()] if max_sfs_slew is not None else obs
|
|
1379
|
+
nadir = nadir[~nadir["product_id"].astype(str).str.upper().isin(chosen)]
|
|
1380
|
+
if len(nadir):
|
|
1381
|
+
cover = select_for_coverage(
|
|
1382
|
+
nadir, aoi, azimuth_sep_deg=azimuth_sep, views_per_cell=views, max_images=max_images - len(chosen)
|
|
1383
|
+
)
|
|
1384
|
+
take(cover["product_id"].astype(str).str.upper().tolist(), max_images)
|
|
1385
|
+
rest = nadir.assign(_res=_numeric(nadir.get("pixel_scale"))).sort_values("_res")
|
|
1386
|
+
take(rest["product_id"].astype(str).str.upper().tolist(), max_images)
|
|
1387
|
+
log.info("Capped %d candidates to %d (stereo pairs + illumination coverage)", len(obs), len(chosen))
|
|
1388
|
+
return obs[ids.isin(set(chosen)).to_numpy()]
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
def _candidate_pairs(obs: pd.DataFrame, aoi: Any, convergence: tuple[float, float]) -> pd.DataFrame:
|
|
1392
|
+
from .stereo import find_stereo_pairs
|
|
1393
|
+
from .workspace import PAIRS_ENGINE_THRESHOLD, find_stereo_pairs_duckdb, spatial_available
|
|
1394
|
+
|
|
1395
|
+
if len(obs) >= PAIRS_ENGINE_THRESHOLD and spatial_available():
|
|
1396
|
+
return find_stereo_pairs_duckdb(obs, aoi, convergence=convergence)
|
|
1397
|
+
return find_stereo_pairs(obs, aoi=aoi, convergence=convergence)
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
def _numeric(values: Any) -> pd.Series:
|
|
1401
|
+
import pandas as pd
|
|
1402
|
+
|
|
1403
|
+
if values is None:
|
|
1404
|
+
return pd.Series(dtype=float)
|
|
1405
|
+
return pd.to_numeric(values, errors="coerce")
|
|
1406
|
+
|
|
1407
|
+
|
|
1408
|
+
def search_images(
|
|
1409
|
+
aoi: Any,
|
|
1410
|
+
*,
|
|
1411
|
+
max_slew: float | None = None,
|
|
1412
|
+
max_resolution: float | None = 1.5,
|
|
1413
|
+
max_images: int | None = None,
|
|
1414
|
+
config: SfsConfig | None = None,
|
|
1415
|
+
data_dir: Path | None = None,
|
|
1416
|
+
) -> pd.DataFrame:
|
|
1417
|
+
"""NAC EDRs over *aoi* from the local ``CumIndex``, pre-filtered and capped; observation schema.
|
|
1418
|
+
|
|
1419
|
+
*config* supplies the SfS slew cut and stereo settings used by :func:`cap_candidates`.
|
|
1420
|
+
"""
|
|
1421
|
+
from .lroc.cumindex import CumIndex
|
|
1422
|
+
|
|
1423
|
+
cfg = config or SfsConfig()
|
|
1424
|
+
ci = CumIndex("edr", data_dir=data_dir)
|
|
1425
|
+
df = ci.search(aoi=aoi, nac_only=True, wkt=True)
|
|
1426
|
+
kept = prefilter_candidates(df, max_slew, max_resolution)
|
|
1427
|
+
log.info("%d NAC images over the AOI, %d after the resolution cut", len(df), len(kept))
|
|
1428
|
+
obs = ci.to_observations(kept)
|
|
1429
|
+
if max_images is not None:
|
|
1430
|
+
obs = cap_candidates(
|
|
1431
|
+
obs,
|
|
1432
|
+
aoi,
|
|
1433
|
+
max_images,
|
|
1434
|
+
max_sfs_slew=cfg.max_sfs_slew,
|
|
1435
|
+
stereo_pairs=cfg.stereo_pairs if cfg.registration == "stereo" else 0,
|
|
1436
|
+
stereo_convergence=cfg.stereo_convergence,
|
|
1437
|
+
views=cfg.selection_views,
|
|
1438
|
+
azimuth_sep=cfg.selection_azimuth_sep,
|
|
1439
|
+
)
|
|
1440
|
+
return obs
|
|
1441
|
+
|
|
1442
|
+
|
|
1443
|
+
@dataclass(frozen=True)
|
|
1444
|
+
class PolarStereographic:
|
|
1445
|
+
"""A polar stereographic projection (sphere, true scale at the pole) from a PROJ string."""
|
|
1446
|
+
|
|
1447
|
+
hemisphere: str
|
|
1448
|
+
lon_0: float = 0.0
|
|
1449
|
+
radius: float = MOON_RADIUS_KM * 1000.0
|
|
1450
|
+
x_0: float = 0.0
|
|
1451
|
+
y_0: float = 0.0
|
|
1452
|
+
|
|
1453
|
+
@classmethod
|
|
1454
|
+
def from_proj4(cls, proj4: str) -> PolarStereographic:
|
|
1455
|
+
"""Parse e.g. ``+proj=stere +lat_0=-90 +lon_0=0 +k=1 +x_0=0 +y_0=0 +R=1737400 +units=m``.
|
|
1456
|
+
|
|
1457
|
+
Raises
|
|
1458
|
+
------
|
|
1459
|
+
ValueError
|
|
1460
|
+
For anything else (other projections, oblique stereographic, scale ≠ 1, non-metre units).
|
|
1461
|
+
"""
|
|
1462
|
+
p: dict[str, str] = dict(re.findall(r"\+(\w+)(?:=(\S+))?", proj4))
|
|
1463
|
+
lat_0 = float(p.get("lat_0", "nan"))
|
|
1464
|
+
k = float(p.get("k", p.get("k_0", "1")))
|
|
1465
|
+
lat_ts = float(p["lat_ts"]) if "lat_ts" in p else lat_0
|
|
1466
|
+
sphere_a = p.get("a") if p.get("a") == p.get("b", p.get("a")) else None
|
|
1467
|
+
radius = float(p.get("R") or sphere_a or "nan")
|
|
1468
|
+
if (
|
|
1469
|
+
p.get("proj") not in ("stere", "ups")
|
|
1470
|
+
or not math.isclose(abs(lat_0), 90.0)
|
|
1471
|
+
or not math.isclose(k, 1.0)
|
|
1472
|
+
or not math.isclose(lat_ts, lat_0)
|
|
1473
|
+
or p.get("units", "m") != "m"
|
|
1474
|
+
or not math.isfinite(radius)
|
|
1475
|
+
):
|
|
1476
|
+
raise ValueError(
|
|
1477
|
+
f"Expected a spherical polar stereographic projection in metres, got {proj4!r}; "
|
|
1478
|
+
"pass --projwin in the DEM's own units instead"
|
|
1479
|
+
)
|
|
1480
|
+
return cls(
|
|
1481
|
+
"S" if lat_0 < 0 else "N",
|
|
1482
|
+
float(p.get("lon_0", 0.0)),
|
|
1483
|
+
radius,
|
|
1484
|
+
float(p.get("x_0", 0.0)),
|
|
1485
|
+
float(p.get("y_0", 0.0)),
|
|
1486
|
+
)
|
|
1487
|
+
|
|
1488
|
+
def xy(self, lat: float, lon: float) -> tuple[float, float]:
|
|
1489
|
+
x, y = ps_xy(lat, lon - self.lon_0, self.hemisphere)
|
|
1490
|
+
scale = self.radius / (MOON_RADIUS_KM * 1000.0)
|
|
1491
|
+
return x * scale + self.x_0, y * scale + self.y_0
|
|
1492
|
+
|
|
1493
|
+
|
|
1494
|
+
def aoi_projwin(aoi: Any, dem: Path, snap: float | None = None) -> tuple[float, float, float, float]:
|
|
1495
|
+
"""``(ulx, uly, lrx, lry)`` bounding *aoi* in the polar stereographic projection of *dem*.
|
|
1496
|
+
|
|
1497
|
+
*aoi* is anything :func:`lunar_toolkit.geometry.load_aoi` takes (GeoJSON / WKT file,
|
|
1498
|
+
WKT string). The window is widened to multiples of *snap* (m) when given.
|
|
1499
|
+
"""
|
|
1500
|
+
proj = PolarStereographic.from_proj4(asp.srs_proj4(dem))
|
|
1501
|
+
pts = [proj.xy(lat, lon) for poly in load_aoi(aoi).polygons for lon, lat in poly[0]]
|
|
1502
|
+
xs, ys = [p[0] for p in pts], [p[1] for p in pts]
|
|
1503
|
+
x0, x1, y0, y1 = min(xs), max(xs), min(ys), max(ys)
|
|
1504
|
+
if snap:
|
|
1505
|
+
x0, y0 = math.floor(x0 / snap) * snap, math.floor(y0 / snap) * snap
|
|
1506
|
+
x1, y1 = math.ceil(x1 / snap) * snap, math.ceil(y1 / snap) * snap
|
|
1507
|
+
return x0, y1, x1, y0
|
|
1508
|
+
|
|
1509
|
+
|
|
1510
|
+
def _by_id(observations: pd.DataFrame | None, column: str) -> dict[str, float]:
|
|
1511
|
+
"""Finite values of a numeric observation *column* keyed by upper-case product ID."""
|
|
1512
|
+
if observations is None or column not in observations:
|
|
1513
|
+
return {}
|
|
1514
|
+
import pandas as pd
|
|
1515
|
+
|
|
1516
|
+
values = pd.to_numeric(observations[column], errors="coerce")
|
|
1517
|
+
ids = observations["product_id"].astype(str).str.upper()
|
|
1518
|
+
return {i: float(v) for i, v in zip(ids, values) if np.isfinite(v)}
|
|
1519
|
+
|
|
1520
|
+
|
|
1521
|
+
def _write_csv(path: Path, header: list[str], rows: list[list[Any]]) -> None:
|
|
1522
|
+
if is_dry_run():
|
|
1523
|
+
return
|
|
1524
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1525
|
+
with open(path, "w", newline="") as f:
|
|
1526
|
+
w = csv.writer(f)
|
|
1527
|
+
w.writerow(header)
|
|
1528
|
+
w.writerows([["" if v is None else v for v in r] for r in rows])
|
|
1529
|
+
|
|
1530
|
+
|
|
1531
|
+
def _write_lines(path: Path, lines: list[str]) -> None:
|
|
1532
|
+
if not is_dry_run():
|
|
1533
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1534
|
+
path.write_text("".join(f"{ln}\n" for ln in lines))
|