lunar-toolkit 0.1.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lunar_toolkit/__init__.py +6 -0
- lunar_toolkit/asp.py +432 -0
- lunar_toolkit/cli/__init__.py +88 -0
- lunar_toolkit/cli/_jobs.py +437 -0
- lunar_toolkit/cli/_status.py +368 -0
- lunar_toolkit/cli/_utils.py +128 -0
- lunar_toolkit/cli/app.py +205 -0
- lunar_toolkit/cli/build_db.py +130 -0
- lunar_toolkit/cli/download.py +111 -0
- lunar_toolkit/cli/kaguya.py +253 -0
- lunar_toolkit/cli/ode_search.py +144 -0
- lunar_toolkit/cli/pairs.py +236 -0
- lunar_toolkit/cli/process.py +138 -0
- lunar_toolkit/cli/query.py +70 -0
- lunar_toolkit/cli/reference.py +233 -0
- lunar_toolkit/cli/search.py +90 -0
- lunar_toolkit/cli/serve.py +706 -0
- lunar_toolkit/cli/sfs.py +344 -0
- lunar_toolkit/cli/sfs_report.py +148 -0
- lunar_toolkit/cli/shadowcam.py +246 -0
- lunar_toolkit/cli/stereo.py +85 -0
- lunar_toolkit/cli/workspace.py +84 -0
- lunar_toolkit/config.py +345 -0
- lunar_toolkit/constants.py +217 -0
- lunar_toolkit/csm.py +340 -0
- lunar_toolkit/downloader.py +447 -0
- lunar_toolkit/geometry.py +556 -0
- lunar_toolkit/isis.py +222 -0
- lunar_toolkit/kaguya/__init__.py +45 -0
- lunar_toolkit/kaguya/mi.py +274 -0
- lunar_toolkit/kaguya/tc.py +370 -0
- lunar_toolkit/lroc/__init__.py +6 -0
- lunar_toolkit/lroc/cumindex.py +1197 -0
- lunar_toolkit/lroc/nac.py +702 -0
- lunar_toolkit/observation.py +127 -0
- lunar_toolkit/ode.py +485 -0
- lunar_toolkit/provenance.py +312 -0
- lunar_toolkit/reference.py +1111 -0
- lunar_toolkit/selection.py +714 -0
- lunar_toolkit/sfs.py +1534 -0
- lunar_toolkit/sfs_products.py +676 -0
- lunar_toolkit/sfs_report.py +1328 -0
- lunar_toolkit/shadowcam/__init__.py +29 -0
- lunar_toolkit/shadowcam/sc.py +392 -0
- lunar_toolkit/static/index.html +3259 -0
- lunar_toolkit/stereo.py +519 -0
- lunar_toolkit/stereo_dem.py +145 -0
- lunar_toolkit/tools.py +264 -0
- lunar_toolkit/workspace.py +1173 -0
- lunar_toolkit-0.1.0.dev2.dist-info/METADATA +177 -0
- lunar_toolkit-0.1.0.dev2.dist-info/RECORD +55 -0
- lunar_toolkit-0.1.0.dev2.dist-info/WHEEL +5 -0
- lunar_toolkit-0.1.0.dev2.dist-info/entry_points.txt +2 -0
- lunar_toolkit-0.1.0.dev2.dist-info/licenses/LICENSE +28 -0
- lunar_toolkit-0.1.0.dev2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1328 @@
|
|
|
1
|
+
"""Validation report of an SfS work dir: metrics, figures and HTML (Bertone et al. 2026, Sec. 3)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import html
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
import math
|
|
10
|
+
import warnings
|
|
11
|
+
from collections.abc import Callable, Iterable, Sequence
|
|
12
|
+
from dataclasses import dataclass, field, fields
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
import pandas as pd
|
|
18
|
+
|
|
19
|
+
from . import asp
|
|
20
|
+
from .sfs_products import Grid, illumination_maps, product_name, read_raster, slope_deg
|
|
21
|
+
from .tools import ToolError
|
|
22
|
+
|
|
23
|
+
log = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
SERIES = ("#2a78d6", "#eb6834", "#1baf7a")
|
|
26
|
+
COVERAGE_COLORS = ("#000000", "#e34948", "#eda100", "#b8b8b8")
|
|
27
|
+
COVERAGE_LABELS = ("0", "1", "2", ">=3")
|
|
28
|
+
DIFF_EDGES = np.linspace(-5.0, 5.0, 101)
|
|
29
|
+
NORM_EDGES = np.linspace(-5.0, 5.0, 101)
|
|
30
|
+
SLOPE_EDGES = (0, 5, 10, 15, 20, 25, 30, 35, 40, 90)
|
|
31
|
+
BA_STAGES = (("ba", "ba/run"), ("refine", "ba_refine/run"))
|
|
32
|
+
|
|
33
|
+
SECTIONS = {
|
|
34
|
+
"ba": "Bundle adjustment residuals and match offsets (Fig. 5)",
|
|
35
|
+
"removed": "Removed images (Sec. 2)",
|
|
36
|
+
"coverage": "Illumination coverage (Figs. 3, 6)",
|
|
37
|
+
"prior": "SfS minus prior DEM (Table 2, Fig. 7 left)",
|
|
38
|
+
"lola": "SfS minus LOLA points vs slope (Table 2, Fig. 7)",
|
|
39
|
+
"ensemble": "Ensemble uncertainty (Fig. 8)",
|
|
40
|
+
"uncertainty": "Residuals normalised by the LDEM uncertainty maps (Fig. 9)",
|
|
41
|
+
"published": "Comparison with the published SDEM (GLDELEV)",
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class ReportInputs:
|
|
47
|
+
"""Files and settings of a report; anything None is discovered or its section skipped.
|
|
48
|
+
|
|
49
|
+
Parameters
|
|
50
|
+
----------
|
|
51
|
+
work_dir : Path
|
|
52
|
+
An :class:`~lunar_toolkit.sfs.SfsPipeline` work directory.
|
|
53
|
+
sfs_dem, prior_dem : Path, optional
|
|
54
|
+
Default ``products/<ROI>_GLDELEV_<RES>.tif`` (else ``sfs/run-DEM-final.tif``)
|
|
55
|
+
and ``dem/prior-<RES>m.tif``.
|
|
56
|
+
region_id, resolution : optional
|
|
57
|
+
Product name parts (default: ``config.yaml``, else the work dir name and 5).
|
|
58
|
+
shadow_threshold, max_offset_p85 : float, optional
|
|
59
|
+
Default from ``config.yaml`` (or the Bertone preset).
|
|
60
|
+
ldec, toterr, slperr, ldem_slope : Path, optional
|
|
61
|
+
PGDA LOLA count, elevation / slope uncertainty and slope rasters.
|
|
62
|
+
lola_points : Path, optional
|
|
63
|
+
ASP CSV of LOLA points (:func:`lunar_toolkit.reference.write_asp_csv`).
|
|
64
|
+
lola_xyzi : Path, optional
|
|
65
|
+
PGDA ``.xyzi`` point cloud, converted to *lola_points* when those are not given.
|
|
66
|
+
published_dem : Path, optional
|
|
67
|
+
Published Bertone et al. SDEM (``<ROI>_GLDELEV_005.tif``).
|
|
68
|
+
slope_edges : sequence of float
|
|
69
|
+
Slope bins (deg) for the differences vs slope.
|
|
70
|
+
slope_split : float
|
|
71
|
+
Slope (deg) separating the low / high-slope ensemble regimes.
|
|
72
|
+
preview_size : int
|
|
73
|
+
Longer side (px) of the grid maps and coverage comparisons are drawn on.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
work_dir: Path
|
|
77
|
+
sfs_dem: Path | None = None
|
|
78
|
+
prior_dem: Path | None = None
|
|
79
|
+
region_id: str | None = None
|
|
80
|
+
resolution: float | None = None
|
|
81
|
+
shadow_threshold: float | None = None
|
|
82
|
+
max_offset_p85: float | None = None
|
|
83
|
+
ldec: Path | None = None
|
|
84
|
+
toterr: Path | None = None
|
|
85
|
+
slperr: Path | None = None
|
|
86
|
+
ldem_slope: Path | None = None
|
|
87
|
+
lola_points: Path | None = None
|
|
88
|
+
lola_xyzi: Path | None = None
|
|
89
|
+
published_dem: Path | None = None
|
|
90
|
+
slope_edges: Sequence[float] = SLOPE_EDGES
|
|
91
|
+
slope_split: float = 20.0
|
|
92
|
+
preview_size: int = 800
|
|
93
|
+
|
|
94
|
+
@classmethod
|
|
95
|
+
def from_work_dir(cls, work_dir: Path, **overrides: Any) -> ReportInputs:
|
|
96
|
+
"""Inputs with defaults read from ``<work_dir>/config.yaml`` and the standard paths."""
|
|
97
|
+
work_dir = Path(work_dir)
|
|
98
|
+
kw: dict[str, Any] = {k: v for k, v in overrides.items() if v is not None}
|
|
99
|
+
cfg = _work_dir_config(work_dir)
|
|
100
|
+
if cfg is not None:
|
|
101
|
+
kw.setdefault("resolution", cfg.resolution)
|
|
102
|
+
kw.setdefault("region_id", cfg.region_id)
|
|
103
|
+
kw.setdefault("shadow_threshold", cfg.options("sfs").get("shadow_threshold"))
|
|
104
|
+
kw.setdefault("max_offset_p85", cfg.max_offset_p85)
|
|
105
|
+
inp = cls(work_dir, **{k: v for k, v in kw.items() if v is not None})
|
|
106
|
+
inp.resolution = inp.resolution or 5.0
|
|
107
|
+
inp.region_id = inp.region_id or work_dir.resolve().name
|
|
108
|
+
inp.shadow_threshold = 0.002 if inp.shadow_threshold is None else inp.shadow_threshold
|
|
109
|
+
inp.max_offset_p85 = 1.8 if inp.max_offset_p85 is None else inp.max_offset_p85
|
|
110
|
+
if inp.sfs_dem is None:
|
|
111
|
+
inp.sfs_dem = _first_existing(inp.product("ELEV"), work_dir / "sfs" / "run-DEM-final.tif")
|
|
112
|
+
if inp.prior_dem is None:
|
|
113
|
+
inp.prior_dem = _first_existing(work_dir / "dem" / f"prior-{inp.resolution:g}m.tif")
|
|
114
|
+
return inp
|
|
115
|
+
|
|
116
|
+
def product(self, code: str, *suffix: Any) -> Path:
|
|
117
|
+
"""``<work_dir>/products/<ROI>_GLD<code>_<RES>[_suffix].tif``."""
|
|
118
|
+
return (
|
|
119
|
+
self.work_dir
|
|
120
|
+
/ "products"
|
|
121
|
+
/ product_name(self.region_id or "", code, self.resolution or 5, *suffix)
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _work_dir_config(work_dir: Path) -> Any:
|
|
126
|
+
path = work_dir / "config.yaml"
|
|
127
|
+
if not path.exists():
|
|
128
|
+
return None
|
|
129
|
+
from .config import ConfigError, load_config
|
|
130
|
+
|
|
131
|
+
try:
|
|
132
|
+
return load_config(path).sfs
|
|
133
|
+
except ConfigError as exc:
|
|
134
|
+
log.warning("Ignoring %s: %s", path, exc)
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _first_existing(*paths: Path) -> Path | None:
|
|
139
|
+
return next((p for p in paths if p.exists()), None)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@dataclass
|
|
143
|
+
class Histogram:
|
|
144
|
+
"""Counts of *values* in *edges*; values beyond the edges go to the outer bins."""
|
|
145
|
+
|
|
146
|
+
label: str
|
|
147
|
+
edges: np.ndarray
|
|
148
|
+
counts: np.ndarray
|
|
149
|
+
|
|
150
|
+
@classmethod
|
|
151
|
+
def of(cls, label: str, values: np.ndarray, edges: np.ndarray | Sequence[float]) -> Histogram:
|
|
152
|
+
e = np.asarray(edges, float)
|
|
153
|
+
v = np.asarray(values, float)
|
|
154
|
+
v = np.clip(v[np.isfinite(v)], e[0], e[-1])
|
|
155
|
+
return cls(label, e, np.histogram(v, e)[0])
|
|
156
|
+
|
|
157
|
+
@property
|
|
158
|
+
def total(self) -> int:
|
|
159
|
+
return int(self.counts.sum())
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@dataclass
|
|
163
|
+
class Metrics:
|
|
164
|
+
"""Everything a report shows: JSON summary, tables and histograms per section."""
|
|
165
|
+
|
|
166
|
+
inputs: ReportInputs
|
|
167
|
+
summary: dict[str, Any] = field(default_factory=dict)
|
|
168
|
+
tables: dict[str, pd.DataFrame] = field(default_factory=dict)
|
|
169
|
+
histograms: dict[str, list[Histogram]] = field(default_factory=dict)
|
|
170
|
+
hist2d: dict[str, tuple[np.ndarray, np.ndarray, np.ndarray]] = field(default_factory=dict)
|
|
171
|
+
skipped: dict[str, str] = field(default_factory=dict)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def stats(values: Any) -> dict[str, float]:
|
|
175
|
+
"""Count, mean, median, std, NMAD, RMS, range, 5/95th percentiles and the same of ``|values|``."""
|
|
176
|
+
v = np.asarray(values, float).ravel()
|
|
177
|
+
v = v[np.isfinite(v)]
|
|
178
|
+
if v.size == 0:
|
|
179
|
+
return {"count": 0}
|
|
180
|
+
med = float(np.median(v))
|
|
181
|
+
a = np.abs(v)
|
|
182
|
+
return {
|
|
183
|
+
"count": int(v.size),
|
|
184
|
+
"mean": float(v.mean()),
|
|
185
|
+
"median": med,
|
|
186
|
+
"std": float(v.std()),
|
|
187
|
+
"nmad": float(1.4826 * np.median(np.abs(v - med))),
|
|
188
|
+
"rms": float(np.sqrt(np.mean(v * v))),
|
|
189
|
+
"min": float(v.min()),
|
|
190
|
+
"max": float(v.max()),
|
|
191
|
+
"p05": float(np.percentile(v, 5)),
|
|
192
|
+
"p95": float(np.percentile(v, 95)),
|
|
193
|
+
"abs_mean": float(a.mean()),
|
|
194
|
+
"abs_median": float(np.median(a)),
|
|
195
|
+
"abs_std": float(a.std()),
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def split_stats(values: np.ndarray, ldec: np.ndarray | None = None) -> dict[str, dict[str, float]]:
|
|
200
|
+
""":func:`stats` of all values and, with a LOLA count raster, of ``LDEC > 0`` / ``LDEC = 0`` pixels."""
|
|
201
|
+
out = {"all": stats(values)}
|
|
202
|
+
if ldec is not None:
|
|
203
|
+
ok = np.isfinite(ldec)
|
|
204
|
+
out["ldec>0"] = stats(values[ok & (ldec > 0)])
|
|
205
|
+
out["ldec=0"] = stats(values[ok & (ldec <= 0)])
|
|
206
|
+
return out
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def binned(values: np.ndarray, by: np.ndarray, edges: Sequence[float]) -> pd.DataFrame:
|
|
210
|
+
"""Per bin of *by*: count, median, quartiles, mean ``|values|`` and RMS of *values*."""
|
|
211
|
+
v, b = np.asarray(values, float).ravel(), np.asarray(by, float).ravel()
|
|
212
|
+
ok = np.isfinite(v) & np.isfinite(b)
|
|
213
|
+
v, b = v[ok], b[ok]
|
|
214
|
+
rows = []
|
|
215
|
+
for lo, hi in zip(edges[:-1], edges[1:]):
|
|
216
|
+
x = v[(b >= lo) & (b < hi)]
|
|
217
|
+
row: dict[str, Any] = {"bin_min": lo, "bin_max": hi, "count": int(x.size)}
|
|
218
|
+
if x.size:
|
|
219
|
+
p25, p50, p75 = np.percentile(x, [25, 50, 75])
|
|
220
|
+
row.update(
|
|
221
|
+
median=p50,
|
|
222
|
+
p25=p25,
|
|
223
|
+
p75=p75,
|
|
224
|
+
abs_mean=float(np.abs(x).mean()),
|
|
225
|
+
rms=float(np.sqrt(np.mean(x * x))),
|
|
226
|
+
)
|
|
227
|
+
rows.append(row)
|
|
228
|
+
return pd.DataFrame(
|
|
229
|
+
rows, columns=["bin_min", "bin_max", "count", "median", "p25", "p75", "abs_mean", "rms"]
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def normal_summary(z: np.ndarray) -> dict[str, float]:
|
|
234
|
+
"""How close normalised residuals are to a unit Gaussian: std, NMAD and fractions within 1-3 sigma."""
|
|
235
|
+
z = np.asarray(z, float).ravel()
|
|
236
|
+
z = z[np.isfinite(z)]
|
|
237
|
+
if z.size == 0:
|
|
238
|
+
return {"count": 0}
|
|
239
|
+
a = np.abs(z)
|
|
240
|
+
return {
|
|
241
|
+
"count": int(z.size),
|
|
242
|
+
"mean": float(z.mean()),
|
|
243
|
+
"std": float(z.std()),
|
|
244
|
+
"nmad": float(1.4826 * np.median(np.abs(z - np.median(z)))),
|
|
245
|
+
"within_1": float(np.mean(a < 1)),
|
|
246
|
+
"within_2": float(np.mean(a < 2)),
|
|
247
|
+
"within_3": float(np.mean(a < 3)),
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def coverage_fractions(solar_bins: np.ndarray, valid: np.ndarray | None = None) -> dict[str, float]:
|
|
252
|
+
"""Fractions of pixels lit from 0, 1, 2 and >=3 non-redundant Sun directions (and >=1, >=2)."""
|
|
253
|
+
b = np.asarray(solar_bins)
|
|
254
|
+
b = b[valid] if valid is not None else b.ravel()
|
|
255
|
+
n = b.size
|
|
256
|
+
if n == 0:
|
|
257
|
+
return {"pixels": 0}
|
|
258
|
+
f = [float(np.mean(b == k)) for k in (0, 1, 2)] + [float(np.mean(b >= 3))]
|
|
259
|
+
return {
|
|
260
|
+
"pixels": int(n),
|
|
261
|
+
"frac_0": f[0],
|
|
262
|
+
"frac_1": f[1],
|
|
263
|
+
"frac_2": f[2],
|
|
264
|
+
"frac_3plus": f[3],
|
|
265
|
+
"frac_ge1": f[1] + f[2] + f[3],
|
|
266
|
+
"frac_ge2": f[2] + f[3],
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def read_ba_reports(prefix: Path) -> pd.DataFrame | None:
|
|
271
|
+
"""Per-image bundle-adjustment report of ``<prefix>-*``: residuals before / after and match offsets.
|
|
272
|
+
|
|
273
|
+
Columns ``image``, ``initial_mean`` / ``initial_median`` / ``initial_count``,
|
|
274
|
+
``final_*`` (pixels) and ``offset_p25`` ... ``offset_p95``, ``offset_count`` (m).
|
|
275
|
+
None when none of the reports exist.
|
|
276
|
+
"""
|
|
277
|
+
files = {
|
|
278
|
+
"initial": Path(f"{prefix}-initial_residuals_stats.txt"),
|
|
279
|
+
"final": Path(f"{prefix}-final_residuals_stats.txt"),
|
|
280
|
+
"offsets": Path(f"{prefix}-mapproj_match_offset_stats.txt"),
|
|
281
|
+
}
|
|
282
|
+
if not any(p.exists() for p in files.values()):
|
|
283
|
+
return None
|
|
284
|
+
rows: dict[str, dict[str, Any]] = {}
|
|
285
|
+
for key in ("initial", "final"):
|
|
286
|
+
if files[key].exists():
|
|
287
|
+
for name, s in asp.read_residual_stats(files[key]).items():
|
|
288
|
+
rows.setdefault(name, {}).update(
|
|
289
|
+
{f"{key}_mean": s.mean, f"{key}_median": s.median, f"{key}_count": s.count}
|
|
290
|
+
)
|
|
291
|
+
if files["offsets"].exists():
|
|
292
|
+
for name, o in asp.read_mapproj_offset_stats(files["offsets"]).items():
|
|
293
|
+
rows.setdefault(name, {}).update(
|
|
294
|
+
{f"offset_p{p}": getattr(o, f"p{p}") for p in (25, 50, 75, 85, 95)}, offset_count=o.count
|
|
295
|
+
)
|
|
296
|
+
cols = ["image"] + [f"{k}_{s}" for k in ("initial", "final") for s in ("mean", "median", "count")]
|
|
297
|
+
cols += [f"offset_p{p}" for p in (25, 50, 75, 85, 95)] + ["offset_count"]
|
|
298
|
+
df = pd.DataFrame([{"image": k, **v} for k, v in sorted(rows.items())], columns=cols)
|
|
299
|
+
offsets = [c for c in cols if c.startswith("offset_p")]
|
|
300
|
+
df[offsets] = df[offsets].where(df[offsets] >= 0)
|
|
301
|
+
return df
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def ba_summary(table: pd.DataFrame, resolution: float, max_offset_p85: float) -> dict[str, Any]:
|
|
305
|
+
"""Fig. 5 numbers: per-image median residual mean +- std before / after, offset percentiles."""
|
|
306
|
+
out: dict[str, Any] = {"images": int(len(table))}
|
|
307
|
+
for key in ("initial", "final"):
|
|
308
|
+
med = table[f"{key}_median"].dropna()
|
|
309
|
+
if len(med):
|
|
310
|
+
out[f"{key}_median_px"] = {
|
|
311
|
+
"mean": float(med.mean()),
|
|
312
|
+
"std": float(med.std(ddof=0)),
|
|
313
|
+
"median": float(med.median()),
|
|
314
|
+
}
|
|
315
|
+
p85, p95 = table["offset_p85"].dropna(), table["offset_p95"].dropna()
|
|
316
|
+
if len(p85):
|
|
317
|
+
out["offset_p85_m"] = {"median": float(p85.median()), "max": float(p85.max())}
|
|
318
|
+
out["offset_p85_over_threshold"] = int((p85 > max_offset_p85).sum())
|
|
319
|
+
out["offset_threshold_m"] = max_offset_p85
|
|
320
|
+
if len(p95):
|
|
321
|
+
out["offset_p95_m"] = {"median": float(p95.median()), "max": float(p95.max())}
|
|
322
|
+
out["frac_p95_below_resolution"] = float((p95 < resolution).mean())
|
|
323
|
+
return out
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def read_convergence(path: Path) -> pd.DataFrame:
|
|
327
|
+
"""``<prefix>-convergence_angles.txt``: ``left``, ``right``, ``p25``, ``p50``, ``p75`` (deg), ``matches``."""
|
|
328
|
+
rows = []
|
|
329
|
+
for line in Path(path).read_text().splitlines():
|
|
330
|
+
parts = line.split()
|
|
331
|
+
if len(parts) >= 6 and not parts[0].startswith("#"):
|
|
332
|
+
rows.append(
|
|
333
|
+
[Path(parts[0]).name, Path(parts[1]).name, *map(float, parts[2:5]), int(float(parts[5]))]
|
|
334
|
+
)
|
|
335
|
+
return pd.DataFrame(rows, columns=["left", "right", "p25", "p50", "p75", "matches"])
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def read_pointmap_residuals(path: Path) -> np.ndarray:
|
|
339
|
+
"""``mean_residual`` column (pixels) of a ``*_residuals_pointmap.csv`` (GCP rows included)."""
|
|
340
|
+
df = pd.read_csv(path, comment="#", header=None, usecols=[3], skipinitialspace=True)
|
|
341
|
+
return pd.to_numeric(df[3], errors="coerce").to_numpy(float)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def removed_images(work_dir: Path) -> pd.DataFrame:
|
|
345
|
+
"""``selection/removed.csv`` with a ``cause`` column (matches, offset, residuals, shadow, other)."""
|
|
346
|
+
path = Path(work_dir) / "selection" / "removed.csv"
|
|
347
|
+
cols = ["product_id", "stage", "reason", "cause"]
|
|
348
|
+
if not path.exists():
|
|
349
|
+
return pd.DataFrame(columns=cols)
|
|
350
|
+
df = pd.read_csv(path, dtype=str).fillna("")
|
|
351
|
+
df["cause"] = df["reason"].map(_cause)
|
|
352
|
+
return df[cols]
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _cause(reason: str) -> str:
|
|
356
|
+
for key, cause in (
|
|
357
|
+
("matches", "matches"),
|
|
358
|
+
("offset", "offset"),
|
|
359
|
+
("residuals", "residuals"),
|
|
360
|
+
("shadow", "shadow"),
|
|
361
|
+
):
|
|
362
|
+
if key in reason:
|
|
363
|
+
return cause
|
|
364
|
+
return "other"
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def selection_table(work_dir: Path) -> pd.DataFrame | None:
|
|
368
|
+
"""``selection/selection.csv`` (every kept image; ``selected`` as bool), or None."""
|
|
369
|
+
path = Path(work_dir) / "selection" / "selection.csv"
|
|
370
|
+
if not path.exists():
|
|
371
|
+
return None
|
|
372
|
+
df = pd.read_csv(path)
|
|
373
|
+
df["selected"] = df["selected"].astype(str).str.lower().isin(("true", "1"))
|
|
374
|
+
return df
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def preview_grid(grid: Grid, max_size: int) -> Grid:
|
|
378
|
+
"""*grid* coarsened by an integer factor so its longer side is at most *max_size*."""
|
|
379
|
+
f = max(1, math.ceil(max(grid.width, grid.height) / max_size))
|
|
380
|
+
if f == 1:
|
|
381
|
+
return grid
|
|
382
|
+
from rasterio.transform import Affine
|
|
383
|
+
|
|
384
|
+
t = grid.transform
|
|
385
|
+
coarse = Affine(t.a * f, t.b * f, t.c, t.d * f, t.e * f, t.f)
|
|
386
|
+
return Grid(grid.crs, coarse, math.ceil(grid.width / f), math.ceil(grid.height / f))
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def solar_bins_from_images(
|
|
390
|
+
paths: Sequence[Path],
|
|
391
|
+
azimuths: Sequence[float],
|
|
392
|
+
grid: Grid,
|
|
393
|
+
shadow_threshold: float,
|
|
394
|
+
bin_width: float = 30.0,
|
|
395
|
+
) -> np.ndarray:
|
|
396
|
+
"""Non-redundant Sun directions per pixel of *grid* from mapprojected images (``illumination_maps``)."""
|
|
397
|
+
from .sfs_products import ProductImage
|
|
398
|
+
|
|
399
|
+
ims = [ProductImage(Path(p), float(a)) for p, a in zip(paths, azimuths)]
|
|
400
|
+
return illumination_maps(ims, grid, shadow_threshold=shadow_threshold, bin_width=bin_width).solar_bins
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def sample(array: np.ndarray, grid: Grid, x: np.ndarray, y: np.ndarray) -> np.ndarray:
|
|
404
|
+
"""Nearest-pixel values of *array* on *grid* at map coordinates; NaN outside."""
|
|
405
|
+
inv = ~grid.transform
|
|
406
|
+
x, y = np.asarray(x, float), np.asarray(y, float)
|
|
407
|
+
col, row = inv.a * x + inv.b * y + inv.c, inv.d * x + inv.e * y + inv.f
|
|
408
|
+
c, r = np.floor(col).astype(int), np.floor(row).astype(int)
|
|
409
|
+
ok = (c >= 0) & (c < grid.width) & (r >= 0) & (r < grid.height)
|
|
410
|
+
out = np.full(c.shape, np.nan)
|
|
411
|
+
out[ok] = array[r[ok], c[ok]]
|
|
412
|
+
return out
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def lola_differences(
|
|
416
|
+
dem: Path, points_csv: Path, out_prefix: Path, z: np.ndarray | None = None, grid: Grid | None = None
|
|
417
|
+
) -> pd.DataFrame:
|
|
418
|
+
"""DEM minus LOLA points (``x``, ``y`` polar stereographic m; ``diff`` m).
|
|
419
|
+
|
|
420
|
+
Uses ``geodiff`` (:func:`~lunar_toolkit.sfs_products.compare_to_points`);
|
|
421
|
+
without ASP, the nearest DEM pixel of each point (*z* on *grid*, read when not given).
|
|
422
|
+
"""
|
|
423
|
+
from . import reference as ref
|
|
424
|
+
from .geometry import ps_xy
|
|
425
|
+
from .sfs_products import compare_to_points
|
|
426
|
+
|
|
427
|
+
try:
|
|
428
|
+
cmp = compare_to_points(
|
|
429
|
+
dem, points_csv, out_prefix, csv_format=ref.ASP_CSV_FORMAT, csv_srs=ref.PS_SRS
|
|
430
|
+
)
|
|
431
|
+
except ToolError as exc:
|
|
432
|
+
log.info("geodiff unavailable (%s); sampling the DEM at the points", exc)
|
|
433
|
+
else:
|
|
434
|
+
xy = [ps_xy(la, lo, "S") for lo, la in zip(cmp.lon, cmp.lat)]
|
|
435
|
+
x = np.array([p[0] for p in xy], float)
|
|
436
|
+
y = np.array([p[1] for p in xy], float)
|
|
437
|
+
return pd.DataFrame({"x": x, "y": y, "diff": cmp.diff})
|
|
438
|
+
pts = np.loadtxt(points_csv, delimiter=",", comments="#", ndmin=2)
|
|
439
|
+
if z is None or grid is None:
|
|
440
|
+
z, grid = read_raster(dem)
|
|
441
|
+
return pd.DataFrame(
|
|
442
|
+
{"x": pts[:, 0], "y": pts[:, 1], "diff": sample(z, grid, pts[:, 0], pts[:, 1]) - pts[:, 2]}
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
class _Terrain:
|
|
447
|
+
"""The SfS DEM, prior and derived arrays on the SfS grid, read once (float32)."""
|
|
448
|
+
|
|
449
|
+
def __init__(self, inputs: ReportInputs):
|
|
450
|
+
self.inputs = inputs
|
|
451
|
+
self._cache: dict[str, np.ndarray | None] = {}
|
|
452
|
+
self.grid = Grid.from_raster(inputs.sfs_dem) if inputs.sfs_dem else None
|
|
453
|
+
|
|
454
|
+
def get(self, name: str, make: Callable[[], np.ndarray | None]) -> np.ndarray | None:
|
|
455
|
+
if name not in self._cache:
|
|
456
|
+
self._cache[name] = make()
|
|
457
|
+
return self._cache[name]
|
|
458
|
+
|
|
459
|
+
def raster(self, path: Path | None) -> np.ndarray | None:
|
|
460
|
+
if path is None or self.grid is None or not Path(path).exists():
|
|
461
|
+
return None
|
|
462
|
+
return read_raster(path, self.grid)[0].astype(np.float32)
|
|
463
|
+
|
|
464
|
+
@property
|
|
465
|
+
def z(self) -> np.ndarray | None:
|
|
466
|
+
return self.get("z", lambda: self.raster(self.inputs.sfs_dem))
|
|
467
|
+
|
|
468
|
+
@property
|
|
469
|
+
def prior(self) -> np.ndarray | None:
|
|
470
|
+
return self.get("prior", lambda: self.raster(self.inputs.prior_dem))
|
|
471
|
+
|
|
472
|
+
@property
|
|
473
|
+
def diff(self) -> np.ndarray | None:
|
|
474
|
+
return self.get("diff", lambda: None if self.z is None or self.prior is None else self.z - self.prior)
|
|
475
|
+
|
|
476
|
+
@property
|
|
477
|
+
def slope(self) -> np.ndarray | None:
|
|
478
|
+
def make() -> np.ndarray | None:
|
|
479
|
+
res = self.inputs.resolution or 5
|
|
480
|
+
product = self.raster(_first_existing(self.inputs.product("TSLP", res)))
|
|
481
|
+
if product is not None:
|
|
482
|
+
return product
|
|
483
|
+
return None if self.z is None or self.grid is None else _slope(self.z, self.grid.res)
|
|
484
|
+
|
|
485
|
+
return self.get("slope", make)
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def _slope(z: np.ndarray, res: float) -> np.ndarray:
|
|
489
|
+
return slope_deg(z.astype(np.float64), res).astype(np.float32)
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _quiet(func: Callable[..., Any], *args: Any) -> Any:
|
|
493
|
+
with warnings.catch_warnings():
|
|
494
|
+
warnings.simplefilter("ignore", RuntimeWarning)
|
|
495
|
+
return func(*args)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _ba_section(m: Metrics, t: _Terrain) -> None:
|
|
499
|
+
inp = m.inputs
|
|
500
|
+
ws = open_workspace(inp.work_dir)
|
|
501
|
+
try:
|
|
502
|
+
for stage, rel in BA_STAGES:
|
|
503
|
+
prefix = inp.work_dir / rel
|
|
504
|
+
run = Path(rel).parent.name
|
|
505
|
+
from_ws = ws is not None and run in ws.runs()
|
|
506
|
+
table = ws.ba_report(run) if ws is not None and from_ws else read_ba_reports(prefix)
|
|
507
|
+
if table is None:
|
|
508
|
+
continue
|
|
509
|
+
m.tables[f"ba_{stage}"] = table
|
|
510
|
+
summary = ba_summary(table, inp.resolution or 5, inp.max_offset_p85 or 1.8)
|
|
511
|
+
summary["source"] = "workspace" if from_ws else "files"
|
|
512
|
+
conv = Path(f"{prefix}-convergence_angles.txt")
|
|
513
|
+
angles = None
|
|
514
|
+
if ws is not None and from_ws:
|
|
515
|
+
angles = ws.convergence(run)
|
|
516
|
+
elif conv.exists():
|
|
517
|
+
angles = read_convergence(conv)
|
|
518
|
+
if angles is not None and len(angles):
|
|
519
|
+
summary["convergence_deg"] = {"pairs": int(len(angles)), **_med(angles["p50"])}
|
|
520
|
+
m.summary.setdefault("ba", {})[stage] = summary
|
|
521
|
+
for key in ("initial", "final"):
|
|
522
|
+
pm = Path(f"{prefix}-{key}_residuals_pointmap.csv")
|
|
523
|
+
values = None
|
|
524
|
+
if ws is not None and from_ws:
|
|
525
|
+
values = ws.point_residuals(run, key)
|
|
526
|
+
values = values if len(values) else None
|
|
527
|
+
elif pm.exists():
|
|
528
|
+
values = read_pointmap_residuals(pm)
|
|
529
|
+
if values is not None:
|
|
530
|
+
m.histograms.setdefault(f"pointmap_{stage}", []).append(
|
|
531
|
+
Histogram.of(key, values, np.logspace(-2, 2, 81))
|
|
532
|
+
)
|
|
533
|
+
finally:
|
|
534
|
+
if ws is not None:
|
|
535
|
+
ws.close()
|
|
536
|
+
if "ba" not in m.summary:
|
|
537
|
+
m.skipped["ba"] = "no bundle-adjustment reports under ba/ or ba_refine/"
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def open_workspace(work_dir: Path) -> Any:
|
|
541
|
+
"""The work dir's ``lunar.duckdb`` :class:`~lunar_toolkit.workspace.Workspace` (read-only), or None.
|
|
542
|
+
|
|
543
|
+
None when there is none or it cannot be opened (e.g. locked by a running pipeline).
|
|
544
|
+
"""
|
|
545
|
+
import duckdb
|
|
546
|
+
|
|
547
|
+
from .workspace import Workspace, workspace_path
|
|
548
|
+
|
|
549
|
+
path = workspace_path(work_dir)
|
|
550
|
+
if not path.exists():
|
|
551
|
+
return None
|
|
552
|
+
try:
|
|
553
|
+
return Workspace(path, read_only=True, spatial=False)
|
|
554
|
+
except duckdb.Error as exc:
|
|
555
|
+
log.warning("Not reading %s: %s", path, exc)
|
|
556
|
+
return None
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def _med(s: pd.Series) -> dict[str, float]:
|
|
560
|
+
s = s.dropna()
|
|
561
|
+
return {"median": float(s.median()), "max": float(s.max())} if len(s) else {}
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _removed_section(m: Metrics, t: _Terrain) -> None:
|
|
565
|
+
wd = m.inputs.work_dir
|
|
566
|
+
removed = removed_images(wd)
|
|
567
|
+
sel = selection_table(wd)
|
|
568
|
+
ba = m.tables.get("ba_ba")
|
|
569
|
+
counts: dict[str, Any] = {}
|
|
570
|
+
if ba is not None:
|
|
571
|
+
counts["bundle_adjusted"] = int(len(ba))
|
|
572
|
+
if sel is not None:
|
|
573
|
+
counts["kept"] = int(len(sel))
|
|
574
|
+
counts["selected"] = int(sel["selected"].sum())
|
|
575
|
+
if removed.empty and not counts:
|
|
576
|
+
m.skipped["removed"] = "no selection/removed.csv or selection.csv"
|
|
577
|
+
return
|
|
578
|
+
m.tables["removed"] = removed
|
|
579
|
+
counts["removed"] = int(len(removed))
|
|
580
|
+
counts["by_stage"] = {str(k): int(v) for k, v in removed["stage"].value_counts().items()}
|
|
581
|
+
counts["by_cause"] = {str(k): int(v) for k, v in removed["cause"].value_counts().items()}
|
|
582
|
+
m.summary["removed"] = counts
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def _coverage_section(m: Metrics, t: _Terrain) -> None:
|
|
586
|
+
inp = m.inputs
|
|
587
|
+
out: dict[str, Any] = {}
|
|
588
|
+
product = inp.product("SBCT")
|
|
589
|
+
if product.exists():
|
|
590
|
+
bins = read_raster(product)[0]
|
|
591
|
+
out["selected_full_res"] = coverage_fractions(np.nan_to_num(bins, nan=0.0))
|
|
592
|
+
sel = selection_table(inp.work_dir)
|
|
593
|
+
if sel is not None and t.grid is not None:
|
|
594
|
+
sel = sel[sel["mapprojected"].map(lambda p: isinstance(p, str) and Path(p).exists())]
|
|
595
|
+
sel = sel[np.isfinite(pd.to_numeric(sel["sun_azimuth"], errors="coerce"))]
|
|
596
|
+
coarse = preview_grid(t.grid, inp.preview_size)
|
|
597
|
+
groups = {"kept": sel, "selected": sel[sel["selected"]]}
|
|
598
|
+
for name, df in groups.items():
|
|
599
|
+
if len(df):
|
|
600
|
+
bins = solar_bins_from_images(
|
|
601
|
+
df["mapprojected"].tolist(),
|
|
602
|
+
df["sun_azimuth"].astype(float).tolist(),
|
|
603
|
+
coarse,
|
|
604
|
+
inp.shadow_threshold or 0.0,
|
|
605
|
+
)
|
|
606
|
+
out[name] = {"images": int(len(df)), **coverage_fractions(bins)}
|
|
607
|
+
if not out:
|
|
608
|
+
m.skipped["coverage"] = "no solar-bin product (GLDSBCT) or mapprojected SfS images"
|
|
609
|
+
return
|
|
610
|
+
m.summary["coverage"] = out
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _prior_section(m: Metrics, t: _Terrain) -> None:
|
|
614
|
+
inp = m.inputs
|
|
615
|
+
if t.diff is None:
|
|
616
|
+
m.skipped["prior"] = "SfS DEM or prior DEM missing"
|
|
617
|
+
return
|
|
618
|
+
ldec = t.get("ldec", lambda: t.raster(inp.ldec))
|
|
619
|
+
m.summary["prior"] = split_stats(t.diff, ldec)
|
|
620
|
+
hists = [Histogram.of("SfS - prior (all)", t.diff, DIFF_EDGES)]
|
|
621
|
+
if ldec is not None:
|
|
622
|
+
hists += [
|
|
623
|
+
Histogram.of("LDEC > 0", t.diff[np.isfinite(ldec) & (ldec > 0)], DIFF_EDGES),
|
|
624
|
+
Histogram.of("LDEC = 0", t.diff[np.isfinite(ldec) & (ldec <= 0)], DIFF_EDGES),
|
|
625
|
+
]
|
|
626
|
+
m.histograms["prior_diff"] = hists
|
|
627
|
+
if t.slope is not None:
|
|
628
|
+
m.tables["prior_vs_slope"] = binned(t.diff, t.slope, inp.slope_edges)
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def _lola_section(m: Metrics, t: _Terrain, work: Path) -> None:
|
|
632
|
+
inp = m.inputs
|
|
633
|
+
points = inp.lola_points
|
|
634
|
+
if points is None and inp.lola_xyzi is not None and t.grid is not None and Path(inp.lola_xyzi).exists():
|
|
635
|
+
from . import reference as ref
|
|
636
|
+
|
|
637
|
+
x0, y0, x1, y1 = t.grid.bounds
|
|
638
|
+
points = ref.write_asp_csv(Path(inp.lola_xyzi), work / "lola_points.csv", ref.PsBox(x0, y0, x1, y1))
|
|
639
|
+
if points is None or not Path(points).exists() or inp.sfs_dem is None or t.z is None or t.grid is None:
|
|
640
|
+
m.skipped["lola"] = "no LOLA points (--lola-points / --lola-xyzi) or SfS DEM"
|
|
641
|
+
return
|
|
642
|
+
pts = lola_differences(inp.sfs_dem, Path(points), work / "lola", t.z, t.grid)
|
|
643
|
+
m.summary["lola"] = stats(pts["diff"])
|
|
644
|
+
m.histograms["lola_diff"] = [Histogram.of("SfS - LOLA", pts["diff"].to_numpy(), DIFF_EDGES)]
|
|
645
|
+
if t.slope is not None:
|
|
646
|
+
slope = sample(t.slope, t.grid, pts["x"].to_numpy(), pts["y"].to_numpy())
|
|
647
|
+
m.tables["lola_vs_slope"] = binned(pts["diff"].to_numpy(), slope, inp.slope_edges)
|
|
648
|
+
ok = np.isfinite(slope) & np.isfinite(pts["diff"].to_numpy())
|
|
649
|
+
counts, xe, ye = np.histogram2d(
|
|
650
|
+
slope[ok],
|
|
651
|
+
np.clip(pts["diff"].to_numpy()[ok], DIFF_EDGES[0], DIFF_EDGES[-1]),
|
|
652
|
+
[np.linspace(0, 45, 46), DIFF_EDGES],
|
|
653
|
+
)
|
|
654
|
+
m.hist2d["lola_vs_slope"] = (xe, ye, counts)
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _ensemble_section(m: Metrics, t: _Terrain) -> None:
|
|
658
|
+
inp = m.inputs
|
|
659
|
+
paths = {k: inp.product(c, "ENS") for k, c in (("elevation_std", "ISGM"), ("slope_std", "SSGM"))}
|
|
660
|
+
if not paths["elevation_std"].exists():
|
|
661
|
+
m.skipped["ensemble"] = "no ensemble products (*_GLDISGM_*_ENS.tif)"
|
|
662
|
+
return
|
|
663
|
+
esd, grid = read_raster(paths["elevation_std"])
|
|
664
|
+
ssd = read_raster(paths["slope_std"], grid)[0] if paths["slope_std"].exists() else None
|
|
665
|
+
out: dict[str, Any] = {"elevation_std_m": stats(esd)}
|
|
666
|
+
if ssd is not None:
|
|
667
|
+
out["slope_std_deg"] = stats(ssd)
|
|
668
|
+
m.histograms["ensemble"] = [Histogram.of("elevation std (m)", esd, np.linspace(0, 5, 101))]
|
|
669
|
+
slope = None
|
|
670
|
+
if inp.sfs_dem is not None:
|
|
671
|
+
z = read_raster(inp.sfs_dem, grid)[0]
|
|
672
|
+
slope = slope_deg(z, grid.res)
|
|
673
|
+
if slope is not None:
|
|
674
|
+
lo, hi = slope < inp.slope_split, slope >= inp.slope_split
|
|
675
|
+
out["by_slope"] = {
|
|
676
|
+
f"slope<{inp.slope_split:g}": stats(esd[lo]),
|
|
677
|
+
f"slope>={inp.slope_split:g}": stats(esd[hi]),
|
|
678
|
+
}
|
|
679
|
+
m.tables["ensemble_vs_slope"] = binned(esd, slope, inp.slope_edges)
|
|
680
|
+
height_error = inp.product("ISGM")
|
|
681
|
+
if height_error.exists():
|
|
682
|
+
he = read_raster(height_error, grid)[0]
|
|
683
|
+
ratio = _quiet(np.divide, esd, np.where(he > 0, he, np.nan))
|
|
684
|
+
out["ensemble_over_height_error"] = stats(ratio)
|
|
685
|
+
m.histograms["ensemble"].append(Histogram.of("ASP height error (m)", he, np.linspace(0, 5, 101)))
|
|
686
|
+
m.summary["ensemble"] = out
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def _uncertainty_section(m: Metrics, t: _Terrain) -> None:
|
|
690
|
+
inp = m.inputs
|
|
691
|
+
out: dict[str, Any] = {}
|
|
692
|
+
hists = []
|
|
693
|
+
toterr = t.raster(inp.toterr)
|
|
694
|
+
if toterr is not None and t.diff is not None:
|
|
695
|
+
z = _quiet(np.divide, t.diff, np.where(toterr > 0, toterr, np.nan))
|
|
696
|
+
out["elevation"] = normal_summary(z)
|
|
697
|
+
hists.append(Histogram.of("elevation", z, NORM_EDGES))
|
|
698
|
+
slperr = t.raster(inp.slperr)
|
|
699
|
+
if slperr is not None and t.slope is not None:
|
|
700
|
+
ldem_slope = t.raster(inp.ldem_slope)
|
|
701
|
+
if ldem_slope is None and t.prior is not None and t.grid is not None:
|
|
702
|
+
ldem_slope = _slope(t.prior, t.grid.res)
|
|
703
|
+
if ldem_slope is not None:
|
|
704
|
+
z = _quiet(np.divide, t.slope - ldem_slope, np.where(slperr > 0, slperr, np.nan))
|
|
705
|
+
out["slope"] = normal_summary(z)
|
|
706
|
+
hists.append(Histogram.of("slope", z, NORM_EDGES))
|
|
707
|
+
if not out:
|
|
708
|
+
m.skipped["uncertainty"] = "no PGDA toterr / slperr maps (--toterr / --slperr / --region)"
|
|
709
|
+
return
|
|
710
|
+
m.summary["uncertainty"] = out
|
|
711
|
+
m.histograms["normalised"] = hists
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def _published_section(m: Metrics, t: _Terrain) -> None:
|
|
715
|
+
inp = m.inputs
|
|
716
|
+
pub = t.raster(inp.published_dem)
|
|
717
|
+
if pub is None or t.z is None or t.grid is None:
|
|
718
|
+
m.skipped["published"] = "no published SDEM (--published-dem)"
|
|
719
|
+
return
|
|
720
|
+
diff = t.z - pub
|
|
721
|
+
t._cache["published_diff"] = diff
|
|
722
|
+
out = {"elevation": stats(diff)}
|
|
723
|
+
if t.slope is not None:
|
|
724
|
+
out["slope"] = stats(t.slope - _slope(pub, t.grid.res))
|
|
725
|
+
m.summary["published"] = out
|
|
726
|
+
m.histograms["published_diff"] = [Histogram.of("SfS - published", diff, DIFF_EDGES)]
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def metrics(
|
|
730
|
+
inputs: ReportInputs | Path, work: Path | None = None, sections: Iterable[str] | None = None
|
|
731
|
+
) -> Metrics:
|
|
732
|
+
"""All report metrics; sections whose inputs are missing are listed in ``skipped``.
|
|
733
|
+
|
|
734
|
+
Parameters
|
|
735
|
+
----------
|
|
736
|
+
inputs : ReportInputs or Path
|
|
737
|
+
Inputs, or a work dir (:meth:`ReportInputs.from_work_dir`).
|
|
738
|
+
work : Path, optional
|
|
739
|
+
Scratch directory for converted LOLA points and ``geodiff`` output
|
|
740
|
+
(default: ``<work_dir>/report/work``).
|
|
741
|
+
sections : iterable of str, optional
|
|
742
|
+
Subset of :data:`SECTIONS` (default: all).
|
|
743
|
+
"""
|
|
744
|
+
inp = inputs if isinstance(inputs, ReportInputs) else ReportInputs.from_work_dir(inputs)
|
|
745
|
+
work = Path(work) if work else inp.work_dir / "report" / "work"
|
|
746
|
+
m = Metrics(inp)
|
|
747
|
+
t = _Terrain(inp)
|
|
748
|
+
runners: dict[str, Callable[[], None]] = {
|
|
749
|
+
"ba": lambda: _ba_section(m, t),
|
|
750
|
+
"removed": lambda: _removed_section(m, t),
|
|
751
|
+
"coverage": lambda: _coverage_section(m, t),
|
|
752
|
+
"prior": lambda: _prior_section(m, t),
|
|
753
|
+
"lola": lambda: _lola_section(m, t, work),
|
|
754
|
+
"ensemble": lambda: _ensemble_section(m, t),
|
|
755
|
+
"uncertainty": lambda: _uncertainty_section(m, t),
|
|
756
|
+
"published": lambda: _published_section(m, t),
|
|
757
|
+
}
|
|
758
|
+
wanted = list(sections) if sections is not None else list(SECTIONS)
|
|
759
|
+
unknown = set(wanted) - set(runners)
|
|
760
|
+
if unknown:
|
|
761
|
+
raise ValueError(f"Unknown report sections {sorted(unknown)}; choose from {', '.join(SECTIONS)}")
|
|
762
|
+
for name in wanted:
|
|
763
|
+
log.info("Report: %s", SECTIONS[name])
|
|
764
|
+
runners[name]()
|
|
765
|
+
m.summary["inputs"] = {f.name: _jsonable(getattr(inp, f.name)) for f in fields(inp)}
|
|
766
|
+
m.summary["skipped"] = dict(m.skipped)
|
|
767
|
+
return m
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def _jsonable(v: Any) -> Any:
|
|
771
|
+
if isinstance(v, Path):
|
|
772
|
+
return str(v)
|
|
773
|
+
if isinstance(v, (tuple, list)):
|
|
774
|
+
return [_jsonable(x) for x in v]
|
|
775
|
+
if isinstance(v, (np.floating, np.integer)):
|
|
776
|
+
return v.item()
|
|
777
|
+
if isinstance(v, dict):
|
|
778
|
+
return {str(k): _jsonable(x) for k, x in v.items()}
|
|
779
|
+
if isinstance(v, float) and not math.isfinite(v):
|
|
780
|
+
return None
|
|
781
|
+
return v
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
def write_tables(m: Metrics, out_dir: Path) -> dict[str, Path]:
|
|
785
|
+
"""``tables/<name>.csv`` for every table, plus ``metrics.json`` (summary and histograms)."""
|
|
786
|
+
out_dir = Path(out_dir)
|
|
787
|
+
(out_dir / "tables").mkdir(parents=True, exist_ok=True)
|
|
788
|
+
paths = {}
|
|
789
|
+
for name, df in m.tables.items():
|
|
790
|
+
paths[name] = out_dir / "tables" / f"{name}.csv"
|
|
791
|
+
df.to_csv(paths[name], index=False)
|
|
792
|
+
for name, df in summary_tables(m).items():
|
|
793
|
+
paths[name] = out_dir / "tables" / f"{name}.csv"
|
|
794
|
+
df.to_csv(paths[name], index=False)
|
|
795
|
+
doc = {
|
|
796
|
+
**_jsonable(m.summary),
|
|
797
|
+
"histograms": {
|
|
798
|
+
k: [{"label": h.label, "edges": h.edges.tolist(), "counts": h.counts.tolist()} for h in v]
|
|
799
|
+
for k, v in m.histograms.items()
|
|
800
|
+
},
|
|
801
|
+
}
|
|
802
|
+
paths["metrics"] = out_dir / "metrics.json"
|
|
803
|
+
paths["metrics"].write_text(json.dumps(doc, indent=1))
|
|
804
|
+
return paths
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def summary_tables(m: Metrics) -> dict[str, pd.DataFrame]:
|
|
808
|
+
"""Table 2-style rows (one per comparison) and the coverage fractions as tables."""
|
|
809
|
+
out = {}
|
|
810
|
+
rows = []
|
|
811
|
+
for label, s in (
|
|
812
|
+
*((f"SfS - prior ({k})", v) for k, v in m.summary.get("prior", {}).items()),
|
|
813
|
+
*((("SfS - LOLA points", m.summary["lola"]),) if "lola" in m.summary else ()),
|
|
814
|
+
*((("SfS - published", m.summary["published"]["elevation"]),) if "published" in m.summary else ()),
|
|
815
|
+
):
|
|
816
|
+
rows.append({"comparison": label, **s})
|
|
817
|
+
if rows:
|
|
818
|
+
out["elevation_differences"] = pd.DataFrame(rows)
|
|
819
|
+
cov = m.summary.get("coverage", {})
|
|
820
|
+
if cov:
|
|
821
|
+
out["coverage"] = pd.DataFrame([{"images_set": k, **v} for k, v in cov.items()])
|
|
822
|
+
return out
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def _figure(nrows: int = 1, ncols: int = 1, size: tuple[float, float] = (6.4, 4.2)) -> tuple[Any, Any]:
|
|
826
|
+
import matplotlib
|
|
827
|
+
|
|
828
|
+
matplotlib.use("Agg")
|
|
829
|
+
import matplotlib.pyplot as plt
|
|
830
|
+
|
|
831
|
+
fig, ax = plt.subplots(nrows, ncols, figsize=size, constrained_layout=True)
|
|
832
|
+
return fig, ax
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
def _save(fig: Any, path: Path) -> Path:
|
|
836
|
+
import matplotlib.pyplot as plt
|
|
837
|
+
|
|
838
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
839
|
+
fig.savefig(path, dpi=110)
|
|
840
|
+
plt.close(fig)
|
|
841
|
+
return path
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def _style(ax: Any) -> None:
|
|
845
|
+
ax.grid(True, color="#e0e0e0", linewidth=0.6)
|
|
846
|
+
ax.set_axisbelow(True)
|
|
847
|
+
for side in ("top", "right"):
|
|
848
|
+
ax.spines[side].set_visible(False)
|
|
849
|
+
|
|
850
|
+
|
|
851
|
+
def _hist_figure(
|
|
852
|
+
hists: Sequence[Histogram],
|
|
853
|
+
xlabel: str,
|
|
854
|
+
path: Path,
|
|
855
|
+
*,
|
|
856
|
+
density: bool = True,
|
|
857
|
+
logx: bool = False,
|
|
858
|
+
gaussians: Sequence[tuple[float, str]] = (),
|
|
859
|
+
) -> Path:
|
|
860
|
+
fig, ax = _figure()
|
|
861
|
+
for h, color in zip(hists, SERIES):
|
|
862
|
+
y = h.counts / max(h.total, 1) / np.diff(h.edges) if density else h.counts
|
|
863
|
+
ax.stairs(y, h.edges, color=color, linewidth=1.6, label=f"{h.label} (n={h.total:,})")
|
|
864
|
+
for sigma, color in gaussians:
|
|
865
|
+
x = np.linspace(hists[0].edges[0], hists[0].edges[-1], 400)
|
|
866
|
+
ax.plot(
|
|
867
|
+
x,
|
|
868
|
+
np.exp(-0.5 * (x / sigma) ** 2) / (sigma * math.sqrt(2 * math.pi)),
|
|
869
|
+
"--",
|
|
870
|
+
color=color,
|
|
871
|
+
linewidth=1.2,
|
|
872
|
+
label=f"Gaussian sigma={sigma:.2f}",
|
|
873
|
+
)
|
|
874
|
+
if logx:
|
|
875
|
+
ax.set_xscale("log")
|
|
876
|
+
ax.set_xlabel(xlabel)
|
|
877
|
+
ax.set_ylabel("density" if density else "count")
|
|
878
|
+
_style(ax)
|
|
879
|
+
if len(hists) + len(gaussians) > 1:
|
|
880
|
+
ax.legend(frameon=False, fontsize=8)
|
|
881
|
+
return _save(fig, path)
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
def _map_figure(
|
|
885
|
+
array: np.ndarray,
|
|
886
|
+
grid: Grid,
|
|
887
|
+
path: Path,
|
|
888
|
+
label: str,
|
|
889
|
+
*,
|
|
890
|
+
cmap: str = "cividis",
|
|
891
|
+
symmetric: bool = False,
|
|
892
|
+
vmin: float | None = None,
|
|
893
|
+
vmax: float | None = None,
|
|
894
|
+
) -> Path:
|
|
895
|
+
fig, ax = _figure(size=(6.0, 5.0))
|
|
896
|
+
x0, y0, x1, y1 = grid.bounds
|
|
897
|
+
finite = array[np.isfinite(array)]
|
|
898
|
+
if symmetric and finite.size:
|
|
899
|
+
lim = float(np.percentile(np.abs(finite), 98)) or 1.0
|
|
900
|
+
vmin, vmax = -lim, lim
|
|
901
|
+
elif finite.size and vmin is None:
|
|
902
|
+
vmin, vmax = (float(v) for v in np.percentile(finite, [2, 98]))
|
|
903
|
+
im = ax.imshow(
|
|
904
|
+
array,
|
|
905
|
+
extent=(x0 / 1e3, x1 / 1e3, y0 / 1e3, y1 / 1e3),
|
|
906
|
+
cmap=cmap,
|
|
907
|
+
vmin=vmin,
|
|
908
|
+
vmax=vmax,
|
|
909
|
+
interpolation="nearest",
|
|
910
|
+
)
|
|
911
|
+
fig.colorbar(im, ax=ax, label=label, shrink=0.85)
|
|
912
|
+
_map_axes(ax)
|
|
913
|
+
return _save(fig, path)
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
def _map_axes(ax: Any) -> None:
|
|
917
|
+
from matplotlib.ticker import MaxNLocator
|
|
918
|
+
|
|
919
|
+
ax.xaxis.set_major_locator(MaxNLocator(5))
|
|
920
|
+
ax.yaxis.set_major_locator(MaxNLocator(5))
|
|
921
|
+
ax.set_xlabel("x (km, polar stereographic)")
|
|
922
|
+
ax.set_ylabel("y (km)")
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
def hillshade_array(z: np.ndarray, res: float, azimuth: float = 315.0, elevation: float = 45.0) -> np.ndarray:
|
|
926
|
+
"""Lambertian hillshade (0-1) of *z* with the Sun at *azimuth* / *elevation* (deg)."""
|
|
927
|
+
from .sfs_products import gradient
|
|
928
|
+
|
|
929
|
+
p, q = gradient(z, res)
|
|
930
|
+
az, el = math.radians(azimuth), math.radians(elevation)
|
|
931
|
+
sx, sy, sz = math.sin(az) * math.cos(el), math.cos(az) * math.cos(el), math.sin(el)
|
|
932
|
+
return np.clip((-p * sx - q * sy + sz) / np.sqrt(1 + p * p + q * q), 0, 1)
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
def figures(m: Metrics, out_dir: Path) -> dict[str, Path]:
|
|
936
|
+
"""PNG figures of every section with data, in ``<out_dir>/figures``; returns name -> path."""
|
|
937
|
+
import matplotlib.colors as mcolors
|
|
938
|
+
|
|
939
|
+
inp = m.inputs
|
|
940
|
+
fig_dir = Path(out_dir) / "figures"
|
|
941
|
+
out: dict[str, Path] = {}
|
|
942
|
+
if inp.sfs_dem is not None and Path(inp.sfs_dem).exists():
|
|
943
|
+
grid = preview_grid(Grid.from_raster(inp.sfs_dem), inp.preview_size)
|
|
944
|
+
z = read_raster(inp.sfs_dem, grid)[0]
|
|
945
|
+
out["hillshade"] = _map_figure(
|
|
946
|
+
hillshade_array(z, grid.res),
|
|
947
|
+
grid,
|
|
948
|
+
fig_dir / "hillshade.png",
|
|
949
|
+
"hillshade (Sun az 315, el 45 deg)",
|
|
950
|
+
cmap="gray",
|
|
951
|
+
vmin=0,
|
|
952
|
+
vmax=1,
|
|
953
|
+
)
|
|
954
|
+
out["slope_map"] = _map_figure(
|
|
955
|
+
slope_deg(z, grid.res),
|
|
956
|
+
grid,
|
|
957
|
+
fig_dir / "slope_map.png",
|
|
958
|
+
"slope (deg)",
|
|
959
|
+
cmap="cividis",
|
|
960
|
+
vmin=0,
|
|
961
|
+
vmax=40,
|
|
962
|
+
)
|
|
963
|
+
if inp.prior_dem is not None and Path(inp.prior_dem).exists():
|
|
964
|
+
d = z - read_raster(inp.prior_dem, grid)[0]
|
|
965
|
+
out["prior_diff_map"] = _map_figure(
|
|
966
|
+
d, grid, fig_dir / "prior_diff_map.png", "SfS - prior (m)", cmap="RdBu_r", symmetric=True
|
|
967
|
+
)
|
|
968
|
+
if inp.published_dem is not None and Path(inp.published_dem).exists():
|
|
969
|
+
d = z - read_raster(inp.published_dem, grid)[0]
|
|
970
|
+
out["published_diff_map"] = _map_figure(
|
|
971
|
+
d,
|
|
972
|
+
grid,
|
|
973
|
+
fig_dir / "published_diff_map.png",
|
|
974
|
+
"SfS - published SDEM (m)",
|
|
975
|
+
cmap="RdBu_r",
|
|
976
|
+
symmetric=True,
|
|
977
|
+
)
|
|
978
|
+
sbct = inp.product("SBCT")
|
|
979
|
+
if sbct.exists():
|
|
980
|
+
g = preview_grid(Grid.from_raster(sbct), inp.preview_size)
|
|
981
|
+
bins = np.nan_to_num(_nearest(sbct, g), nan=0.0)
|
|
982
|
+
cmap = mcolors.ListedColormap(COVERAGE_COLORS)
|
|
983
|
+
fig, ax = _figure(size=(6.0, 5.0))
|
|
984
|
+
x0, y0, x1, y1 = g.bounds
|
|
985
|
+
im = ax.imshow(
|
|
986
|
+
np.minimum(bins, 3),
|
|
987
|
+
extent=(x0 / 1e3, x1 / 1e3, y0 / 1e3, y1 / 1e3),
|
|
988
|
+
cmap=cmap,
|
|
989
|
+
norm=mcolors.BoundaryNorm([-0.5, 0.5, 1.5, 2.5, 3.5], 4),
|
|
990
|
+
interpolation="nearest",
|
|
991
|
+
)
|
|
992
|
+
cb = fig.colorbar(im, ax=ax, ticks=[0, 1, 2, 3], shrink=0.85, label="non-redundant Sun directions")
|
|
993
|
+
cb.ax.set_yticklabels(COVERAGE_LABELS)
|
|
994
|
+
_map_axes(ax)
|
|
995
|
+
out["coverage_map"] = _save(fig, fig_dir / "coverage_map.png")
|
|
996
|
+
cov = {k: v for k, v in m.summary.get("coverage", {}).items() if v.get("pixels")}
|
|
997
|
+
if cov:
|
|
998
|
+
out["coverage_bars"] = _coverage_bars(cov, fig_dir / "coverage_bars.png")
|
|
999
|
+
for stage in ("ba", "refine"):
|
|
1000
|
+
table = m.tables.get(f"ba_{stage}")
|
|
1001
|
+
if table is not None:
|
|
1002
|
+
out.update(_ba_figures(table, stage, inp, fig_dir))
|
|
1003
|
+
if f"pointmap_{stage}" in m.histograms:
|
|
1004
|
+
out[f"pointmap_{stage}"] = _hist_figure(
|
|
1005
|
+
m.histograms[f"pointmap_{stage}"],
|
|
1006
|
+
"triangulated point mean residual (px)",
|
|
1007
|
+
fig_dir / f"pointmap_residuals_{stage}.png",
|
|
1008
|
+
logx=True,
|
|
1009
|
+
)
|
|
1010
|
+
diffs = m.histograms.get("prior_diff", []) + m.histograms.get("lola_diff", [])
|
|
1011
|
+
if diffs:
|
|
1012
|
+
out["diff_histograms"] = _hist_figure(
|
|
1013
|
+
diffs[:3], "elevation difference (m)", fig_dir / "diff_histograms.png"
|
|
1014
|
+
)
|
|
1015
|
+
if "lola_diff" in m.histograms and len(diffs) > 3:
|
|
1016
|
+
out["lola_histogram"] = _hist_figure(
|
|
1017
|
+
m.histograms["lola_diff"], "SfS - LOLA (m)", fig_dir / "lola_histogram.png"
|
|
1018
|
+
)
|
|
1019
|
+
for name in ("lola_vs_slope", "prior_vs_slope", "ensemble_vs_slope"):
|
|
1020
|
+
if name in m.tables:
|
|
1021
|
+
out[name] = _slope_figure(m, name, fig_dir / f"{name}.png")
|
|
1022
|
+
if "ensemble" in m.histograms:
|
|
1023
|
+
out["ensemble_histogram"] = _hist_figure(
|
|
1024
|
+
m.histograms["ensemble"], "elevation uncertainty (m)", fig_dir / "ensemble_histogram.png"
|
|
1025
|
+
)
|
|
1026
|
+
esd = inp.product("ISGM", "ENS")
|
|
1027
|
+
g = preview_grid(Grid.from_raster(esd), inp.preview_size)
|
|
1028
|
+
out["ensemble_map"] = _map_figure(
|
|
1029
|
+
read_raster(esd, g)[0],
|
|
1030
|
+
g,
|
|
1031
|
+
fig_dir / "ensemble_map.png",
|
|
1032
|
+
"ensemble elevation std (m)",
|
|
1033
|
+
cmap="cividis",
|
|
1034
|
+
)
|
|
1035
|
+
for h in m.histograms.get("normalised", []):
|
|
1036
|
+
sigma = m.summary["uncertainty"][h.label]["std"]
|
|
1037
|
+
out[f"normalised_{h.label}"] = _hist_figure(
|
|
1038
|
+
[h],
|
|
1039
|
+
f"(SfS - LDEM) / LDEM uncertainty, {h.label}",
|
|
1040
|
+
fig_dir / f"normalised_{h.label}.png",
|
|
1041
|
+
gaussians=[(1.0, "#e34948"), (sigma, "#008300")],
|
|
1042
|
+
)
|
|
1043
|
+
return out
|
|
1044
|
+
|
|
1045
|
+
|
|
1046
|
+
def _nearest(path: Path, grid: Grid) -> np.ndarray:
|
|
1047
|
+
import rasterio
|
|
1048
|
+
from rasterio.enums import Resampling
|
|
1049
|
+
|
|
1050
|
+
with rasterio.open(path) as src:
|
|
1051
|
+
data = src.read(1, out_shape=(grid.height, grid.width), resampling=Resampling.nearest, masked=True)
|
|
1052
|
+
return data.astype(np.float64).filled(np.nan)
|
|
1053
|
+
|
|
1054
|
+
|
|
1055
|
+
def _coverage_bars(cov: dict[str, dict[str, float]], path: Path) -> Path:
|
|
1056
|
+
fig, ax = _figure(size=(6.4, 2.2 + 0.5 * len(cov)))
|
|
1057
|
+
names = list(cov)
|
|
1058
|
+
left = np.zeros(len(names))
|
|
1059
|
+
keys = ("frac_3plus", "frac_2", "frac_1", "frac_0")
|
|
1060
|
+
for key, color, label in zip(keys, COVERAGE_COLORS[::-1], ("3+", "2", "1", "0")):
|
|
1061
|
+
vals = np.array([100 * cov[n].get(key, 0.0) for n in names])
|
|
1062
|
+
ax.barh(
|
|
1063
|
+
names, vals, left=left, color=color, edgecolor="white", linewidth=1, label=f"{label} directions"
|
|
1064
|
+
)
|
|
1065
|
+
left += vals
|
|
1066
|
+
ax.set_xlim(0, 100)
|
|
1067
|
+
ax.set_xlabel("pixels (%)")
|
|
1068
|
+
ax.legend(frameon=False, fontsize=8, ncol=4, loc="upper center", bbox_to_anchor=(0.5, -0.35))
|
|
1069
|
+
_style(ax)
|
|
1070
|
+
return _save(fig, path)
|
|
1071
|
+
|
|
1072
|
+
|
|
1073
|
+
def _ba_figures(table: pd.DataFrame, stage: str, inp: ReportInputs, fig_dir: Path) -> dict[str, Path]:
|
|
1074
|
+
out = {}
|
|
1075
|
+
med = [table[c].dropna().to_numpy() for c in ("initial_median", "final_median")]
|
|
1076
|
+
if all(len(x) for x in med):
|
|
1077
|
+
edges = np.logspace(-2, 2, 61)
|
|
1078
|
+
hists = [Histogram.of(k, x, edges) for k, x in zip(("before BA", "after BA"), med)]
|
|
1079
|
+
out[f"ba_residuals_{stage}"] = _hist_figure(
|
|
1080
|
+
hists,
|
|
1081
|
+
"per-image median reprojection error (px)",
|
|
1082
|
+
fig_dir / f"ba_residuals_{stage}.png",
|
|
1083
|
+
density=False,
|
|
1084
|
+
logx=True,
|
|
1085
|
+
)
|
|
1086
|
+
both = table.dropna(subset=["initial_median", "final_median"])
|
|
1087
|
+
fig, ax = _figure(size=(5.0, 4.6))
|
|
1088
|
+
ax.scatter(both["initial_median"], both["final_median"], s=10, color=SERIES[0])
|
|
1089
|
+
ax.set_xscale("log")
|
|
1090
|
+
ax.set_yscale("log")
|
|
1091
|
+
ax.set_xlabel("median residual before BA (px)")
|
|
1092
|
+
ax.set_ylabel("median residual after BA (px)")
|
|
1093
|
+
_style(ax)
|
|
1094
|
+
out[f"ba_before_after_{stage}"] = _save(fig, fig_dir / f"ba_before_after_{stage}.png")
|
|
1095
|
+
p85, p95 = table["offset_p85"].dropna(), table["offset_p95"].dropna()
|
|
1096
|
+
if len(p85):
|
|
1097
|
+
hi = max(float(np.nanmax(p95)) if len(p95) else 0.0, (inp.resolution or 5) * 1.2, 2.0)
|
|
1098
|
+
edges = np.linspace(0, hi, 61)
|
|
1099
|
+
fig, ax = _figure()
|
|
1100
|
+
for s, color, label in ((p85, SERIES[0], "p85"), (p95, SERIES[1], "p95")):
|
|
1101
|
+
ax.stairs(
|
|
1102
|
+
np.histogram(np.clip(s, 0, hi), edges)[0], edges, color=color, linewidth=1.6, label=label
|
|
1103
|
+
)
|
|
1104
|
+
ax.axvline(
|
|
1105
|
+
inp.max_offset_p85 or 1.8,
|
|
1106
|
+
color="#52514e",
|
|
1107
|
+
linestyle="--",
|
|
1108
|
+
linewidth=1,
|
|
1109
|
+
label=f"p85 threshold {inp.max_offset_p85:g} m",
|
|
1110
|
+
)
|
|
1111
|
+
ax.axvline(
|
|
1112
|
+
inp.resolution or 5,
|
|
1113
|
+
color="#52514e",
|
|
1114
|
+
linestyle=":",
|
|
1115
|
+
linewidth=1,
|
|
1116
|
+
label=f"SfS resolution {inp.resolution:g} m",
|
|
1117
|
+
)
|
|
1118
|
+
ax.set_xlabel("per-image mapprojected match offset (m)")
|
|
1119
|
+
ax.set_ylabel("images")
|
|
1120
|
+
ax.legend(frameon=False, fontsize=8)
|
|
1121
|
+
_style(ax)
|
|
1122
|
+
out[f"match_offsets_{stage}"] = _save(fig, fig_dir / f"match_offsets_{stage}.png")
|
|
1123
|
+
return out
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
def _slope_figure(m: Metrics, name: str, path: Path) -> Path:
|
|
1127
|
+
t = m.tables[name].dropna(subset=["median"])
|
|
1128
|
+
fig, ax = _figure()
|
|
1129
|
+
if name in m.hist2d:
|
|
1130
|
+
xe, ye, counts = m.hist2d[name]
|
|
1131
|
+
ax.pcolormesh(
|
|
1132
|
+
xe, ye, np.where(counts.T > 0, counts.T, np.nan), cmap="Greys", norm="log", shading="flat"
|
|
1133
|
+
)
|
|
1134
|
+
mid = (t["bin_min"] + np.minimum(t["bin_max"], 45)) / 2
|
|
1135
|
+
ax.fill_between(mid, t["p25"], t["p75"], color=SERIES[0], alpha=0.25, linewidth=0, label="interquartile")
|
|
1136
|
+
ax.plot(mid, t["median"], "-o", color=SERIES[0], markersize=4, linewidth=1.6, label="median")
|
|
1137
|
+
ylabel = {
|
|
1138
|
+
"lola_vs_slope": "SfS - LOLA (m)",
|
|
1139
|
+
"prior_vs_slope": "SfS - prior (m)",
|
|
1140
|
+
"ensemble_vs_slope": "ensemble elevation std (m)",
|
|
1141
|
+
}[name]
|
|
1142
|
+
ax.set_xlabel("slope (deg)")
|
|
1143
|
+
ax.set_ylabel(ylabel)
|
|
1144
|
+
ax.legend(frameon=False, fontsize=8)
|
|
1145
|
+
_style(ax)
|
|
1146
|
+
return _save(fig, path)
|
|
1147
|
+
|
|
1148
|
+
|
|
1149
|
+
_CSS = """
|
|
1150
|
+
body { background: #ffffff; color: #111111; font-family: Georgia, 'Times New Roman', serif;
|
|
1151
|
+
max-width: 60em; margin: 2em auto; padding: 0 16px; line-height: 1.45; }
|
|
1152
|
+
h1, h2, h3 { font-weight: normal; }
|
|
1153
|
+
h2 { border-bottom: 1px solid #999999; padding-bottom: 0.2em; margin-top: 2em; }
|
|
1154
|
+
table { border-collapse: collapse; margin: 0.8em 0; font-size: 0.85em; display: block; overflow-x: auto; }
|
|
1155
|
+
th, td { border-top: 1px solid #cccccc; border-bottom: 1px solid #cccccc; padding: 0.25em 0.6em; text-align: right; }
|
|
1156
|
+
th { background: #f2f2f2; }
|
|
1157
|
+
td:first-child, th:first-child { text-align: left; }
|
|
1158
|
+
figure { margin: 1em 0; }
|
|
1159
|
+
figure img { max-width: 100%; height: auto; border: 1px solid #dddddd; }
|
|
1160
|
+
figcaption, .note { font-size: 0.85em; color: #444444; }
|
|
1161
|
+
code { font-size: 0.9em; }
|
|
1162
|
+
"""
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
def _fmt(v: Any) -> str:
|
|
1166
|
+
if isinstance(v, float):
|
|
1167
|
+
if not math.isfinite(v):
|
|
1168
|
+
return ""
|
|
1169
|
+
return f"{v:.4g}" if abs(v) < 1e5 else f"{v:,.0f}"
|
|
1170
|
+
if isinstance(v, (int, np.integer)) and not isinstance(v, bool):
|
|
1171
|
+
return f"{v:,}"
|
|
1172
|
+
return html.escape(str(v))
|
|
1173
|
+
|
|
1174
|
+
|
|
1175
|
+
def _table_html(df: pd.DataFrame, max_rows: int = 40) -> str:
|
|
1176
|
+
head = "".join(f"<th>{html.escape(str(c))}</th>" for c in df.columns)
|
|
1177
|
+
body = "".join(
|
|
1178
|
+
"<tr>" + "".join(f"<td>{_fmt(v)}</td>" for v in row) + "</tr>"
|
|
1179
|
+
for row in df.head(max_rows).itertuples(index=False)
|
|
1180
|
+
)
|
|
1181
|
+
more = (
|
|
1182
|
+
f'<p class="note">First {max_rows} of {len(df)} rows; see the CSV.</p>' if len(df) > max_rows else ""
|
|
1183
|
+
)
|
|
1184
|
+
return f"<table><tr>{head}</tr>{body}</table>{more}"
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
def _dict_table(d: dict[str, Any]) -> str:
|
|
1188
|
+
rows = []
|
|
1189
|
+
for k, v in d.items():
|
|
1190
|
+
if isinstance(v, dict) and all(not isinstance(x, dict) for x in v.values()):
|
|
1191
|
+
v = ", ".join(f"{a}={_fmt(b)}" for a, b in v.items())
|
|
1192
|
+
rows.append(f"<tr><td>{html.escape(k)}</td><td>{v}</td></tr>")
|
|
1193
|
+
elif not isinstance(v, dict):
|
|
1194
|
+
rows.append(f"<tr><td>{html.escape(k)}</td><td>{_fmt(v)}</td></tr>")
|
|
1195
|
+
return f"<table>{''.join(rows)}</table>" if rows else ""
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
def _stats_rows(d: dict[str, dict[str, Any]]) -> str:
|
|
1199
|
+
rows = [{"subset": k, **v} for k, v in d.items() if isinstance(v, dict)]
|
|
1200
|
+
return _table_html(pd.DataFrame(rows)) if rows else ""
|
|
1201
|
+
|
|
1202
|
+
|
|
1203
|
+
def _img(path: Path, caption: str) -> str:
|
|
1204
|
+
data = base64.b64encode(Path(path).read_bytes()).decode("ascii")
|
|
1205
|
+
return f'<figure><img src="data:image/png;base64,{data}" alt="{html.escape(caption)}"><figcaption>{html.escape(caption)}</figcaption></figure>'
|
|
1206
|
+
|
|
1207
|
+
|
|
1208
|
+
_SECTION_FIGURES = {
|
|
1209
|
+
"ba": (
|
|
1210
|
+
"ba_residuals_ba",
|
|
1211
|
+
"ba_before_after_ba",
|
|
1212
|
+
"match_offsets_ba",
|
|
1213
|
+
"pointmap_ba",
|
|
1214
|
+
"ba_residuals_refine",
|
|
1215
|
+
"ba_before_after_refine",
|
|
1216
|
+
"match_offsets_refine",
|
|
1217
|
+
"pointmap_refine",
|
|
1218
|
+
),
|
|
1219
|
+
"coverage": ("coverage_map", "coverage_bars"),
|
|
1220
|
+
"prior": ("hillshade", "slope_map", "prior_diff_map", "diff_histograms", "prior_vs_slope"),
|
|
1221
|
+
"lola": ("lola_histogram", "lola_vs_slope"),
|
|
1222
|
+
"ensemble": ("ensemble_map", "ensemble_histogram", "ensemble_vs_slope"),
|
|
1223
|
+
"uncertainty": ("normalised_elevation", "normalised_slope"),
|
|
1224
|
+
"published": ("published_diff_map",),
|
|
1225
|
+
}
|
|
1226
|
+
_SECTION_TABLES = {
|
|
1227
|
+
"ba": ("ba_ba", "ba_refine"),
|
|
1228
|
+
"removed": ("removed",),
|
|
1229
|
+
"prior": ("prior_vs_slope",),
|
|
1230
|
+
"lola": ("lola_vs_slope",),
|
|
1231
|
+
"ensemble": ("ensemble_vs_slope",),
|
|
1232
|
+
}
|
|
1233
|
+
_CAPTIONS = {
|
|
1234
|
+
"ba_residuals": "Per-image median reprojection error before and after bundle adjustment (cf. Fig. 5 top right).",
|
|
1235
|
+
"ba_before_after": "Median reprojection error per image, before vs after bundle adjustment.",
|
|
1236
|
+
"match_offsets": "Per-image 85th / 95th percentile mapprojected match offsets (cf. Fig. 5 bottom right).",
|
|
1237
|
+
"pointmap": "Mean reprojection error of the triangulated points, before and after bundle adjustment.",
|
|
1238
|
+
"coverage_map": "Non-redundant Sun directions (30 deg bins) per pixel of the SfS inputs (cf. Fig. 6).",
|
|
1239
|
+
"coverage_bars": "Pixels lit from 0, 1, 2 and >=3 directions, kept vs selected images (cf. Fig. 3).",
|
|
1240
|
+
"hillshade": "Hillshade of the SfS DEM (preview grid).",
|
|
1241
|
+
"slope_map": "Slope of the SfS DEM on the preview grid (baseline = 2 preview pixels).",
|
|
1242
|
+
"prior_diff_map": "SfS DEM minus prior DEM.",
|
|
1243
|
+
"diff_histograms": "Elevation differences (cf. Fig. 7 left); values beyond the axis go to the outer bins.",
|
|
1244
|
+
"lola_histogram": "SfS DEM minus LOLA points.",
|
|
1245
|
+
"lola_vs_slope": "SfS minus LOLA points vs slope: median and interquartile range (cf. Fig. 7 right).",
|
|
1246
|
+
"prior_vs_slope": "SfS minus prior DEM vs slope: median and interquartile range.",
|
|
1247
|
+
"ensemble_map": "Ensemble elevation standard deviation (cf. Fig. 8 top left).",
|
|
1248
|
+
"ensemble_histogram": "Ensemble elevation std and ASP height error (cf. Fig. 8 bottom right).",
|
|
1249
|
+
"ensemble_vs_slope": "Ensemble elevation std vs slope (cf. Fig. 8 bottom left).",
|
|
1250
|
+
"normalised": "SfS - LDEM residuals over the LDEM clone uncertainty, with a unit Gaussian (red) and "
|
|
1251
|
+
"a Gaussian of the measured sigma (green) (cf. Fig. 9).",
|
|
1252
|
+
"published_diff_map": "SfS DEM minus the published Bertone et al. SDEM.",
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
|
|
1256
|
+
def _caption(name: str) -> str:
|
|
1257
|
+
for key in sorted(_CAPTIONS, key=len, reverse=True):
|
|
1258
|
+
if name.startswith(key):
|
|
1259
|
+
return _CAPTIONS[key]
|
|
1260
|
+
return name
|
|
1261
|
+
|
|
1262
|
+
|
|
1263
|
+
def write_html(m: Metrics, figs: dict[str, Path], out: Path, tables: dict[str, Path] | None = None) -> Path:
|
|
1264
|
+
"""Self-contained ``report.html``: summary tables, embedded figures, skipped sections."""
|
|
1265
|
+
inp = m.inputs
|
|
1266
|
+
out = Path(out)
|
|
1267
|
+
parts = [
|
|
1268
|
+
"<!doctype html><html lang='en'><head><meta charset='utf-8'>",
|
|
1269
|
+
"<meta name='viewport' content='width=device-width, initial-scale=1'>",
|
|
1270
|
+
f"<title>SfS report {html.escape(inp.region_id or '')}</title><style>{_CSS}</style></head><body>",
|
|
1271
|
+
f"<h1>SfS DEM report: {html.escape(inp.region_id or '')}</h1>",
|
|
1272
|
+
f"<p class='note'>Work dir <code>{html.escape(str(inp.work_dir))}</code>; SfS DEM "
|
|
1273
|
+
f"<code>{html.escape(str(inp.sfs_dem))}</code>; prior <code>{html.escape(str(inp.prior_dem))}</code>; "
|
|
1274
|
+
f"{inp.resolution:g} m/px. Metrics follow Bertone et al. (2026, PSJ 7:118); "
|
|
1275
|
+
"tables are in <code>tables/</code> and <code>metrics.json</code>.</p>",
|
|
1276
|
+
]
|
|
1277
|
+
summary_tabs = summary_tables(m)
|
|
1278
|
+
if "elevation_differences" in summary_tabs:
|
|
1279
|
+
parts += [
|
|
1280
|
+
"<h2>Elevation differences (Table 2)</h2>",
|
|
1281
|
+
_table_html(summary_tabs["elevation_differences"]),
|
|
1282
|
+
]
|
|
1283
|
+
for key, title in SECTIONS.items():
|
|
1284
|
+
parts.append(f"<h2>{html.escape(title)}</h2>")
|
|
1285
|
+
if key in m.skipped:
|
|
1286
|
+
parts.append(f"<p class='note'>Skipped: {html.escape(m.skipped[key])}.</p>")
|
|
1287
|
+
continue
|
|
1288
|
+
s = m.summary.get(key, {})
|
|
1289
|
+
if key == "coverage" and "coverage" in summary_tabs:
|
|
1290
|
+
parts.append(_table_html(summary_tabs["coverage"]))
|
|
1291
|
+
elif key in ("prior", "uncertainty", "published") or (key == "ensemble" and "by_slope" in s):
|
|
1292
|
+
parts.append(_stats_rows({k: v for k, v in s.items() if k != "by_slope"}))
|
|
1293
|
+
if "by_slope" in s:
|
|
1294
|
+
parts.append(_stats_rows(s["by_slope"]))
|
|
1295
|
+
elif key == "ba":
|
|
1296
|
+
for stage, st in s.items():
|
|
1297
|
+
parts += [f"<h3>{html.escape(stage)}</h3>", _dict_table(st)]
|
|
1298
|
+
else:
|
|
1299
|
+
parts.append(_dict_table(s) if key != "lola" else _stats_rows({"SfS - LOLA": s}))
|
|
1300
|
+
for name in _SECTION_FIGURES.get(key, ()):
|
|
1301
|
+
if name in figs:
|
|
1302
|
+
parts.append(_img(figs[name], _caption(name)))
|
|
1303
|
+
for name in _SECTION_TABLES.get(key, ()):
|
|
1304
|
+
if name in m.tables and len(m.tables[name]):
|
|
1305
|
+
parts += [f"<h3>{html.escape(name)}</h3>", _table_html(m.tables[name])]
|
|
1306
|
+
if tables:
|
|
1307
|
+
links = "".join(
|
|
1308
|
+
f"<li><a href='{html.escape(p.relative_to(out.parent).as_posix())}'>{html.escape(p.name)}</a></li>"
|
|
1309
|
+
for p in tables.values()
|
|
1310
|
+
if p.is_relative_to(out.parent)
|
|
1311
|
+
)
|
|
1312
|
+
parts += ["<h2>Files</h2>", f"<ul>{links}</ul>"]
|
|
1313
|
+
parts.append("</body></html>")
|
|
1314
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
1315
|
+
out.write_text("\n".join(parts), encoding="ascii", errors="xmlcharrefreplace")
|
|
1316
|
+
return out
|
|
1317
|
+
|
|
1318
|
+
|
|
1319
|
+
def make_report(
|
|
1320
|
+
inputs: ReportInputs | Path, out_dir: Path | None = None, sections: Iterable[str] | None = None
|
|
1321
|
+
) -> Path:
|
|
1322
|
+
"""Metrics, tables, figures and ``report.html`` in *out_dir* (default ``<work_dir>/report``)."""
|
|
1323
|
+
inp = inputs if isinstance(inputs, ReportInputs) else ReportInputs.from_work_dir(inputs)
|
|
1324
|
+
out_dir = Path(out_dir) if out_dir else inp.work_dir / "report"
|
|
1325
|
+
m = metrics(inp, out_dir / "work", sections)
|
|
1326
|
+
tables = write_tables(m, out_dir)
|
|
1327
|
+
figs = figures(m, out_dir)
|
|
1328
|
+
return write_html(m, figs, out_dir / "report.html", tables)
|