lunar-toolkit 0.1.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lunar_toolkit/__init__.py +6 -0
- lunar_toolkit/asp.py +432 -0
- lunar_toolkit/cli/__init__.py +88 -0
- lunar_toolkit/cli/_jobs.py +437 -0
- lunar_toolkit/cli/_status.py +368 -0
- lunar_toolkit/cli/_utils.py +128 -0
- lunar_toolkit/cli/app.py +205 -0
- lunar_toolkit/cli/build_db.py +130 -0
- lunar_toolkit/cli/download.py +111 -0
- lunar_toolkit/cli/kaguya.py +253 -0
- lunar_toolkit/cli/ode_search.py +144 -0
- lunar_toolkit/cli/pairs.py +236 -0
- lunar_toolkit/cli/process.py +138 -0
- lunar_toolkit/cli/query.py +70 -0
- lunar_toolkit/cli/reference.py +233 -0
- lunar_toolkit/cli/search.py +90 -0
- lunar_toolkit/cli/serve.py +706 -0
- lunar_toolkit/cli/sfs.py +344 -0
- lunar_toolkit/cli/sfs_report.py +148 -0
- lunar_toolkit/cli/shadowcam.py +246 -0
- lunar_toolkit/cli/stereo.py +85 -0
- lunar_toolkit/cli/workspace.py +84 -0
- lunar_toolkit/config.py +345 -0
- lunar_toolkit/constants.py +217 -0
- lunar_toolkit/csm.py +340 -0
- lunar_toolkit/downloader.py +447 -0
- lunar_toolkit/geometry.py +556 -0
- lunar_toolkit/isis.py +222 -0
- lunar_toolkit/kaguya/__init__.py +45 -0
- lunar_toolkit/kaguya/mi.py +274 -0
- lunar_toolkit/kaguya/tc.py +370 -0
- lunar_toolkit/lroc/__init__.py +6 -0
- lunar_toolkit/lroc/cumindex.py +1197 -0
- lunar_toolkit/lroc/nac.py +702 -0
- lunar_toolkit/observation.py +127 -0
- lunar_toolkit/ode.py +485 -0
- lunar_toolkit/provenance.py +312 -0
- lunar_toolkit/reference.py +1111 -0
- lunar_toolkit/selection.py +714 -0
- lunar_toolkit/sfs.py +1534 -0
- lunar_toolkit/sfs_products.py +676 -0
- lunar_toolkit/sfs_report.py +1328 -0
- lunar_toolkit/shadowcam/__init__.py +29 -0
- lunar_toolkit/shadowcam/sc.py +392 -0
- lunar_toolkit/static/index.html +3259 -0
- lunar_toolkit/stereo.py +519 -0
- lunar_toolkit/stereo_dem.py +145 -0
- lunar_toolkit/tools.py +264 -0
- lunar_toolkit/workspace.py +1173 -0
- lunar_toolkit-0.1.0.dev2.dist-info/METADATA +177 -0
- lunar_toolkit-0.1.0.dev2.dist-info/RECORD +55 -0
- lunar_toolkit-0.1.0.dev2.dist-info/WHEEL +5 -0
- lunar_toolkit-0.1.0.dev2.dist-info/entry_points.txt +2 -0
- lunar_toolkit-0.1.0.dev2.dist-info/licenses/LICENSE +28 -0
- lunar_toolkit-0.1.0.dev2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1173 @@
|
|
|
1
|
+
"""DuckDB workspace (``<work_dir>/lunar.duckdb``): image footprints, stereo overlaps and bundle-adjustment statistics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import math
|
|
7
|
+
import re
|
|
8
|
+
from collections.abc import Mapping, Sequence
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import TYPE_CHECKING, Any
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
|
|
14
|
+
from .constants import MOON_RADIUS_KM, parse_nac_id
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
17
|
+
import pandas as pd
|
|
18
|
+
|
|
19
|
+
from .asp import MatchOffsetStats, ResidualStats
|
|
20
|
+
from .sfs import SfsConfig
|
|
21
|
+
|
|
22
|
+
log = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
DB_NAME = "lunar.duckdb"
|
|
25
|
+
# lunar pairs --engine auto uses DuckDB from this many images up
|
|
26
|
+
PAIRS_ENGINE_THRESHOLD = 300
|
|
27
|
+
# Footprint edges are great circles (as in find_stereo_pairs), densified at this step (deg)
|
|
28
|
+
GREAT_CIRCLE_STEP_DEG = 0.1
|
|
29
|
+
# The sphere and projections of the images.footprint geometries (by images.hemi)
|
|
30
|
+
LONLAT_PROJ4 = f"+proj=longlat +R={MOON_RADIUS_KM * 1000:g} +no_defs"
|
|
31
|
+
PS_PROJ4 = {
|
|
32
|
+
hemi: f"+proj=stere +lat_0={lat} +lon_0=0 +k=1 +x_0=0 +y_0=0 +R={MOON_RADIUS_KM * 1000:g} +units=m +no_defs"
|
|
33
|
+
for hemi, lat in (("S", -90), ("N", 90))
|
|
34
|
+
}
|
|
35
|
+
BA_TABLES = (
|
|
36
|
+
"ba_residual_stats",
|
|
37
|
+
"ba_pixel_residuals",
|
|
38
|
+
"ba_match_offsets",
|
|
39
|
+
"ba_points",
|
|
40
|
+
"ba_convergence",
|
|
41
|
+
"ba_camera_offsets",
|
|
42
|
+
)
|
|
43
|
+
_R_M = MOON_RADIUS_KM * 1000.0
|
|
44
|
+
_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
45
|
+
|
|
46
|
+
_OBS_TYPES = {
|
|
47
|
+
"product_id": "VARCHAR",
|
|
48
|
+
"instrument": "VARCHAR",
|
|
49
|
+
"product_type": "VARCHAR",
|
|
50
|
+
"start_time": "TIMESTAMP",
|
|
51
|
+
"center_lat": "DOUBLE",
|
|
52
|
+
"center_lon": "DOUBLE",
|
|
53
|
+
"min_lat": "DOUBLE",
|
|
54
|
+
"max_lat": "DOUBLE",
|
|
55
|
+
"lon_west": "DOUBLE",
|
|
56
|
+
"lon_east": "DOUBLE",
|
|
57
|
+
"footprint_wkt": "VARCHAR",
|
|
58
|
+
"incidence": "DOUBLE",
|
|
59
|
+
"emission": "DOUBLE",
|
|
60
|
+
"phase": "DOUBLE",
|
|
61
|
+
"sub_solar_azimuth": "DOUBLE",
|
|
62
|
+
"north_azimuth": "DOUBLE",
|
|
63
|
+
"sub_spacecraft_lat": "DOUBLE",
|
|
64
|
+
"sub_spacecraft_lon": "DOUBLE",
|
|
65
|
+
"spacecraft_altitude": "DOUBLE",
|
|
66
|
+
"pixel_scale": "DOUBLE",
|
|
67
|
+
"url": "VARCHAR",
|
|
68
|
+
"label_url": "VARCHAR",
|
|
69
|
+
"source": "VARCHAR",
|
|
70
|
+
}
|
|
71
|
+
_IMAGE_EXTRA = {
|
|
72
|
+
"obs_key": "VARCHAR",
|
|
73
|
+
"hemi": "VARCHAR",
|
|
74
|
+
"cap_lat": "DOUBLE",
|
|
75
|
+
"cap_lon": "DOUBLE",
|
|
76
|
+
"footprint_km2": "DOUBLE",
|
|
77
|
+
"xmin": "DOUBLE",
|
|
78
|
+
"xmax": "DOUBLE",
|
|
79
|
+
"ymin": "DOUBLE",
|
|
80
|
+
"ymax": "DOUBLE",
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
_BA_DDL = [
|
|
84
|
+
"CREATE TABLE IF NOT EXISTS ba_runs (run VARCHAR, prefix VARCHAR, loaded_at TIMESTAMP, files VARCHAR[])",
|
|
85
|
+
"CREATE TABLE IF NOT EXISTS ba_residual_stats (run VARCHAR, stage VARCHAR, image VARCHAR, "
|
|
86
|
+
"image_path VARCHAR, product_id VARCHAR, mean_px DOUBLE, median_px DOUBLE, residual_count BIGINT)",
|
|
87
|
+
"CREATE TABLE IF NOT EXISTS ba_pixel_residuals (run VARCHAR, stage VARCHAR, image VARCHAR, "
|
|
88
|
+
"product_id VARCHAR, norm_px DOUBLE)",
|
|
89
|
+
"CREATE TABLE IF NOT EXISTS ba_match_offsets (run VARCHAR, image VARCHAR, image_path VARCHAR, "
|
|
90
|
+
"product_id VARCHAR, p25 DOUBLE, p50 DOUBLE, p75 DOUBLE, p85 DOUBLE, p95 DOUBLE, match_count BIGINT)",
|
|
91
|
+
"CREATE TABLE IF NOT EXISTS ba_points (run VARCHAR, stage VARCHAR, lon DOUBLE, lat DOUBLE, "
|
|
92
|
+
"height_m DOUBLE, mean_residual_px DOUBLE, num_observations INTEGER, kind VARCHAR, hemi VARCHAR, "
|
|
93
|
+
"x DOUBLE, y DOUBLE)",
|
|
94
|
+
"CREATE TABLE IF NOT EXISTS ba_convergence (run VARCHAR, left_image VARCHAR, right_image VARCHAR, "
|
|
95
|
+
"left_product_id VARCHAR, right_product_id VARCHAR, p25 DOUBLE, p50 DOUBLE, p75 DOUBLE, matches BIGINT)",
|
|
96
|
+
"CREATE TABLE IF NOT EXISTS ba_camera_offsets (run VARCHAR, image VARCHAR, product_id VARCHAR, "
|
|
97
|
+
"horizontal_m DOUBLE, vertical_m DOUBLE)",
|
|
98
|
+
"CREATE TABLE IF NOT EXISTS rejection_params (max_offset_p85 DOUBLE, "
|
|
99
|
+
"max_large_residual_fraction DOUBLE, large_residual_px DOUBLE, min_matches BIGINT)",
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
_PS_MACROS = [
|
|
103
|
+
# areal scale of the polar stereographic projection is _ps_k^2 (k = 1 at the pole)
|
|
104
|
+
f"CREATE OR REPLACE MACRO _ps_k(x, y) AS 1.0 + (x * x + y * y) / {4.0 * _R_M * _R_M!r}",
|
|
105
|
+
f"CREATE OR REPLACE MACRO _ps_lat(x, y, hemi) AS (CASE WHEN hemi = 'S' THEN 1.0 ELSE -1.0 END) * "
|
|
106
|
+
f"(degrees(2.0 * atan(sqrt(x * x + y * y) / {2.0 * _R_M!r})) - 90.0)",
|
|
107
|
+
"CREATE OR REPLACE MACRO _ps_lon(x, y, hemi) AS "
|
|
108
|
+
"(degrees(atan2(x, CASE WHEN hemi = 'S' THEN y ELSE -y END)) + 360.0) % 360.0",
|
|
109
|
+
"CREATE OR REPLACE MACRO _az_diff(a, b) AS least(((b - a) % 360.0 + 360.0) % 360.0, "
|
|
110
|
+
"360.0 - ((b - a) % 360.0 + 360.0) % 360.0)",
|
|
111
|
+
"CREATE OR REPLACE MACRO _true_km2(g) AS "
|
|
112
|
+
"ST_Area(g) / pow(_ps_k(ST_X(ST_Centroid(g)), ST_Y(ST_Centroid(g))), 2) / 1e6",
|
|
113
|
+
]
|
|
114
|
+
|
|
115
|
+
# Per-image residual stats before / after BA; {params} is a rejection_params-shaped relation.
|
|
116
|
+
_IMAGE_STATS_SQL = """
|
|
117
|
+
WITH names AS (
|
|
118
|
+
SELECT DISTINCT run, image FROM (
|
|
119
|
+
SELECT run, image FROM ba_residual_stats
|
|
120
|
+
UNION ALL SELECT run, image FROM ba_match_offsets
|
|
121
|
+
UNION ALL SELECT DISTINCT run, image FROM ba_pixel_residuals
|
|
122
|
+
)
|
|
123
|
+
),
|
|
124
|
+
raw AS (
|
|
125
|
+
SELECT r.run, r.image,
|
|
126
|
+
count(*) FILTER (WHERE r.stage = 'final') AS raw_count,
|
|
127
|
+
avg(CAST(r.norm_px > p.large_residual_px AS DOUBLE)) FILTER (WHERE r.stage = 'final')
|
|
128
|
+
AS large_fraction,
|
|
129
|
+
count(*) FILTER (WHERE r.stage = 'initial') AS raw_count_initial,
|
|
130
|
+
avg(CAST(r.norm_px > p.large_residual_px AS DOUBLE)) FILTER (WHERE r.stage = 'initial')
|
|
131
|
+
AS large_fraction_initial
|
|
132
|
+
FROM ba_pixel_residuals r CROSS JOIN ({params}) p GROUP BY r.run, r.image
|
|
133
|
+
)
|
|
134
|
+
SELECT n.run, n.image, upper(split_part(n.image, '.', 1)) AS product_id,
|
|
135
|
+
i.mean_px AS initial_mean, i.median_px AS initial_median, i.residual_count AS initial_count,
|
|
136
|
+
f.mean_px AS final_mean, f.median_px AS final_median, f.residual_count AS final_count,
|
|
137
|
+
f.median_px - i.median_px AS median_change,
|
|
138
|
+
o.p25 AS offset_p25, o.p50 AS offset_p50, o.p75 AS offset_p75, o.p85 AS offset_p85,
|
|
139
|
+
o.p95 AS offset_p95, o.match_count AS offset_count,
|
|
140
|
+
r.raw_count, r.large_fraction, r.raw_count_initial, r.large_fraction_initial,
|
|
141
|
+
c.horizontal_m AS camera_offset_horizontal_m, c.vertical_m AS camera_offset_vertical_m
|
|
142
|
+
FROM names n
|
|
143
|
+
LEFT JOIN ba_residual_stats i ON i.run = n.run AND i.image = n.image AND i.stage = 'initial'
|
|
144
|
+
LEFT JOIN ba_residual_stats f ON f.run = n.run AND f.image = n.image AND f.stage = 'final'
|
|
145
|
+
LEFT JOIN ba_match_offsets o ON o.run = n.run AND o.image = n.image
|
|
146
|
+
LEFT JOIN raw r ON r.run = n.run AND r.image = n.image
|
|
147
|
+
LEFT JOIN ba_camera_offsets c ON c.run = n.run AND c.image = n.image
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
# Bertone et al. (2026, Sec. 2) rejection rules, first match wins (as sfs.flag_images).
|
|
151
|
+
_REJECTIONS_SQL = """
|
|
152
|
+
SELECT run, image, product_id, rule, value, threshold,
|
|
153
|
+
CASE rule
|
|
154
|
+
WHEN 'matches' THEN printf('%d matches', CAST(value AS BIGINT))
|
|
155
|
+
WHEN 'offset' THEN printf('match offset p85 %.2f m > %g m', value, threshold)
|
|
156
|
+
ELSE printf('%.0f%% of residuals > %g px', 100.0 * value, large_residual_px)
|
|
157
|
+
END AS reason
|
|
158
|
+
FROM (
|
|
159
|
+
SELECT s.*, p.large_residual_px,
|
|
160
|
+
CASE WHEN s.min_count < p.min_matches THEN 'matches'
|
|
161
|
+
WHEN s.offset_p85 > p.max_offset_p85 THEN 'offset'
|
|
162
|
+
WHEN s.raw_count > 0 AND s.large_fraction > p.max_large_residual_fraction THEN 'residuals'
|
|
163
|
+
END AS rule,
|
|
164
|
+
CASE rule WHEN 'matches' THEN s.min_count WHEN 'offset' THEN s.offset_p85
|
|
165
|
+
ELSE s.large_fraction END AS value,
|
|
166
|
+
CASE rule WHEN 'matches' THEN p.min_matches WHEN 'offset' THEN p.max_offset_p85
|
|
167
|
+
ELSE p.max_large_residual_fraction END AS threshold
|
|
168
|
+
FROM (SELECT *, least(final_count, offset_count) AS min_count FROM ({stats})) s CROSS JOIN ({params}) p
|
|
169
|
+
)
|
|
170
|
+
WHERE rule IS NOT NULL
|
|
171
|
+
"""
|
|
172
|
+
|
|
173
|
+
_RUN_SUMMARY_SQL = """
|
|
174
|
+
WITH s AS (
|
|
175
|
+
SELECT run,
|
|
176
|
+
count(*) AS images,
|
|
177
|
+
avg(initial_median) AS initial_median_mean,
|
|
178
|
+
stddev_pop(initial_median) AS initial_median_std,
|
|
179
|
+
median(initial_median) AS initial_median_median,
|
|
180
|
+
avg(final_median) AS final_median_mean,
|
|
181
|
+
stddev_pop(final_median) AS final_median_std,
|
|
182
|
+
median(final_median) AS final_median_median,
|
|
183
|
+
median(offset_p85) AS offset_p85_median,
|
|
184
|
+
max(offset_p85) AS offset_p85_max,
|
|
185
|
+
count(*) FILTER (WHERE offset_p85 > (SELECT max_offset_p85 FROM rejection_params))
|
|
186
|
+
AS offset_p85_over_threshold,
|
|
187
|
+
median(offset_p95) AS offset_p95_median,
|
|
188
|
+
max(offset_p95) AS offset_p95_max
|
|
189
|
+
FROM ba_image_stats GROUP BY run
|
|
190
|
+
),
|
|
191
|
+
rej AS (SELECT run, count(*) AS rejected FROM image_rejections GROUP BY run),
|
|
192
|
+
conv AS (SELECT run, count(*) AS convergence_pairs, median(p50) AS convergence_p50_median
|
|
193
|
+
FROM ba_convergence GROUP BY run),
|
|
194
|
+
pts AS (
|
|
195
|
+
SELECT run,
|
|
196
|
+
count(*) FILTER (WHERE stage = 'initial') AS points_initial,
|
|
197
|
+
count(*) FILTER (WHERE stage = 'final') AS points_final,
|
|
198
|
+
median(mean_residual_px) FILTER (WHERE stage = 'initial') AS point_residual_median_initial,
|
|
199
|
+
median(mean_residual_px) FILTER (WHERE stage = 'final') AS point_residual_median_final
|
|
200
|
+
FROM ba_points GROUP BY run
|
|
201
|
+
)
|
|
202
|
+
SELECT s.*, coalesce(rej.rejected, 0) AS rejected, conv.convergence_pairs, conv.convergence_p50_median,
|
|
203
|
+
pts.points_initial, pts.points_final, pts.point_residual_median_initial,
|
|
204
|
+
pts.point_residual_median_final
|
|
205
|
+
FROM s LEFT JOIN rej USING (run) LEFT JOIN conv USING (run) LEFT JOIN pts USING (run)
|
|
206
|
+
ORDER BY s.run
|
|
207
|
+
"""
|
|
208
|
+
|
|
209
|
+
_PARAMS_VIEW = "SELECT * FROM rejection_params"
|
|
210
|
+
_PARAMS_MACRO = (
|
|
211
|
+
"SELECT p85 AS max_offset_p85, frac AS max_large_residual_fraction, px AS large_residual_px, "
|
|
212
|
+
"min_matches AS min_matches"
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class WorkspaceError(RuntimeError):
|
|
217
|
+
"""The workspace cannot do what was asked (e.g. no spatial extension)."""
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def workspace_path(work_dir: Path | str) -> Path:
|
|
221
|
+
"""``<work_dir>/lunar.duckdb``, or *work_dir* itself when it names a ``.duckdb`` file."""
|
|
222
|
+
p = Path(work_dir)
|
|
223
|
+
return p if p.suffix == ".duckdb" else p / DB_NAME
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def spatial_available() -> bool:
|
|
227
|
+
"""Whether DuckDB's spatial extension loads (installing it when needed)."""
|
|
228
|
+
global _SPATIAL
|
|
229
|
+
if _SPATIAL is None:
|
|
230
|
+
import duckdb
|
|
231
|
+
|
|
232
|
+
con = duckdb.connect()
|
|
233
|
+
try:
|
|
234
|
+
_SPATIAL = _load_spatial(con) is None
|
|
235
|
+
finally:
|
|
236
|
+
con.close()
|
|
237
|
+
return _SPATIAL
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
_SPATIAL: bool | None = None
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _load_spatial(con: Any) -> str | None:
|
|
244
|
+
"""Load (installing if needed) the spatial extension; the error message, or None."""
|
|
245
|
+
import duckdb
|
|
246
|
+
|
|
247
|
+
try:
|
|
248
|
+
con.execute("LOAD spatial")
|
|
249
|
+
return None
|
|
250
|
+
except duckdb.Error:
|
|
251
|
+
pass
|
|
252
|
+
try:
|
|
253
|
+
con.execute("INSTALL spatial")
|
|
254
|
+
con.execute("LOAD spatial")
|
|
255
|
+
return None
|
|
256
|
+
except duckdb.Error as exc:
|
|
257
|
+
return str(exc).splitlines()[0]
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _ident(name: str) -> str:
|
|
261
|
+
if not _IDENT_RE.match(name):
|
|
262
|
+
raise ValueError(f"Not a table name: {name!r}")
|
|
263
|
+
return name
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _sql_str(value: Any) -> str:
|
|
267
|
+
return "'" + str(value).replace("'", "''") + "'"
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _num(v: float) -> float | None:
|
|
271
|
+
"""*v*, None for NaN (SQL NULL)."""
|
|
272
|
+
return None if v is None or math.isnan(v) else float(v)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _product_id(image: str) -> str:
|
|
276
|
+
return Path(str(image)).name.split(".")[0].upper()
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class Workspace:
|
|
280
|
+
"""A DuckDB database of images, stereo pairs and bundle-adjustment results.
|
|
281
|
+
|
|
282
|
+
Parameters
|
|
283
|
+
----------
|
|
284
|
+
path : Path or str
|
|
285
|
+
Database file (``":memory:"`` for a scratch one); a directory means
|
|
286
|
+
``<path>/lunar.duckdb``. Parent directories are created.
|
|
287
|
+
memory_limit : str
|
|
288
|
+
DuckDB ``memory_limit``.
|
|
289
|
+
threads : int, optional
|
|
290
|
+
DuckDB ``threads`` (default: DuckDB's, all cores).
|
|
291
|
+
read_only : bool
|
|
292
|
+
Open an existing database read-only (no schema changes).
|
|
293
|
+
spatial : bool
|
|
294
|
+
Load the spatial extension (``INSTALL spatial`` once needs network; it is
|
|
295
|
+
cached in ``~/.duckdb``). Without it the bundle-adjustment tables and views
|
|
296
|
+
work, while :meth:`add_observations` and :meth:`pairs` raise
|
|
297
|
+
:class:`WorkspaceError`.
|
|
298
|
+
|
|
299
|
+
Notes
|
|
300
|
+
-----
|
|
301
|
+
Tables: ``images`` (observation columns plus ``footprint``, a GEOMETRY in metres
|
|
302
|
+
in the polar stereographic projection of ``hemi``, :data:`PS_PROJ4`),
|
|
303
|
+
``pairs`` (the last :meth:`pairs` result), ``ba_runs``, ``ba_residual_stats``,
|
|
304
|
+
``ba_pixel_residuals``, ``ba_match_offsets``, ``ba_points``, ``ba_convergence``,
|
|
305
|
+
``ba_camera_offsets`` (by ``run``) and ``rejection_params``. Views:
|
|
306
|
+
``ba_image_stats`` (per image before / after), ``ba_run_summary`` and
|
|
307
|
+
``image_rejections`` (Bertone et al. 2026 rules with ``rejection_params``); the
|
|
308
|
+
table macro ``image_rejections_at(max_offset_p85, max_large_residual_fraction,
|
|
309
|
+
large_residual_px, min_matches)`` takes other thresholds.
|
|
310
|
+
|
|
311
|
+
Example
|
|
312
|
+
-------
|
|
313
|
+
::
|
|
314
|
+
|
|
315
|
+
with Workspace("work/malapert") as ws:
|
|
316
|
+
ws.add_observations(obs)
|
|
317
|
+
pairs = ws.pairs(aoi={"point": (-85.9, 0.0), "radius_km": 5})
|
|
318
|
+
ws.load_bundle_adjust("work/malapert/ba/run")
|
|
319
|
+
ws.query("SELECT * FROM image_rejections")
|
|
320
|
+
"""
|
|
321
|
+
|
|
322
|
+
def __init__(
|
|
323
|
+
self,
|
|
324
|
+
path: Path | str,
|
|
325
|
+
*,
|
|
326
|
+
memory_limit: str = "1GB",
|
|
327
|
+
threads: int | None = None,
|
|
328
|
+
read_only: bool = False,
|
|
329
|
+
spatial: bool = True,
|
|
330
|
+
) -> None:
|
|
331
|
+
import duckdb
|
|
332
|
+
|
|
333
|
+
self.path = path if str(path) == ":memory:" else workspace_path(path)
|
|
334
|
+
if not read_only and isinstance(self.path, Path):
|
|
335
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
336
|
+
config: dict[str, Any] = {"memory_limit": memory_limit}
|
|
337
|
+
if threads:
|
|
338
|
+
config["threads"] = int(threads)
|
|
339
|
+
self.read_only = read_only
|
|
340
|
+
self.con = duckdb.connect(str(self.path), read_only=read_only, config=config)
|
|
341
|
+
self.con.execute("SET enable_progress_bar = false")
|
|
342
|
+
self.spatial_error = _load_spatial(self.con) if spatial else "not requested"
|
|
343
|
+
if not read_only:
|
|
344
|
+
self._create_schema()
|
|
345
|
+
|
|
346
|
+
@property
|
|
347
|
+
def has_spatial(self) -> bool:
|
|
348
|
+
return self.spatial_error is None
|
|
349
|
+
|
|
350
|
+
def __enter__(self) -> Workspace:
|
|
351
|
+
return self
|
|
352
|
+
|
|
353
|
+
def __exit__(self, *exc: Any) -> None:
|
|
354
|
+
self.close()
|
|
355
|
+
|
|
356
|
+
def close(self) -> None:
|
|
357
|
+
self.con.close()
|
|
358
|
+
|
|
359
|
+
def _require_spatial(self) -> None:
|
|
360
|
+
if self.spatial_error is not None:
|
|
361
|
+
raise WorkspaceError(
|
|
362
|
+
f"DuckDB spatial extension unavailable ({self.spatial_error}). Install it once "
|
|
363
|
+
"with network access: python -c \"import duckdb; duckdb.sql('INSTALL spatial')\""
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
def _create_schema(self) -> None:
|
|
367
|
+
from .lroc.cumindex import _MACROS
|
|
368
|
+
|
|
369
|
+
for sql in [*_MACROS, *_BA_DDL]:
|
|
370
|
+
self.con.execute(sql)
|
|
371
|
+
if self.has_spatial:
|
|
372
|
+
for sql in _PS_MACROS:
|
|
373
|
+
self.con.execute(sql)
|
|
374
|
+
if not self._scalar("SELECT count(*) FROM rejection_params"):
|
|
375
|
+
self.set_rejection_params()
|
|
376
|
+
stats = _IMAGE_STATS_SQL.format(params=_PARAMS_VIEW)
|
|
377
|
+
rejections = _REJECTIONS_SQL.format(params=_PARAMS_VIEW, stats="SELECT * FROM ba_image_stats")
|
|
378
|
+
macro = _REJECTIONS_SQL.format(
|
|
379
|
+
params=_PARAMS_MACRO, stats=_IMAGE_STATS_SQL.format(params=_PARAMS_MACRO)
|
|
380
|
+
)
|
|
381
|
+
for sql in (
|
|
382
|
+
f"CREATE OR REPLACE VIEW ba_image_stats AS {stats}",
|
|
383
|
+
f"CREATE OR REPLACE VIEW image_rejections AS {rejections}",
|
|
384
|
+
f"CREATE OR REPLACE MACRO image_rejections_at(p85, frac, px, min_matches) AS TABLE {macro}",
|
|
385
|
+
f"CREATE OR REPLACE VIEW ba_run_summary AS {_RUN_SUMMARY_SQL}",
|
|
386
|
+
):
|
|
387
|
+
self.con.execute(sql)
|
|
388
|
+
|
|
389
|
+
def query(self, sql: str, params: Sequence[Any] | None = None) -> pd.DataFrame:
|
|
390
|
+
"""Run SQL; returns the result as a DataFrame (empty for statements without one)."""
|
|
391
|
+
import pandas as pd
|
|
392
|
+
|
|
393
|
+
cur = self.con.execute(sql, params) if params is not None else self.con.execute(sql)
|
|
394
|
+
return cur.df() if cur.description else pd.DataFrame()
|
|
395
|
+
|
|
396
|
+
def tables(self) -> pd.DataFrame:
|
|
397
|
+
"""``name``, ``type`` (table / view) and ``rows`` of the workspace's tables and views."""
|
|
398
|
+
import pandas as pd
|
|
399
|
+
|
|
400
|
+
rows = self.con.execute(
|
|
401
|
+
"SELECT table_name, CASE table_type WHEN 'VIEW' THEN 'view' ELSE 'table' END "
|
|
402
|
+
"FROM information_schema.tables WHERE table_schema = 'main' ORDER BY table_type, table_name"
|
|
403
|
+
).fetchall()
|
|
404
|
+
out = []
|
|
405
|
+
for name, kind in rows:
|
|
406
|
+
n = self._scalar(f'SELECT count(*) FROM "{name}"')
|
|
407
|
+
out.append({"name": name, "type": kind, "rows": int(n)})
|
|
408
|
+
return pd.DataFrame(out, columns=["name", "type", "rows"])
|
|
409
|
+
|
|
410
|
+
def _scalar(self, sql: str, params: Sequence[Any] | None = None) -> Any:
|
|
411
|
+
row = (
|
|
412
|
+
self.con.execute(sql, params).fetchone()
|
|
413
|
+
if params is not None
|
|
414
|
+
else self.con.execute(sql).fetchone()
|
|
415
|
+
)
|
|
416
|
+
return None if row is None else row[0]
|
|
417
|
+
|
|
418
|
+
def has_table(self, name: str) -> bool:
|
|
419
|
+
return bool(
|
|
420
|
+
self._scalar(
|
|
421
|
+
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'main' AND table_name = ?",
|
|
422
|
+
[name],
|
|
423
|
+
)
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
def export(self, table: str, path: Path | str) -> Path:
|
|
427
|
+
"""Write a table or view to Parquet (``.parquet``) or CSV (anything else).
|
|
428
|
+
|
|
429
|
+
GEOMETRY columns are written as WKT in CSV and as WKB (GeoParquet) in Parquet.
|
|
430
|
+
"""
|
|
431
|
+
name = _ident(table)
|
|
432
|
+
if not self.has_table(name):
|
|
433
|
+
raise WorkspaceError(f"No table or view {name!r} in {self.path}")
|
|
434
|
+
path = Path(path)
|
|
435
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
436
|
+
parquet = path.suffix.lower() in (".parquet", ".pq")
|
|
437
|
+
cols = self.con.execute(f'DESCRIBE "{name}"').fetchall()
|
|
438
|
+
sel = ", ".join(
|
|
439
|
+
f'ST_AsText("{c}") AS "{c}"' if t.startswith("GEOMETRY") and not parquet else f'"{c}"'
|
|
440
|
+
for c, t, *_ in cols
|
|
441
|
+
)
|
|
442
|
+
fmt = "(FORMAT parquet)" if parquet else "(FORMAT csv, HEADER true)"
|
|
443
|
+
self.con.execute(f'COPY (SELECT {sel} FROM "{name}") TO {_sql_str(path)} {fmt}')
|
|
444
|
+
return path
|
|
445
|
+
|
|
446
|
+
def add_observations(self, df: pd.DataFrame, *, rings: Sequence[Any] | None = None) -> int:
|
|
447
|
+
"""Insert or replace (by ``product_id``) observation rows in ``images``.
|
|
448
|
+
|
|
449
|
+
Parameters
|
|
450
|
+
----------
|
|
451
|
+
df : DataFrame
|
|
452
|
+
:mod:`lunar_toolkit.observation` schema rows; missing columns are NULL.
|
|
453
|
+
rings : sequence, optional
|
|
454
|
+
Per row, :func:`lunar_toolkit.geometry.footprint_rings` output to use instead
|
|
455
|
+
of the rows' own (``footprint_wkt``, else the ``min_lat`` ... ``lon_east`` box).
|
|
456
|
+
|
|
457
|
+
Returns
|
|
458
|
+
-------
|
|
459
|
+
int
|
|
460
|
+
Rows written. ``footprint`` is the footprint polygon in polar stereographic
|
|
461
|
+
metres (:func:`lunar_toolkit.geometry.footprint_geometry` with
|
|
462
|
+
:func:`~lunar_toolkit.geometry.ps_projection` of ``hemi``, the hemisphere of
|
|
463
|
+
the image centre; edges densified along great circles every
|
|
464
|
+
:data:`GREAT_CIRCLE_STEP_DEG`), ``footprint_km2`` its area corrected for the projection's
|
|
465
|
+
scale, ``cap_lat`` / ``cap_lon`` the centre of its vertices, ``obs_key`` the
|
|
466
|
+
observation (NAC LE / RE halves share one). ``idx`` numbers rows in insertion
|
|
467
|
+
order.
|
|
468
|
+
"""
|
|
469
|
+
import pandas as pd
|
|
470
|
+
|
|
471
|
+
from .geometry import footprint_geometry, footprint_rings, ps_projection
|
|
472
|
+
from .observation import OBSERVATION_COLUMNS, normalize_observations
|
|
473
|
+
from .stereo import _cap
|
|
474
|
+
|
|
475
|
+
self._require_spatial()
|
|
476
|
+
self._create_images()
|
|
477
|
+
if df is None or len(df) == 0:
|
|
478
|
+
return 0
|
|
479
|
+
obs = normalize_observations(df.to_dict("records"))
|
|
480
|
+
if rings is None:
|
|
481
|
+
rings = [footprint_rings(r, "footprint_wkt") for r in obs.to_dict("records")]
|
|
482
|
+
extra: dict[str, list[Any]] = {k: [] for k in (*_IMAGE_EXTRA, "fp_wkb")}
|
|
483
|
+
wkts = []
|
|
484
|
+
for row, rr in zip(obs.to_dict("records"), rings):
|
|
485
|
+
pid = str(row["product_id"])
|
|
486
|
+
extra["obs_key"].append((parse_nac_id(pid) or (pid.upper(),))[0])
|
|
487
|
+
geom = None
|
|
488
|
+
cap = (math.nan, math.nan)
|
|
489
|
+
if rr is not None and len(rr[0][0]) >= 3:
|
|
490
|
+
c, _ = _cap([poly[0] for poly in rr])
|
|
491
|
+
cap = (
|
|
492
|
+
math.degrees(math.asin(max(-1.0, min(1.0, c[2])))),
|
|
493
|
+
math.degrees(math.atan2(c[1], c[0])) % 360,
|
|
494
|
+
)
|
|
495
|
+
lat = row["center_lat"] if pd.notna(row["center_lat"]) else cap[0]
|
|
496
|
+
hemi = None if pd.isna(lat) else ("S" if lat < 0 else "N")
|
|
497
|
+
if hemi is not None and rr is not None and len(rr[0][0]) >= 3:
|
|
498
|
+
geom = footprint_geometry(_great_circle_rings(rr), ps_projection(hemi))
|
|
499
|
+
if geom is not None and (geom.is_empty or geom.area <= 0):
|
|
500
|
+
geom = None
|
|
501
|
+
extra["hemi"].append(hemi)
|
|
502
|
+
extra["cap_lat"].append(cap[0])
|
|
503
|
+
extra["cap_lon"].append(cap[1])
|
|
504
|
+
extra["fp_wkb"].append(None if geom is None else geom.wkb)
|
|
505
|
+
if geom is not None:
|
|
506
|
+
cx, cy = geom.centroid.x, geom.centroid.y
|
|
507
|
+
k = 1.0 + (cx * cx + cy * cy) / (4.0 * _R_M * _R_M)
|
|
508
|
+
xmin, ymin, xmax, ymax = geom.bounds
|
|
509
|
+
vals = (geom.area / k**2 / 1e6, xmin, xmax, ymin, ymax)
|
|
510
|
+
else:
|
|
511
|
+
vals = (math.nan,) * 5
|
|
512
|
+
for key, v in zip(("footprint_km2", "xmin", "xmax", "ymin", "ymax"), vals):
|
|
513
|
+
extra[key].append(v)
|
|
514
|
+
wkt = row.get("footprint_wkt")
|
|
515
|
+
if (not isinstance(wkt, str) or not wkt.strip()) and rr is not None and len(rr[0][0]) >= 3:
|
|
516
|
+
wkt = _rings_wkt(rr)
|
|
517
|
+
wkts.append(wkt if isinstance(wkt, str) else None)
|
|
518
|
+
frame = obs[OBSERVATION_COLUMNS].copy()
|
|
519
|
+
frame["footprint_wkt"] = wkts
|
|
520
|
+
frame["product_id"] = frame["product_id"].astype(str)
|
|
521
|
+
for k, v in extra.items():
|
|
522
|
+
frame[k] = v
|
|
523
|
+
frame = frame.drop_duplicates("product_id", keep="last").reset_index(drop=True)
|
|
524
|
+
start = self._scalar("SELECT coalesce(max(idx) + 1, 0) FROM images")
|
|
525
|
+
frame.insert(0, "idx", np.arange(start, start + len(frame), dtype=np.int64))
|
|
526
|
+
cols = ["idx", *_OBS_TYPES, *_IMAGE_EXTRA]
|
|
527
|
+
sel = ", ".join(
|
|
528
|
+
f"CAST({c} AS {t}) AS {c}" for c, t in {"idx": "BIGINT", **_OBS_TYPES, **_IMAGE_EXTRA}.items()
|
|
529
|
+
)
|
|
530
|
+
self.con.register("_new_images", frame)
|
|
531
|
+
self.con.execute("BEGIN TRANSACTION")
|
|
532
|
+
try:
|
|
533
|
+
self.con.execute("DELETE FROM images WHERE product_id IN (SELECT product_id FROM _new_images)")
|
|
534
|
+
self.con.execute(
|
|
535
|
+
f"INSERT INTO images ({', '.join(cols)}, footprint, added_at) "
|
|
536
|
+
f"SELECT {sel}, CASE WHEN fp_wkb IS NULL THEN NULL ELSE ST_GeomFromWKB(fp_wkb) END, "
|
|
537
|
+
"current_localtimestamp() FROM _new_images"
|
|
538
|
+
)
|
|
539
|
+
self.con.execute("COMMIT")
|
|
540
|
+
except BaseException:
|
|
541
|
+
self.con.execute("ROLLBACK")
|
|
542
|
+
raise
|
|
543
|
+
finally:
|
|
544
|
+
self.con.unregister("_new_images")
|
|
545
|
+
return len(frame)
|
|
546
|
+
|
|
547
|
+
def add_index_rows(self, df: pd.DataFrame, dataset: str | None = "edr") -> int:
|
|
548
|
+
""":meth:`add_observations` of ``CumIndex`` rows (footprints from ``FP_WKT``, corners or ``FP_*``)."""
|
|
549
|
+
from .geometry import footprint_rings
|
|
550
|
+
from .observation import observations_from_cumindex
|
|
551
|
+
|
|
552
|
+
if df is None or len(df) == 0:
|
|
553
|
+
self._require_spatial()
|
|
554
|
+
self._create_images()
|
|
555
|
+
return 0
|
|
556
|
+
rings = [footprint_rings(r, "FP_WKT") for r in df.to_dict("records")]
|
|
557
|
+
return self.add_observations(observations_from_cumindex(df, dataset=dataset), rings=rings)
|
|
558
|
+
|
|
559
|
+
def _create_images(self) -> None:
|
|
560
|
+
cols = ", ".join(f"{c} {t}" for c, t in {"idx": "BIGINT", **_OBS_TYPES, **_IMAGE_EXTRA}.items())
|
|
561
|
+
self.con.execute(
|
|
562
|
+
f"CREATE TABLE IF NOT EXISTS images ({cols}, footprint GEOMETRY, added_at TIMESTAMP)"
|
|
563
|
+
)
|
|
564
|
+
|
|
565
|
+
def pairs(
|
|
566
|
+
self,
|
|
567
|
+
aoi: Any = None,
|
|
568
|
+
*,
|
|
569
|
+
convergence: tuple[float, float] = (10.0, 60.0),
|
|
570
|
+
ideal_convergence: tuple[float, float] = (15.0, 45.0),
|
|
571
|
+
max_incidence_diff: float = 20.0,
|
|
572
|
+
max_azimuth_diff: float = 30.0,
|
|
573
|
+
min_overlap: float = 0.1,
|
|
574
|
+
min_aoi_coverage: float = 0.0,
|
|
575
|
+
max_resolution_ratio: float | None = None,
|
|
576
|
+
same_instrument: bool = True,
|
|
577
|
+
require_geometry: bool = False,
|
|
578
|
+
min_time_sep_s: float = 60.0,
|
|
579
|
+
weights: dict[str, float] | None = None,
|
|
580
|
+
product_ids: Sequence[str] | None = None,
|
|
581
|
+
table: str | None = "pairs",
|
|
582
|
+
) -> pd.DataFrame:
|
|
583
|
+
"""Rank stereo pairs of ``images`` with a SQL spatial self-join.
|
|
584
|
+
|
|
585
|
+
Parameters are those of :func:`lunar_toolkit.stereo.find_stereo_pairs`, plus
|
|
586
|
+
*product_ids* (only these images) and *table* (where the result is stored;
|
|
587
|
+
None keeps it out of the database).
|
|
588
|
+
|
|
589
|
+
Returns
|
|
590
|
+
-------
|
|
591
|
+
DataFrame
|
|
592
|
+
:data:`lunar_toolkit.stereo.PAIR_COLUMNS`; ``index_1`` / ``index_2`` are
|
|
593
|
+
``images.idx``.
|
|
594
|
+
|
|
595
|
+
Notes
|
|
596
|
+
-----
|
|
597
|
+
Candidate pairs come from a bounding-box spatial join (``ST_Intersects_Extent``,
|
|
598
|
+
R-tree built by DuckDB's spatial join operator) of the footprints of each
|
|
599
|
+
hemisphere, clipped to the AOI; then the attribute filters, overlap area / fraction (``ST_Area`` of
|
|
600
|
+
``ST_Intersection`` in metres, divided by the projection's areal scale at the
|
|
601
|
+
centroid), AOI coverage, incidence / azimuth / emission / resolution / time
|
|
602
|
+
differences and the overlap centroid are SQL. The convergence angle and
|
|
603
|
+
``b_h`` are then computed for the candidates with numpy
|
|
604
|
+
(:func:`lunar_toolkit.stereo.convergence_angles`: sub-spacecraft point +
|
|
605
|
+
altitude, pushbroom-corrected), exactly as ``find_stereo_pairs``, and the
|
|
606
|
+
score is :func:`lunar_toolkit.stereo.score_pairs`. Footprint edges are great
|
|
607
|
+
circles in both (densified here), so overlap areas and fractions agree with
|
|
608
|
+
``find_stereo_pairs`` to ~1e-3 relative.
|
|
609
|
+
"""
|
|
610
|
+
import pandas as pd
|
|
611
|
+
|
|
612
|
+
from .geometry import _polar_hemi, aoi_geometry, aoi_region, load_aoi, ps_projection
|
|
613
|
+
from .stereo import PAIR_COLUMNS, convergence_angles, score_pairs
|
|
614
|
+
|
|
615
|
+
self._require_spatial()
|
|
616
|
+
self._create_images()
|
|
617
|
+
lo_c, hi_c = convergence
|
|
618
|
+
aoi_parts: dict[str, tuple[bytes, float]] = {}
|
|
619
|
+
if aoi is not None:
|
|
620
|
+
a = load_aoi(aoi)
|
|
621
|
+
region = aoi_region(a)
|
|
622
|
+
hemi = _polar_hemi(region) or ("S" if sum(region["lat"]) < 0 else "N")
|
|
623
|
+
geom = aoi_geometry(a, ps_projection(hemi))
|
|
624
|
+
k = 1.0 + (geom.centroid.x**2 + geom.centroid.y**2) / (4.0 * _R_M * _R_M)
|
|
625
|
+
aoi_parts[hemi] = (geom.wkb, geom.area / k**2 / 1e6)
|
|
626
|
+
hemis = [hemi]
|
|
627
|
+
else:
|
|
628
|
+
hemis = [
|
|
629
|
+
h
|
|
630
|
+
for (h,) in self.con.execute(
|
|
631
|
+
"SELECT DISTINCT hemi FROM images WHERE hemi IS NOT NULL"
|
|
632
|
+
).fetchall()
|
|
633
|
+
]
|
|
634
|
+
|
|
635
|
+
conds = ["a.obs_key <> b.obs_key"]
|
|
636
|
+
if same_instrument:
|
|
637
|
+
conds.append("a.instrument IS NOT DISTINCT FROM b.instrument")
|
|
638
|
+
conds += [
|
|
639
|
+
f"NOT coalesce(abs(b.t - a.t) < {float(min_time_sep_s)!r}, false)",
|
|
640
|
+
f"NOT coalesce(abs(b.incidence - a.incidence) > {float(max_incidence_diff)!r}, false)",
|
|
641
|
+
f"NOT coalesce(_az_diff(a.az, b.az) > {float(max_azimuth_diff)!r}, false)",
|
|
642
|
+
f"NOT coalesce(a.emission + b.emission < {lo_c - 5.0!r}, false)",
|
|
643
|
+
f"NOT coalesce(abs(b.emission - a.emission) > {hi_c + 5.0!r}, false)",
|
|
644
|
+
]
|
|
645
|
+
if require_geometry:
|
|
646
|
+
conds.append("a.has_geom AND b.has_geom")
|
|
647
|
+
if max_resolution_ratio is not None:
|
|
648
|
+
conds.append(
|
|
649
|
+
f"NOT coalesce({_ratio_sql('a.ps', 'b.ps')} > {float(max_resolution_ratio)!r}, false)"
|
|
650
|
+
)
|
|
651
|
+
|
|
652
|
+
frames = []
|
|
653
|
+
for hemi in hemis:
|
|
654
|
+
wkb, aoi_km2 = aoi_parts.get(hemi, (None, math.nan))
|
|
655
|
+
where = [f"hemi = {_sql_str(hemi)}", "footprint IS NOT NULL"]
|
|
656
|
+
params: list[Any] = []
|
|
657
|
+
if product_ids is not None:
|
|
658
|
+
self.con.register("_pair_ids", pd.DataFrame({"product_id": [str(p) for p in product_ids]}))
|
|
659
|
+
where.append("product_id IN (SELECT product_id FROM _pair_ids)")
|
|
660
|
+
g = "footprint"
|
|
661
|
+
if wkb is not None:
|
|
662
|
+
where.append("ST_Intersects(footprint, ST_GeomFromWKB(?))")
|
|
663
|
+
g = "ST_Intersection(footprint, ST_GeomFromWKB(?))"
|
|
664
|
+
params = [wkb, wkb]
|
|
665
|
+
self.con.execute(
|
|
666
|
+
"CREATE OR REPLACE TEMP TABLE _pair_fp AS SELECT *, _true_km2(g) AS g_km2 FROM ("
|
|
667
|
+
"SELECT idx, product_id, obs_key, instrument, start_time, epoch(start_time) AS t, incidence, "
|
|
668
|
+
"emission, sub_solar_azimuth AS az, pixel_scale AS ps, sub_spacecraft_lat AS sclat, "
|
|
669
|
+
"sub_spacecraft_lon AS sclon, spacecraft_altitude AS alt, "
|
|
670
|
+
"(sub_spacecraft_lat IS NOT NULL AND sub_spacecraft_lon IS NOT NULL "
|
|
671
|
+
"AND spacecraft_altitude IS NOT NULL) AS has_geom, "
|
|
672
|
+
"CASE WHEN center_lat IS NOT NULL AND center_lon IS NOT NULL THEN center_lat ELSE cap_lat END "
|
|
673
|
+
"AS vlat, CASE WHEN center_lat IS NOT NULL AND center_lon IS NOT NULL THEN center_lon "
|
|
674
|
+
f"ELSE cap_lon END AS vlon, {g} AS g FROM images WHERE {' AND '.join(where)}"
|
|
675
|
+
") WHERE ST_Area(g) > 0",
|
|
676
|
+
params or None,
|
|
677
|
+
)
|
|
678
|
+
# Bounding-box prefilter: DuckDB plans its R-tree spatial join only for a bare
|
|
679
|
+
# ST_Intersects[_Extent] (+ inequality) condition, so candidates go to a temp table.
|
|
680
|
+
self.con.execute(
|
|
681
|
+
"CREATE OR REPLACE TEMP TABLE _pair_cand AS SELECT a.idx AS i, b.idx AS j "
|
|
682
|
+
"FROM _pair_fp a JOIN _pair_fp b ON ST_Intersects_Extent(a.g, b.g) AND a.idx < b.idx"
|
|
683
|
+
)
|
|
684
|
+
aoi_sql = f"ov_km2 / {aoi_km2!r}" if wkb is not None else "NULL::DOUBLE"
|
|
685
|
+
cov_filter = f"AND aoi_coverage >= {max(min_aoi_coverage, 1e-9)!r}" if wkb is not None else ""
|
|
686
|
+
self.con.execute(
|
|
687
|
+
"CREATE OR REPLACE TEMP TABLE _pair_ab AS SELECT a, b FROM _pair_cand c "
|
|
688
|
+
"JOIN (SELECT * EXCLUDE (g) FROM _pair_fp) a ON a.idx = c.i "
|
|
689
|
+
"JOIN (SELECT * EXCLUDE (g) FROM _pair_fp) b ON b.idx = c.j"
|
|
690
|
+
)
|
|
691
|
+
# Each step is materialised: otherwise the a / b filters become join conditions
|
|
692
|
+
# and ST_Intersection runs before them, once per reference to its result.
|
|
693
|
+
self.con.execute(f"""
|
|
694
|
+
CREATE OR REPLACE TEMP TABLE _pair_ov AS
|
|
695
|
+
SELECT a.idx AS index_1, b.idx AS index_2, a.product_id AS product_id_1,
|
|
696
|
+
b.product_id AS product_id_2, a.instrument,
|
|
697
|
+
a.start_time AS start_time_1, b.start_time AS start_time_2,
|
|
698
|
+
abs(b.t - a.t) / 86400.0 AS dt_days,
|
|
699
|
+
abs(b.emission - a.emission) AS convergence_min,
|
|
700
|
+
a.emission + b.emission AS convergence_max,
|
|
701
|
+
a.incidence AS incidence_1, b.incidence AS incidence_2,
|
|
702
|
+
abs(b.incidence - a.incidence) AS incidence_diff,
|
|
703
|
+
a.az AS azimuth_1, b.az AS azimuth_2, _az_diff(a.az, b.az) AS azimuth_diff,
|
|
704
|
+
a.emission AS emission_1, b.emission AS emission_2,
|
|
705
|
+
a.ps AS pixel_scale_1, b.ps AS pixel_scale_2,
|
|
706
|
+
{_ratio_sql("a.ps", "b.ps")} AS resolution_ratio,
|
|
707
|
+
a.g_km2 AS g1_km2, b.g_km2 AS g2_km2,
|
|
708
|
+
a.has_geom AND b.has_geom AS has_geom,
|
|
709
|
+
a.sclat AS sclat_1, a.sclon AS sclon_1, a.alt AS alt_1,
|
|
710
|
+
b.sclat AS sclat_2, b.sclon AS sclon_2, b.alt AS alt_2,
|
|
711
|
+
a.vlat AS vlat_1, a.vlon AS vlon_1, b.vlat AS vlat_2, b.vlon AS vlon_2
|
|
712
|
+
FROM _pair_ab
|
|
713
|
+
WHERE {" AND ".join(conds)}
|
|
714
|
+
""")
|
|
715
|
+
self.con.execute(
|
|
716
|
+
"CREATE OR REPLACE TEMP TABLE _pair_x AS SELECT o.*, ST_Intersection(a.g, b.g) AS ov "
|
|
717
|
+
"FROM _pair_ov o JOIN _pair_fp a ON a.idx = o.index_1 JOIN _pair_fp b ON b.idx = o.index_2"
|
|
718
|
+
)
|
|
719
|
+
self.con.execute(
|
|
720
|
+
"CREATE OR REPLACE TEMP TABLE _pair_m AS SELECT * EXCLUDE (ov, c), ST_X(c) AS cx, "
|
|
721
|
+
"ST_Y(c) AS cy FROM (SELECT *, ST_Area(ov) AS ov_m2, ST_Centroid(ov) AS c FROM _pair_x "
|
|
722
|
+
"WHERE NOT ST_IsEmpty(ov))"
|
|
723
|
+
)
|
|
724
|
+
df = self.con.execute(f"""
|
|
725
|
+
SELECT * FROM (
|
|
726
|
+
SELECT *, ov_km2 / greatest(least(g1_km2, g2_km2), 1e-12) AS overlap_frac,
|
|
727
|
+
{aoi_sql} AS aoi_coverage
|
|
728
|
+
FROM (SELECT * EXCLUDE (ov_m2), ov_m2 / pow(_ps_k(cx, cy), 2) / 1e6 AS ov_km2 FROM _pair_m)
|
|
729
|
+
WHERE ov_km2 > 0
|
|
730
|
+
)
|
|
731
|
+
WHERE overlap_frac >= {float(min_overlap)!r} {cov_filter}
|
|
732
|
+
ORDER BY index_1, index_2
|
|
733
|
+
""").df()
|
|
734
|
+
df["overlap_lat"] = _ps_lat(df["cx"].to_numpy(float), df["cy"].to_numpy(float), hemi)
|
|
735
|
+
df["overlap_lon"] = _ps_lon(df["cx"].to_numpy(float), df["cy"].to_numpy(float), hemi)
|
|
736
|
+
frames.append(df)
|
|
737
|
+
for tmp in ("_pair_m", "_pair_x", "_pair_ov", "_pair_ab", "_pair_cand", "_pair_fp"):
|
|
738
|
+
self.con.execute(f"DROP TABLE IF EXISTS {tmp}")
|
|
739
|
+
if product_ids is not None:
|
|
740
|
+
self.con.unregister("_pair_ids")
|
|
741
|
+
|
|
742
|
+
frames = [f for f in frames if len(f)]
|
|
743
|
+
if not frames:
|
|
744
|
+
res = pd.DataFrame(columns=PAIR_COLUMNS)
|
|
745
|
+
else:
|
|
746
|
+
res = pd.concat(frames, ignore_index=True).sort_values(["index_1", "index_2"], kind="stable")
|
|
747
|
+
res = res.reset_index(drop=True)
|
|
748
|
+
geom = res["has_geom"].fillna(False).to_numpy(bool)
|
|
749
|
+
conv = np.full(len(res), np.nan)
|
|
750
|
+
bh = np.full(len(res), np.nan)
|
|
751
|
+
if geom.any():
|
|
752
|
+
r = res[geom]
|
|
753
|
+
|
|
754
|
+
def col(c: str) -> np.ndarray:
|
|
755
|
+
return r[c].to_numpy(float)
|
|
756
|
+
|
|
757
|
+
conv[geom], bh[geom] = convergence_angles(
|
|
758
|
+
(col("overlap_lat"), col("overlap_lon")),
|
|
759
|
+
(col("sclat_1"), col("sclon_1"), col("alt_1")),
|
|
760
|
+
(col("sclat_2"), col("sclon_2"), col("alt_2")),
|
|
761
|
+
(col("vlat_1"), col("vlon_1")),
|
|
762
|
+
(col("vlat_2"), col("vlon_2")),
|
|
763
|
+
)
|
|
764
|
+
res["convergence"], res["b_h"] = conv, bh
|
|
765
|
+
res["overlap_km2"] = res["ov_km2"]
|
|
766
|
+
keep = np.isnan(conv) | ((conv >= lo_c) & (conv <= hi_c))
|
|
767
|
+
res = res[keep].reset_index(drop=True)
|
|
768
|
+
for c in ("start_time_1", "start_time_2"):
|
|
769
|
+
res[c] = pd.to_datetime(res[c])
|
|
770
|
+
res["score"] = score_pairs(
|
|
771
|
+
res,
|
|
772
|
+
convergence=convergence,
|
|
773
|
+
ideal_convergence=ideal_convergence,
|
|
774
|
+
max_incidence_diff=max_incidence_diff,
|
|
775
|
+
max_azimuth_diff=max_azimuth_diff,
|
|
776
|
+
weights=weights,
|
|
777
|
+
aoi=aoi is not None,
|
|
778
|
+
)
|
|
779
|
+
res = (
|
|
780
|
+
res[PAIR_COLUMNS].sort_values("score", ascending=False, kind="stable").reset_index(drop=True)
|
|
781
|
+
)
|
|
782
|
+
if table is not None:
|
|
783
|
+
name = _ident(table)
|
|
784
|
+
self.con.register("_pairs_result", res)
|
|
785
|
+
try:
|
|
786
|
+
self.con.execute(f'CREATE OR REPLACE TABLE "{name}" AS SELECT * FROM _pairs_result')
|
|
787
|
+
finally:
|
|
788
|
+
self.con.unregister("_pairs_result")
|
|
789
|
+
return res
|
|
790
|
+
|
|
791
|
+
def set_rejection_params(self, config: SfsConfig | None = None, **overrides: Any) -> None:
|
|
792
|
+
"""Thresholds of the ``image_rejections`` view, from *config* (default :class:`~lunar_toolkit.sfs.SfsConfig`).
|
|
793
|
+
|
|
794
|
+
Keyword overrides: ``max_offset_p85``, ``max_large_residual_fraction``,
|
|
795
|
+
``large_residual_px``, ``min_matches`` (None disables the first two rules).
|
|
796
|
+
"""
|
|
797
|
+
from .sfs import SfsConfig
|
|
798
|
+
|
|
799
|
+
cfg = config or SfsConfig()
|
|
800
|
+
vals = {
|
|
801
|
+
"max_offset_p85": cfg.max_offset_p85,
|
|
802
|
+
"max_large_residual_fraction": cfg.max_large_residual_fraction,
|
|
803
|
+
"large_residual_px": cfg.large_residual_px,
|
|
804
|
+
"min_matches": cfg.min_matches,
|
|
805
|
+
}
|
|
806
|
+
unknown = set(overrides) - set(vals)
|
|
807
|
+
if unknown:
|
|
808
|
+
raise TypeError(f"Unknown rejection parameter(s): {', '.join(sorted(unknown))}")
|
|
809
|
+
vals.update(overrides)
|
|
810
|
+
self.con.execute("DELETE FROM rejection_params")
|
|
811
|
+
self.con.execute(
|
|
812
|
+
"INSERT INTO rejection_params VALUES (?, ?, ?, ?)",
|
|
813
|
+
[
|
|
814
|
+
vals["max_offset_p85"],
|
|
815
|
+
vals["max_large_residual_fraction"],
|
|
816
|
+
vals["large_residual_px"],
|
|
817
|
+
int(vals["min_matches"] or 0),
|
|
818
|
+
],
|
|
819
|
+
)
|
|
820
|
+
|
|
821
|
+
def load_bundle_adjust(self, prefix: Path | str, run: str | None = None) -> dict[str, int]:
|
|
822
|
+
"""Load ASP ``bundle_adjust`` outputs ``<prefix>-*`` as run *run* (replacing it).
|
|
823
|
+
|
|
824
|
+
*run* defaults to the prefix's directory name (``ba``, ``ba_refine``). Reads,
|
|
825
|
+
when present: ``-initial`` / ``-final_residuals_stats.txt`` and
|
|
826
|
+
``_raw_pixels.txt`` (``ba_residual_stats``, ``ba_pixel_residuals``),
|
|
827
|
+
``-mapproj_match_offset_stats.txt`` (``ba_match_offsets``; -1 becomes NULL),
|
|
828
|
+
``-initial`` / ``-final_residuals_pointmap.csv`` (``ba_points``, with polar
|
|
829
|
+
stereographic ``x`` / ``y``), ``-convergence_angles.txt`` (``ba_convergence``)
|
|
830
|
+
and ``-camera_offsets.txt`` (``ba_camera_offsets``).
|
|
831
|
+
|
|
832
|
+
Returns
|
|
833
|
+
-------
|
|
834
|
+
dict
|
|
835
|
+
Rows loaded per table (empty when no report exists).
|
|
836
|
+
"""
|
|
837
|
+
from . import asp
|
|
838
|
+
|
|
839
|
+
prefix = Path(prefix)
|
|
840
|
+
run = run or prefix.parent.name or prefix.name
|
|
841
|
+
f = {name: Path(f"{prefix}-{name}") for name in _BA_FILES}
|
|
842
|
+
found = [p for p in f.values() if p.exists()]
|
|
843
|
+
if not found:
|
|
844
|
+
return {}
|
|
845
|
+
stats = {
|
|
846
|
+
stage: asp.read_residual_stats(f[f"{stage}_residuals_stats.txt"])
|
|
847
|
+
for stage in ("initial", "final")
|
|
848
|
+
if f[f"{stage}_residuals_stats.txt"].exists()
|
|
849
|
+
}
|
|
850
|
+
raw = {
|
|
851
|
+
stage: _read_raw_pixels(f[f"{stage}_residuals_raw_pixels.txt"])
|
|
852
|
+
for stage in ("initial", "final")
|
|
853
|
+
if f[f"{stage}_residuals_raw_pixels.txt"].exists()
|
|
854
|
+
}
|
|
855
|
+
offsets = (
|
|
856
|
+
asp.read_mapproj_offset_stats(f["mapproj_match_offset_stats.txt"])
|
|
857
|
+
if f["mapproj_match_offset_stats.txt"].exists()
|
|
858
|
+
else None
|
|
859
|
+
)
|
|
860
|
+
self.con.execute("BEGIN TRANSACTION")
|
|
861
|
+
try:
|
|
862
|
+
self._insert_ba(run, stats=stats, raw=raw, offsets=offsets)
|
|
863
|
+
for stage in ("initial", "final"):
|
|
864
|
+
p = f[f"{stage}_residuals_pointmap.csv"]
|
|
865
|
+
if p.exists():
|
|
866
|
+
self._load_points(run, stage, p)
|
|
867
|
+
if f["convergence_angles.txt"].exists():
|
|
868
|
+
rows = _text_rows(self.con, f["convergence_angles.txt"], " ", 6)
|
|
869
|
+
self.con.execute(
|
|
870
|
+
"INSERT INTO ba_convergence SELECT ?, parse_filename(c1), parse_filename(c2), "
|
|
871
|
+
"upper(split_part(parse_filename(c1), '.', 1)), upper(split_part(parse_filename(c2), '.', 1)), "
|
|
872
|
+
f"TRY_CAST(c3 AS DOUBLE), TRY_CAST(c4 AS DOUBLE), TRY_CAST(c5 AS DOUBLE), "
|
|
873
|
+
f"CAST(TRY_CAST(c6 AS DOUBLE) AS BIGINT) FROM ({rows}) WHERE c6 IS NOT NULL",
|
|
874
|
+
[run],
|
|
875
|
+
)
|
|
876
|
+
if f["camera_offsets.txt"].exists():
|
|
877
|
+
rows = _text_rows(self.con, f["camera_offsets.txt"], " ", 3)
|
|
878
|
+
self.con.execute(
|
|
879
|
+
"INSERT INTO ba_camera_offsets SELECT ?, parse_filename(c1), "
|
|
880
|
+
"upper(split_part(parse_filename(c1), '.', 1)), TRY_CAST(c2 AS DOUBLE), "
|
|
881
|
+
f"TRY_CAST(c3 AS DOUBLE) FROM ({rows}) WHERE c3 IS NOT NULL",
|
|
882
|
+
[run],
|
|
883
|
+
)
|
|
884
|
+
self.con.execute("DELETE FROM ba_runs WHERE run = ?", [run])
|
|
885
|
+
self.con.execute(
|
|
886
|
+
"INSERT INTO ba_runs VALUES (?, ?, current_localtimestamp(), ?)",
|
|
887
|
+
[run, str(prefix), [p.name for p in found]],
|
|
888
|
+
)
|
|
889
|
+
self.con.execute("COMMIT")
|
|
890
|
+
except BaseException:
|
|
891
|
+
self.con.execute("ROLLBACK")
|
|
892
|
+
raise
|
|
893
|
+
return {t: int(self._scalar(f"SELECT count(*) FROM {t} WHERE run = ?", [run])) for t in BA_TABLES}
|
|
894
|
+
|
|
895
|
+
def _insert_ba(
|
|
896
|
+
self,
|
|
897
|
+
run: str,
|
|
898
|
+
*,
|
|
899
|
+
stats: Mapping[str, Mapping[str, ResidualStats]] | None = None,
|
|
900
|
+
raw: Mapping[str, pd.DataFrame] | None = None,
|
|
901
|
+
offsets: Mapping[str, MatchOffsetStats] | None = None,
|
|
902
|
+
) -> None:
|
|
903
|
+
"""Replace run *run* with parsed reports: per stage residual stats / raw norms, offsets."""
|
|
904
|
+
import pandas as pd
|
|
905
|
+
|
|
906
|
+
for t in BA_TABLES:
|
|
907
|
+
self.con.execute(f"DELETE FROM {t} WHERE run = ?", [run])
|
|
908
|
+
rows = [
|
|
909
|
+
(
|
|
910
|
+
run,
|
|
911
|
+
stage,
|
|
912
|
+
Path(s.image).name,
|
|
913
|
+
s.image,
|
|
914
|
+
_product_id(s.image),
|
|
915
|
+
_num(s.mean),
|
|
916
|
+
_num(s.median),
|
|
917
|
+
s.count,
|
|
918
|
+
)
|
|
919
|
+
for stage, by_image in (stats or {}).items()
|
|
920
|
+
for s in by_image.values()
|
|
921
|
+
]
|
|
922
|
+
if rows:
|
|
923
|
+
self.con.executemany("INSERT INTO ba_residual_stats VALUES (?, ?, ?, ?, ?, ?, ?, ?)", rows)
|
|
924
|
+
for stage, norms in (raw or {}).items():
|
|
925
|
+
if len(norms):
|
|
926
|
+
frame = pd.DataFrame(
|
|
927
|
+
{
|
|
928
|
+
"run": run,
|
|
929
|
+
"stage": stage,
|
|
930
|
+
"image": norms["image"].astype(str),
|
|
931
|
+
"product_id": [_product_id(i) for i in norms["image"]],
|
|
932
|
+
"norm_px": norms["norm_px"].astype(float),
|
|
933
|
+
}
|
|
934
|
+
)
|
|
935
|
+
self.con.register("_raw", frame)
|
|
936
|
+
try:
|
|
937
|
+
self.con.execute("INSERT INTO ba_pixel_residuals SELECT * FROM _raw")
|
|
938
|
+
finally:
|
|
939
|
+
self.con.unregister("_raw")
|
|
940
|
+
orows = [
|
|
941
|
+
(
|
|
942
|
+
run,
|
|
943
|
+
Path(o.image).name,
|
|
944
|
+
o.image,
|
|
945
|
+
_product_id(o.image),
|
|
946
|
+
*(None if not v >= 0 else v for v in (o.p25, o.p50, o.p75, o.p85, o.p95)),
|
|
947
|
+
o.count,
|
|
948
|
+
)
|
|
949
|
+
for o in (offsets or {}).values()
|
|
950
|
+
]
|
|
951
|
+
if orows:
|
|
952
|
+
self.con.executemany("INSERT INTO ba_match_offsets VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", orows)
|
|
953
|
+
|
|
954
|
+
def _load_points(self, run: str, stage: str, path: Path) -> None:
|
|
955
|
+
rows = _text_rows(self.con, path, ",", 5)
|
|
956
|
+
self.con.execute(
|
|
957
|
+
f"""
|
|
958
|
+
INSERT INTO ba_points
|
|
959
|
+
SELECT run, stage, lon, lat, height_m, res, nobs, kind, hemi,
|
|
960
|
+
_ps_x(lat, lon, hemi), _ps_y(lat, lon, hemi)
|
|
961
|
+
FROM (
|
|
962
|
+
SELECT ? AS run, ? AS stage, TRY_CAST(c1 AS DOUBLE) AS lon, TRY_CAST(c2 AS DOUBLE) AS lat,
|
|
963
|
+
TRY_CAST(c3 AS DOUBLE) AS height_m, TRY_CAST(c4 AS DOUBLE) AS res,
|
|
964
|
+
TRY_CAST(split_part(trim(c5), ' ', 1) AS INTEGER) AS nobs,
|
|
965
|
+
CASE WHEN c5 LIKE '%GCP%' THEN 'gcp' WHEN c5 LIKE '%DEM%' THEN 'dem' ELSE 'tie' END AS kind,
|
|
966
|
+
CASE WHEN TRY_CAST(c2 AS DOUBLE) < 0 THEN 'S' ELSE 'N' END AS hemi
|
|
967
|
+
FROM ({rows}) WHERE c5 IS NOT NULL
|
|
968
|
+
) WHERE lon IS NOT NULL AND lat IS NOT NULL
|
|
969
|
+
""",
|
|
970
|
+
[run, stage],
|
|
971
|
+
)
|
|
972
|
+
|
|
973
|
+
def runs(self) -> list[str]:
|
|
974
|
+
"""Loaded bundle-adjustment runs."""
|
|
975
|
+
return [r for (r,) in self.con.execute("SELECT run FROM ba_runs ORDER BY loaded_at, run").fetchall()]
|
|
976
|
+
|
|
977
|
+
def rejections(self, run: str) -> dict[str, str]:
|
|
978
|
+
"""``image_rejections`` of *run*: image file name -> reason."""
|
|
979
|
+
rows = self.con.execute("SELECT image, reason FROM image_rejections WHERE run = ?", [run]).fetchall()
|
|
980
|
+
return dict(rows)
|
|
981
|
+
|
|
982
|
+
def ba_report(self, run: str) -> pd.DataFrame | None:
|
|
983
|
+
"""Per-image table of *run* in :func:`lunar_toolkit.sfs_report.read_ba_reports` form, or None."""
|
|
984
|
+
if run not in self.runs():
|
|
985
|
+
return None
|
|
986
|
+
cols = ["image"] + [f"{k}_{s}" for k in ("initial", "final") for s in ("mean", "median", "count")]
|
|
987
|
+
cols += [f"offset_p{p}" for p in (25, 50, 75, 85, 95)] + ["offset_count"]
|
|
988
|
+
df = self.con.execute(
|
|
989
|
+
f"SELECT {', '.join(cols)} FROM ba_image_stats WHERE run = ? AND "
|
|
990
|
+
"(initial_count IS NOT NULL OR final_count IS NOT NULL OR offset_count IS NOT NULL) "
|
|
991
|
+
"ORDER BY image",
|
|
992
|
+
[run],
|
|
993
|
+
).df()
|
|
994
|
+
for c in cols[1:]:
|
|
995
|
+
df[c] = df[c].astype(float)
|
|
996
|
+
return df
|
|
997
|
+
|
|
998
|
+
def convergence(self, run: str) -> pd.DataFrame:
|
|
999
|
+
"""``ba_convergence`` of *run* as ``left``, ``right``, ``p25``, ``p50``, ``p75``, ``matches``."""
|
|
1000
|
+
return self.con.execute(
|
|
1001
|
+
"SELECT left_image AS left, right_image AS right, p25, p50, p75, matches "
|
|
1002
|
+
"FROM ba_convergence WHERE run = ?",
|
|
1003
|
+
[run],
|
|
1004
|
+
).df()
|
|
1005
|
+
|
|
1006
|
+
def point_residuals(self, run: str, stage: str = "final") -> np.ndarray:
|
|
1007
|
+
"""``mean_residual_px`` of the *run* / *stage* residual point map."""
|
|
1008
|
+
rows = self.con.execute(
|
|
1009
|
+
"SELECT mean_residual_px FROM ba_points WHERE run = ? AND stage = ?", [run, stage]
|
|
1010
|
+
).fetchnumpy()
|
|
1011
|
+
return np.asarray(rows["mean_residual_px"], float)
|
|
1012
|
+
|
|
1013
|
+
def summary(self) -> pd.DataFrame:
|
|
1014
|
+
"""The ``ba_run_summary`` view."""
|
|
1015
|
+
return self.con.execute("SELECT * FROM ba_run_summary").df()
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
_BA_FILES = (
|
|
1019
|
+
"initial_residuals_stats.txt",
|
|
1020
|
+
"final_residuals_stats.txt",
|
|
1021
|
+
"initial_residuals_raw_pixels.txt",
|
|
1022
|
+
"final_residuals_raw_pixels.txt",
|
|
1023
|
+
"mapproj_match_offset_stats.txt",
|
|
1024
|
+
"initial_residuals_pointmap.csv",
|
|
1025
|
+
"final_residuals_pointmap.csv",
|
|
1026
|
+
"convergence_angles.txt",
|
|
1027
|
+
"camera_offsets.txt",
|
|
1028
|
+
)
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
def _great_circle_rings(rings: list, step_deg: float = GREAT_CIRCLE_STEP_DEG) -> list:
|
|
1032
|
+
"""*rings* with vertices inserted along great circles, at most *step_deg* apart."""
|
|
1033
|
+
from .stereo import unit_vector
|
|
1034
|
+
|
|
1035
|
+
out = []
|
|
1036
|
+
for poly in rings:
|
|
1037
|
+
dense = []
|
|
1038
|
+
for ring in poly:
|
|
1039
|
+
r = list(ring[:-1] if len(ring) > 1 and tuple(ring[0]) == tuple(ring[-1]) else ring)
|
|
1040
|
+
if len(r) < 3:
|
|
1041
|
+
dense.append(r)
|
|
1042
|
+
continue
|
|
1043
|
+
a = np.asarray(r, float)
|
|
1044
|
+
v = unit_vector(a[:, 1], a[:, 0])
|
|
1045
|
+
w = np.roll(v, -1, axis=0)
|
|
1046
|
+
theta = np.arctan2(np.linalg.norm(np.cross(v, w), axis=1), np.sum(v * w, axis=1))
|
|
1047
|
+
pts: list[tuple[float, float]] = []
|
|
1048
|
+
for k, (vk, wk, th) in enumerate(zip(v, w, theta)):
|
|
1049
|
+
pts.append((float(a[k, 0]), float(a[k, 1])))
|
|
1050
|
+
n = int(math.ceil(math.degrees(th) / step_deg))
|
|
1051
|
+
if n < 2 or th < 1e-12:
|
|
1052
|
+
continue
|
|
1053
|
+
f = np.arange(1, n)[:, None] / n
|
|
1054
|
+
q = (np.sin((1 - f) * th) * vk + np.sin(f * th) * wk) / math.sin(th)
|
|
1055
|
+
lat = np.degrees(np.arcsin(np.clip(q[:, 2], -1, 1)))
|
|
1056
|
+
lon = np.degrees(np.arctan2(q[:, 1], q[:, 0]))
|
|
1057
|
+
# keep the ring's longitude unwrapping (values may lie outside 0..360)
|
|
1058
|
+
lon = a[k, 0] + (lon - a[k, 0] + 180.0) % 360.0 - 180.0
|
|
1059
|
+
pts += list(zip(lon.tolist(), lat.tolist()))
|
|
1060
|
+
dense.append(pts)
|
|
1061
|
+
out.append(dense)
|
|
1062
|
+
return out
|
|
1063
|
+
|
|
1064
|
+
|
|
1065
|
+
def _ratio_sql(a: str, b: str) -> str:
|
|
1066
|
+
return f"(CASE WHEN {a} >= {b} THEN {a} / {b} ELSE {b} / {a} END)"
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def _ps_lat(x: np.ndarray, y: np.ndarray, hemi: str) -> np.ndarray:
|
|
1070
|
+
sgn = 1.0 if hemi == "S" else -1.0
|
|
1071
|
+
return sgn * (np.degrees(2.0 * np.arctan(np.hypot(x, y) / (2.0 * _R_M))) - 90.0)
|
|
1072
|
+
|
|
1073
|
+
|
|
1074
|
+
def _ps_lon(x: np.ndarray, y: np.ndarray, hemi: str) -> np.ndarray:
|
|
1075
|
+
return np.degrees(np.arctan2(x, y if hemi == "S" else -y)) % 360.0
|
|
1076
|
+
|
|
1077
|
+
|
|
1078
|
+
def _rings_wkt(rings: list) -> str:
|
|
1079
|
+
from shapely.geometry import MultiPolygon, Polygon
|
|
1080
|
+
|
|
1081
|
+
polys = [Polygon(r[0], [h for h in r[1:] if len(h) >= 3]) for r in rings if len(r[0]) >= 3]
|
|
1082
|
+
return (polys[0] if len(polys) == 1 else MultiPolygon(polys)).wkt
|
|
1083
|
+
|
|
1084
|
+
|
|
1085
|
+
def _text_rows(con: Any, path: Path, delim: str, n: int) -> str:
|
|
1086
|
+
"""SQL reading *path* as *n* VARCHAR columns ``c1``.. (``#`` lines dropped, short rows NULL-padded)."""
|
|
1087
|
+
cols = "{" + ", ".join(f"'c{k}': 'VARCHAR'" for k in range(1, n + 1)) + "}"
|
|
1088
|
+
return (
|
|
1089
|
+
f"SELECT * FROM read_csv({_sql_str(path)}, delim={_sql_str(delim)}, header=false, columns={cols}, "
|
|
1090
|
+
"null_padding=true, quote='', escape='', auto_detect=false, strict_mode=false) "
|
|
1091
|
+
"WHERE NOT starts_with(trim(c1), '#')"
|
|
1092
|
+
)
|
|
1093
|
+
|
|
1094
|
+
|
|
1095
|
+
def _read_raw_pixels(path: Path) -> pd.DataFrame:
|
|
1096
|
+
"""``image``, ``norm_px`` of a ``*_residuals_raw_pixels.txt`` (``name, count`` then ``ex, ey`` lines)."""
|
|
1097
|
+
import pandas as pd
|
|
1098
|
+
|
|
1099
|
+
df = pd.read_csv(path, header=None, names=["a", "b"], dtype=str, skipinitialspace=True, comment=None)
|
|
1100
|
+
a = pd.to_numeric(df["a"], errors="coerce")
|
|
1101
|
+
head = a.isna().to_numpy()
|
|
1102
|
+
if not head.any():
|
|
1103
|
+
return pd.DataFrame({"image": pd.Series(dtype=str), "norm_px": pd.Series(dtype=float)})
|
|
1104
|
+
names = df["a"].where(head).ffill()
|
|
1105
|
+
keep = ~head & names.notna().to_numpy()
|
|
1106
|
+
ex, ey = a[keep].to_numpy(float), pd.to_numeric(df["b"][keep], errors="coerce").to_numpy(float)
|
|
1107
|
+
image = names[keep].map(lambda s: Path(str(s).strip()).name)
|
|
1108
|
+
return pd.DataFrame({"image": image.to_numpy(), "norm_px": np.hypot(ex, ey)})
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
def find_stereo_pairs_duckdb(
|
|
1112
|
+
obs_df: pd.DataFrame, aoi: Any = None, *, workspace: Workspace | None = None, **kwargs: Any
|
|
1113
|
+
) -> pd.DataFrame:
|
|
1114
|
+
""":func:`lunar_toolkit.stereo.find_stereo_pairs` through :meth:`Workspace.pairs`.
|
|
1115
|
+
|
|
1116
|
+
The observations are written to ``images`` of *workspace* (default: a scratch
|
|
1117
|
+
in-memory one) and ``index_1`` / ``index_2`` are mapped back to *obs_df* row
|
|
1118
|
+
labels; keyword arguments are those of ``find_stereo_pairs``.
|
|
1119
|
+
"""
|
|
1120
|
+
import pandas as pd
|
|
1121
|
+
|
|
1122
|
+
from .stereo import PAIR_COLUMNS
|
|
1123
|
+
|
|
1124
|
+
if obs_df is None or len(obs_df) < 2:
|
|
1125
|
+
return pd.DataFrame(columns=PAIR_COLUMNS)
|
|
1126
|
+
pids = obs_df["product_id"].astype(str)
|
|
1127
|
+
if pids.duplicated().any():
|
|
1128
|
+
raise ValueError("find_stereo_pairs_duckdb needs unique product_id values")
|
|
1129
|
+
label = dict(zip(pids, obs_df.index))
|
|
1130
|
+
ws = workspace or Workspace(":memory:")
|
|
1131
|
+
try:
|
|
1132
|
+
ws.add_observations(obs_df)
|
|
1133
|
+
res = ws.pairs(
|
|
1134
|
+
aoi, product_ids=list(pids), table="pairs" if workspace is not None else None, **kwargs
|
|
1135
|
+
)
|
|
1136
|
+
finally:
|
|
1137
|
+
if workspace is None:
|
|
1138
|
+
ws.close()
|
|
1139
|
+
res["index_1"] = [label[p] for p in res["product_id_1"]]
|
|
1140
|
+
res["index_2"] = [label[p] for p in res["product_id_2"]]
|
|
1141
|
+
return res
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def flag_images_sql(
|
|
1145
|
+
residual_stats: Mapping[str, ResidualStats],
|
|
1146
|
+
residual_norms: Mapping[str, np.ndarray],
|
|
1147
|
+
offsets: Mapping[str, MatchOffsetStats],
|
|
1148
|
+
**thresholds: Any,
|
|
1149
|
+
) -> dict[str, str]:
|
|
1150
|
+
"""``image_rejections`` of parsed final-stage reports (see :func:`lunar_toolkit.sfs.flag_images`)."""
|
|
1151
|
+
import pandas as pd
|
|
1152
|
+
|
|
1153
|
+
raw = pd.DataFrame(
|
|
1154
|
+
{
|
|
1155
|
+
"image": [n for n, v in residual_norms.items() for _ in range(len(v))],
|
|
1156
|
+
"norm_px": np.concatenate([np.asarray(v, float) for v in residual_norms.values()])
|
|
1157
|
+
if residual_norms
|
|
1158
|
+
else np.empty(0),
|
|
1159
|
+
}
|
|
1160
|
+
)
|
|
1161
|
+
with Workspace(":memory:", spatial=False, threads=1) as ws:
|
|
1162
|
+
ws.set_rejection_params(**thresholds)
|
|
1163
|
+
stats = {k: _with_image(v, k) for k, v in residual_stats.items()}
|
|
1164
|
+
offs = {k: _with_image(v, k) for k, v in offsets.items()}
|
|
1165
|
+
ws._insert_ba("_", stats={"final": stats}, raw={"final": raw}, offsets=offs)
|
|
1166
|
+
return ws.rejections("_")
|
|
1167
|
+
|
|
1168
|
+
|
|
1169
|
+
def _with_image(obj: Any, name: str) -> Any:
|
|
1170
|
+
"""*obj* (a frozen report row) keyed by *name*: its ``image`` replaced when the names differ."""
|
|
1171
|
+
import dataclasses
|
|
1172
|
+
|
|
1173
|
+
return obj if Path(obj.image).name == name else dataclasses.replace(obj, image=name)
|