reconstruct3d 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reconstruct3d/__init__.py +17 -0
- reconstruct3d/api.py +218 -0
- reconstruct3d/bundle_adjust.py +370 -0
- reconstruct3d/calibrate.py +199 -0
- reconstruct3d/cgal_mesh/CMakeLists.txt +22 -0
- reconstruct3d/cgal_mesh/mesh_reconstruct.cpp +312 -0
- reconstruct3d/chunked.py +335 -0
- reconstruct3d/cli.py +448 -0
- reconstruct3d/core.py +515 -0
- reconstruct3d/dense_mvs.py +256 -0
- reconstruct3d/init_sfm.py +151 -0
- reconstruct3d/mesh.py +99 -0
- reconstruct3d/track_sfm.py +253 -0
- reconstruct3d/viewer.py +108 -0
- reconstruct3d-0.1.0.dist-info/METADATA +416 -0
- reconstruct3d-0.1.0.dist-info/RECORD +19 -0
- reconstruct3d-0.1.0.dist-info/WHEEL +4 -0
- reconstruct3d-0.1.0.dist-info/entry_points.txt +2 -0
- reconstruct3d-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""reconstruct3d — pipeline offline de reconstrucción 3D desde video monocular.
|
|
2
|
+
|
|
3
|
+
Structure-from-Motion incremental + Bundle Adjustment + densificación MVS +
|
|
4
|
+
mallado con CGAL, con front-ends de features intercambiables (SIFT, ORB,
|
|
5
|
+
SuperPoint+LightGlue).
|
|
6
|
+
|
|
7
|
+
API rápida:
|
|
8
|
+
|
|
9
|
+
from reconstruct3d import Pipeline
|
|
10
|
+
pipe = Pipeline("outputs/run1", frontend="sift", camera="camera.json")
|
|
11
|
+
pipe.run("video.mp4", stages=["extract", "init", "track", "dense", "mesh"])
|
|
12
|
+
"""
|
|
13
|
+
from reconstruct3d.api import Pipeline, run_all, ALL_STAGES, ARTIFACTS
|
|
14
|
+
from reconstruct3d.core import CameraConfig
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|
|
17
|
+
__all__ = ["Pipeline", "run_all", "CameraConfig", "ALL_STAGES", "ARTIFACTS"]
|
reconstruct3d/api.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""API de alto nivel de reconstruct3d.
|
|
2
|
+
|
|
3
|
+
Envuelve las etapas del pipeline (que en el CLI son subcomandos) en una clase
|
|
4
|
+
`Pipeline` fácil de usar desde código:
|
|
5
|
+
|
|
6
|
+
from reconstruct3d import Pipeline
|
|
7
|
+
|
|
8
|
+
pipe = Pipeline("outputs/run1", frontend="sift", camera="camera.json")
|
|
9
|
+
pipe.extract("video.mp4", k_skip=15, m_window=4)
|
|
10
|
+
pipe.init()
|
|
11
|
+
pipe.track()
|
|
12
|
+
pipe.bundle_adjust()
|
|
13
|
+
pipe.dense("video.mp4")
|
|
14
|
+
pipe.mesh(method="afront")
|
|
15
|
+
print(pipe.artifacts())
|
|
16
|
+
|
|
17
|
+
O todo de una vez, con control de etapas y progreso:
|
|
18
|
+
|
|
19
|
+
def on_event(ev): # ev = {stage, status, message, ...}
|
|
20
|
+
print(ev["stage"], ev["status"], ev.get("message", ""))
|
|
21
|
+
|
|
22
|
+
pipe = Pipeline("outputs/run1", on_event=on_event)
|
|
23
|
+
pipe.run("video.mp4", stages=["extract", "init", "track", "dense"],
|
|
24
|
+
config={"extract": {"k_skip": 15, "m_window": 4}})
|
|
25
|
+
|
|
26
|
+
Las etapas comparten el `out_dir` y persisten su estado en disco (sfm_data.pkl,
|
|
27
|
+
map_state.npy, *.ply), igual que el CLI.
|
|
28
|
+
"""
|
|
29
|
+
import os
|
|
30
|
+
import time
|
|
31
|
+
from typing import Callable, Dict, List, Optional, Union
|
|
32
|
+
|
|
33
|
+
from reconstruct3d.core import CameraConfig
|
|
34
|
+
|
|
35
|
+
# Artefactos por etapa (nombres de archivo dentro de out_dir).
|
|
36
|
+
ARTIFACTS = {
|
|
37
|
+
"extract": "sfm_data.pkl",
|
|
38
|
+
"init": "init_cloud.ply",
|
|
39
|
+
"track": "tracked_cloud.ply",
|
|
40
|
+
"ba": "tracked_cloud.ply",
|
|
41
|
+
"dense": "dense_cloud.ply",
|
|
42
|
+
"mesh": "mesh.ply",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
ALL_STAGES = ["extract", "init", "track", "ba", "dense", "mesh"]
|
|
46
|
+
|
|
47
|
+
CameraLike = Union[None, str, dict, CameraConfig]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _to_camera(cam: CameraLike) -> Optional[CameraConfig]:
|
|
51
|
+
if cam is None or isinstance(cam, CameraConfig):
|
|
52
|
+
return cam
|
|
53
|
+
if isinstance(cam, dict):
|
|
54
|
+
return CameraConfig.from_dict(cam)
|
|
55
|
+
if isinstance(cam, str):
|
|
56
|
+
return CameraConfig.from_json(cam)
|
|
57
|
+
raise TypeError(f"camera debe ser None, ruta, dict o CameraConfig, no {type(cam)}")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class Pipeline:
|
|
61
|
+
"""Orquesta la reconstrucción 3D etapa por etapa sobre un directorio de salida.
|
|
62
|
+
|
|
63
|
+
Parámetros
|
|
64
|
+
----------
|
|
65
|
+
out_dir : str
|
|
66
|
+
Carpeta donde se leen/escriben todos los artefactos.
|
|
67
|
+
frontend : str
|
|
68
|
+
Detector+matcher: 'sift' (default), 'orb' o 'spglue'.
|
|
69
|
+
camera : None | str | dict | CameraConfig
|
|
70
|
+
Intrínsecos: ruta a JSON, dict, CameraConfig, o None (defaults).
|
|
71
|
+
jobs : int
|
|
72
|
+
Hilos para extracción/matching/densificación (0=auto, 1=secuencial).
|
|
73
|
+
on_event : callable, opcional
|
|
74
|
+
Se llama con un dict {stage, status, message, ...} al iniciar/terminar
|
|
75
|
+
cada etapa. Útil para reportar progreso (p.ej. desde un backend).
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def __init__(self, out_dir: str, frontend: str = "sift", camera: CameraLike = None,
|
|
79
|
+
jobs: int = 0, on_event: Optional[Callable[[dict], None]] = None):
|
|
80
|
+
self.out_dir = out_dir
|
|
81
|
+
self.frontend = frontend
|
|
82
|
+
self.jobs = jobs
|
|
83
|
+
self.camera = _to_camera(camera)
|
|
84
|
+
self.on_event = on_event
|
|
85
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
86
|
+
|
|
87
|
+
# ------------------------------------------------------------------ #
|
|
88
|
+
def _emit(self, stage, status, message="", **extra):
|
|
89
|
+
if self.on_event:
|
|
90
|
+
ev = {"stage": stage, "status": status, "message": message,
|
|
91
|
+
"time": time.time(), **extra}
|
|
92
|
+
try:
|
|
93
|
+
self.on_event(ev)
|
|
94
|
+
except Exception:
|
|
95
|
+
pass
|
|
96
|
+
|
|
97
|
+
def _path(self, name):
|
|
98
|
+
return os.path.join(self.out_dir, name)
|
|
99
|
+
|
|
100
|
+
# ------------------------------------------------------------------ #
|
|
101
|
+
# Etapas #
|
|
102
|
+
# ------------------------------------------------------------------ #
|
|
103
|
+
def calibrate(self, board_video, board=(9, 6), square=0.025, k_skip=10,
|
|
104
|
+
max_views=40, out=None) -> CameraConfig:
|
|
105
|
+
"""Calibra K desde un video de tablero y la fija como cámara de la pipeline."""
|
|
106
|
+
from reconstruct3d import calibrate as _cal
|
|
107
|
+
out = out or os.path.join(self.out_dir, "camera.json")
|
|
108
|
+
self._emit("calibrate", "start", f"calibrando desde {board_video}")
|
|
109
|
+
_cal.run_calibration(board_video, board=board, square_size=square,
|
|
110
|
+
k_skip=k_skip, max_views=max_views, out=out)
|
|
111
|
+
self.camera = CameraConfig.from_json(out)
|
|
112
|
+
self._emit("calibrate", "done", f"camera.json -> {out}", artifact=out)
|
|
113
|
+
return self.camera
|
|
114
|
+
|
|
115
|
+
def extract(self, video, k_skip=5, m_window=3, max_seconds=None,
|
|
116
|
+
start_seconds=0.0, force=False):
|
|
117
|
+
from reconstruct3d.core import process_core
|
|
118
|
+
self._emit("extract", "start", f"features+esenciales ({self.frontend})")
|
|
119
|
+
process_core(video, self.out_dir, frontend_name=self.frontend,
|
|
120
|
+
k_skip=k_skip, m_window=m_window, max_seconds=max_seconds,
|
|
121
|
+
start_seconds=start_seconds, camera=self.camera,
|
|
122
|
+
reuse_db=not force, jobs=self.jobs)
|
|
123
|
+
self._emit("extract", "done", artifact=self._path(ARTIFACTS["extract"]))
|
|
124
|
+
return self
|
|
125
|
+
|
|
126
|
+
def init(self):
|
|
127
|
+
from reconstruct3d.init_sfm import init_reconstruction
|
|
128
|
+
self._emit("init", "start", "triangulando par semilla")
|
|
129
|
+
init_reconstruction(self.out_dir)
|
|
130
|
+
self._emit("init", "done", artifact=self._path(ARTIFACTS["init"]))
|
|
131
|
+
return self
|
|
132
|
+
|
|
133
|
+
def track(self, m_window=3, min_angle=1.5, local_ba=True, ba_every=1, fuse=True):
|
|
134
|
+
from reconstruct3d.track_sfm import track_frames
|
|
135
|
+
self._emit("track", "start", "registro incremental + BA local")
|
|
136
|
+
track_frames(self.out_dir, m_window=m_window, min_angle=min_angle,
|
|
137
|
+
local_ba=local_ba, ba_every=ba_every, fuse=fuse)
|
|
138
|
+
self._emit("track", "done", artifact=self._path(ARTIFACTS["track"]))
|
|
139
|
+
return self
|
|
140
|
+
|
|
141
|
+
def bundle_adjust(self, max_reproj=4.0, max_nfev=300):
|
|
142
|
+
from reconstruct3d.bundle_adjust import run_bundle_adjustment
|
|
143
|
+
self._emit("ba", "start", "Bundle Adjustment global")
|
|
144
|
+
run_bundle_adjustment(self.out_dir, max_reproj_err=max_reproj, max_nfev=max_nfev)
|
|
145
|
+
self._emit("ba", "done", artifact=self._path(ARTIFACTS["ba"]))
|
|
146
|
+
return self
|
|
147
|
+
|
|
148
|
+
def dense(self, video, neighbor_steps=(1, 2), min_views=2, num_disp=128,
|
|
149
|
+
block=5, voxel=0.0):
|
|
150
|
+
from reconstruct3d.dense_mvs import run_dense
|
|
151
|
+
self._emit("dense", "start", "densificación MVS")
|
|
152
|
+
run_dense(self.out_dir, video, neighbor_steps=tuple(neighbor_steps),
|
|
153
|
+
min_views=min_views, num_disp=num_disp, block=block,
|
|
154
|
+
voxel=voxel, jobs=self.jobs)
|
|
155
|
+
self._emit("dense", "done", artifact=self._path(ARTIFACTS["dense"]))
|
|
156
|
+
return self
|
|
157
|
+
|
|
158
|
+
def mesh(self, input_ply=None, method="afront", outlier_pct=2.0, simplify=0.0,
|
|
159
|
+
smooth=0, mesh_smooth=2, min_component=0.002, afront_radius=5.0,
|
|
160
|
+
poisson_spacing_mult=1.0):
|
|
161
|
+
from reconstruct3d import mesh as _mesh
|
|
162
|
+
self._emit("mesh", "start", f"mallado CGAL ({method})")
|
|
163
|
+
_mesh.run_mesh(self.out_dir, input_ply=input_ply, method=method,
|
|
164
|
+
outlier_pct=outlier_pct, simplify=simplify, smooth=smooth,
|
|
165
|
+
mesh_smooth=mesh_smooth, min_component=min_component,
|
|
166
|
+
afront_radius=afront_radius, poisson_spacing_mult=poisson_spacing_mult)
|
|
167
|
+
self._emit("mesh", "done", artifact=self._path(ARTIFACTS["mesh"]))
|
|
168
|
+
return self
|
|
169
|
+
|
|
170
|
+
# ------------------------------------------------------------------ #
|
|
171
|
+
def run(self, video, stages: Optional[List[str]] = None,
|
|
172
|
+
config: Optional[Dict[str, dict]] = None) -> Dict[str, str]:
|
|
173
|
+
"""Ejecuta una secuencia de etapas. Devuelve {etapa: ruta_artefacto}.
|
|
174
|
+
|
|
175
|
+
stages: subconjunto/orden de ALL_STAGES (default: todas).
|
|
176
|
+
config: {etapa: {kwargs}} para parametrizar cada etapa.
|
|
177
|
+
"""
|
|
178
|
+
stages = stages or list(ALL_STAGES)
|
|
179
|
+
config = config or {}
|
|
180
|
+
for s in stages:
|
|
181
|
+
if s not in ALL_STAGES:
|
|
182
|
+
raise ValueError(f"Etapa desconocida: {s}. Opciones: {ALL_STAGES}")
|
|
183
|
+
try:
|
|
184
|
+
for s in stages:
|
|
185
|
+
kw = dict(config.get(s, {}))
|
|
186
|
+
if s == "extract":
|
|
187
|
+
self.extract(video, **kw)
|
|
188
|
+
elif s == "init":
|
|
189
|
+
self.init(**kw)
|
|
190
|
+
elif s == "track":
|
|
191
|
+
self.track(**kw)
|
|
192
|
+
elif s == "ba":
|
|
193
|
+
self.bundle_adjust(**kw)
|
|
194
|
+
elif s == "dense":
|
|
195
|
+
self.dense(video, **kw)
|
|
196
|
+
elif s == "mesh":
|
|
197
|
+
self.mesh(**kw)
|
|
198
|
+
except Exception as e: # noqa: BLE001 - reportar y propagar
|
|
199
|
+
self._emit("pipeline", "error", str(e))
|
|
200
|
+
raise
|
|
201
|
+
self._emit("pipeline", "done", "pipeline completo")
|
|
202
|
+
return self.artifacts()
|
|
203
|
+
|
|
204
|
+
def artifacts(self) -> Dict[str, str]:
|
|
205
|
+
"""Rutas de artefactos que existen en disco, por etapa."""
|
|
206
|
+
out = {}
|
|
207
|
+
for stage, name in ARTIFACTS.items():
|
|
208
|
+
p = self._path(name)
|
|
209
|
+
if os.path.exists(p):
|
|
210
|
+
out[stage] = p
|
|
211
|
+
return out
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def run_all(video, out_dir, frontend="sift", camera: CameraLike = None, jobs=0,
|
|
215
|
+
stages=None, config=None, on_event=None) -> Dict[str, str]:
|
|
216
|
+
"""Atajo: crea una Pipeline y corre las etapas indicadas."""
|
|
217
|
+
pipe = Pipeline(out_dir, frontend=frontend, camera=camera, jobs=jobs, on_event=on_event)
|
|
218
|
+
return pipe.run(video, stages=stages, config=config)
|
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""Bundle Adjustment global sobre el mapa reconstruido.
|
|
2
|
+
|
|
3
|
+
Re-optimiza TODAS las poses de cámara y TODOS los puntos 3D a la vez,
|
|
4
|
+
minimizando el error de reproyección global. Es lo que endereza la trayectoria
|
|
5
|
+
y "aprieta" la nube: el pipeline incremental (init/track) solo acumula error,
|
|
6
|
+
BA lo corrige hacia atrás.
|
|
7
|
+
|
|
8
|
+
Entrada/Salida: lee y reescribe `map_state.npy` (poses + puntos + obs) y
|
|
9
|
+
re-exporta `tracked_cloud.ply`. Las observaciones 2D se recuperan de
|
|
10
|
+
`sfm_data.pkl` vía obs[frame][kp_idx] -> features[frame].pts[kp_idx].
|
|
11
|
+
|
|
12
|
+
Proyección vectorizada (Rodrigues sin bucles) + jacobiano disperso: con ~100
|
|
13
|
+
cámaras y decenas de miles de puntos esto corre en segundos/minutos en CPU.
|
|
14
|
+
"""
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
import argparse
|
|
18
|
+
import numpy as np
|
|
19
|
+
import cv2
|
|
20
|
+
from scipy.optimize import least_squares
|
|
21
|
+
from scipy.sparse import lil_matrix
|
|
22
|
+
|
|
23
|
+
from reconstruct3d.core import (
|
|
24
|
+
SfMDatabase, Features,
|
|
25
|
+
DB_FILE, MAP_STATE_FILE, TRACKED_CLOUD_FILE,
|
|
26
|
+
)
|
|
27
|
+
from reconstruct3d.init_sfm import export_ply
|
|
28
|
+
|
|
29
|
+
# Permitir deserializar los pickles de features
|
|
30
|
+
sys.modules['__main__'].Features = Features
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def rotate(points, rot_vecs):
|
|
34
|
+
"""Rota `points` (M,3) por vectores de rotación `rot_vecs` (M,6->3) usando
|
|
35
|
+
la fórmula de Rodrigues, completamente vectorizada."""
|
|
36
|
+
theta = np.linalg.norm(rot_vecs, axis=1)[:, np.newaxis]
|
|
37
|
+
with np.errstate(invalid='ignore', divide='ignore'):
|
|
38
|
+
v = np.nan_to_num(rot_vecs / theta)
|
|
39
|
+
dot = np.sum(points * v, axis=1)[:, np.newaxis]
|
|
40
|
+
cos_t, sin_t = np.cos(theta), np.sin(theta)
|
|
41
|
+
return cos_t * points + sin_t * np.cross(v, points) + dot * (1 - cos_t) * v
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def project(points, cam_params, K):
|
|
45
|
+
"""Proyecta `points` (M,3, mundo) con `cam_params` (M,6 = [rvec|tvec]) a
|
|
46
|
+
píxeles (M,2). Una fila por observación."""
|
|
47
|
+
pc = rotate(points, cam_params[:, :3]) + cam_params[:, 3:6]
|
|
48
|
+
xy = pc[:, :2] / pc[:, 2:3]
|
|
49
|
+
fx, fy, cx, cy = K[0, 0], K[1, 1], K[0, 2], K[1, 2]
|
|
50
|
+
u = fx * xy[:, 0] + cx
|
|
51
|
+
v = fy * xy[:, 1] + cy
|
|
52
|
+
return np.stack([u, v], axis=1)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _residuals(params, n_cam, n_pts, cam_idx, pt_idx, pts2d, K):
|
|
56
|
+
cams = params[:n_cam * 6].reshape(n_cam, 6)
|
|
57
|
+
pts = params[n_cam * 6:].reshape(n_pts, 3)
|
|
58
|
+
proj = project(pts[pt_idx], cams[cam_idx], K)
|
|
59
|
+
return (proj - pts2d).ravel()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _jac_sparsity(n_cam, n_pts, cam_idx, pt_idx):
|
|
63
|
+
"""Patrón de esparsidad: cada residuo (u,v) solo depende de su cámara (6
|
|
64
|
+
params) y su punto (3 params). Imprescindible para que BA sea tratable."""
|
|
65
|
+
m = cam_idx.size * 2
|
|
66
|
+
n = n_cam * 6 + n_pts * 3
|
|
67
|
+
A = lil_matrix((m, n), dtype=int)
|
|
68
|
+
i = np.arange(cam_idx.size)
|
|
69
|
+
for s in range(6):
|
|
70
|
+
A[2 * i, cam_idx * 6 + s] = 1
|
|
71
|
+
A[2 * i + 1, cam_idx * 6 + s] = 1
|
|
72
|
+
for s in range(3):
|
|
73
|
+
A[2 * i, n_cam * 6 + pt_idx * 3 + s] = 1
|
|
74
|
+
A[2 * i + 1, n_cam * 6 + pt_idx * 3 + s] = 1
|
|
75
|
+
return A
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _pose_to_rt(P):
|
|
79
|
+
"""4x4 world->cam -> vector 6 [rvec|tvec]."""
|
|
80
|
+
rv, _ = cv2.Rodrigues(P[:3, :3])
|
|
81
|
+
return np.concatenate([rv.ravel(), P[:3, 3]])
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _rt_to_pose(rt):
|
|
85
|
+
R, _ = cv2.Rodrigues(rt[:3])
|
|
86
|
+
return np.vstack((np.hstack((R, rt[3:].reshape(3, 1))), [0, 0, 0, 1]))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def run_local_ba(map3d, pts_of, K, window_frames, max_nfev=25):
|
|
90
|
+
"""Bundle Adjustment LOCAL por ventana deslizante.
|
|
91
|
+
|
|
92
|
+
Optimiza conjuntamente las poses de `window_frames` y los puntos 3D que
|
|
93
|
+
observan (>=2 vistas en la ventana), manteniendo FIJAS: (a) la cámara ancla
|
|
94
|
+
—la más antigua de la ventana, para fijar el gauge— y (b) las cámaras
|
|
95
|
+
externas que también ven esos puntos (aportan restricciones sin moverse).
|
|
96
|
+
|
|
97
|
+
Esto corrige el error de cada pose nueva + sus puntos recién triangulados
|
|
98
|
+
JUNTOS antes de que se propaguen al siguiente frame (ataca el arrastre y el
|
|
99
|
+
lazo de realimentación). Modifica map3d['poses'] y map3d['points'] in-place.
|
|
100
|
+
|
|
101
|
+
pts_of: callable frame -> ndarray (N,2) de keypoints.
|
|
102
|
+
"""
|
|
103
|
+
window = sorted(window_frames)
|
|
104
|
+
if len(window) < 3:
|
|
105
|
+
return
|
|
106
|
+
win_set = set(window)
|
|
107
|
+
anchor = window[0]
|
|
108
|
+
var_frames = window[1:]
|
|
109
|
+
var_pos = {f: i for i, f in enumerate(var_frames)}
|
|
110
|
+
n_var = len(var_frames)
|
|
111
|
+
|
|
112
|
+
# Puntos observados por >=2 cámaras de la ventana (constreñidos localmente)
|
|
113
|
+
from collections import Counter
|
|
114
|
+
cnt = Counter()
|
|
115
|
+
for f in window:
|
|
116
|
+
for pid in map3d['obs'][f].values():
|
|
117
|
+
cnt[pid] += 1
|
|
118
|
+
pts3d = map3d['points']
|
|
119
|
+
local_pids = [p for p, c in cnt.items() if c >= 2 and np.isfinite(pts3d[p]).all()]
|
|
120
|
+
if len(local_pids) < 20:
|
|
121
|
+
return
|
|
122
|
+
pid_pos = {p: i for i, p in enumerate(local_pids)}
|
|
123
|
+
local_set = set(local_pids)
|
|
124
|
+
n_pts = len(local_pids)
|
|
125
|
+
|
|
126
|
+
# Cámaras externas (fuera de la ventana) que ven puntos locales -> fijas
|
|
127
|
+
ext_fixed = [f for f, d in map3d['obs'].items()
|
|
128
|
+
if f not in win_set and any(pid in local_set for pid in d.values())]
|
|
129
|
+
|
|
130
|
+
fixed_rt = {f: _pose_to_rt(map3d['poses'][f]) for f in [anchor] + ext_fixed}
|
|
131
|
+
|
|
132
|
+
# Separar observaciones: cámara variable vs cámara fija
|
|
133
|
+
vc_cam, vc_pt, vc_2d = [], [], []
|
|
134
|
+
fc_pose, fc_pt, fc_2d = [], [], []
|
|
135
|
+
for f in win_set | set(ext_fixed):
|
|
136
|
+
pts = pts_of(f)
|
|
137
|
+
for kp, pid in map3d['obs'][f].items():
|
|
138
|
+
if pid not in local_set:
|
|
139
|
+
continue
|
|
140
|
+
uv = pts[kp]
|
|
141
|
+
if f in var_pos:
|
|
142
|
+
vc_cam.append(var_pos[f]); vc_pt.append(pid_pos[pid]); vc_2d.append(uv)
|
|
143
|
+
else:
|
|
144
|
+
fc_pose.append(fixed_rt[f]); fc_pt.append(pid_pos[pid]); fc_2d.append(uv)
|
|
145
|
+
|
|
146
|
+
vc_cam = np.asarray(vc_cam, int); vc_pt = np.asarray(vc_pt, int)
|
|
147
|
+
vc_2d = np.asarray(vc_2d, float).reshape(-1, 2)
|
|
148
|
+
fc_pose = np.asarray(fc_pose, float).reshape(-1, 6)
|
|
149
|
+
fc_pt = np.asarray(fc_pt, int); fc_2d = np.asarray(fc_2d, float).reshape(-1, 2)
|
|
150
|
+
n_vc, n_fc = len(vc_2d), len(fc_2d)
|
|
151
|
+
if n_vc == 0:
|
|
152
|
+
return
|
|
153
|
+
|
|
154
|
+
cam0 = np.array([_pose_to_rt(map3d['poses'][f]) for f in var_frames])
|
|
155
|
+
pts0 = pts3d[local_pids].astype(float)
|
|
156
|
+
x0 = np.concatenate([cam0.ravel(), pts0.ravel()])
|
|
157
|
+
|
|
158
|
+
def resid(x):
|
|
159
|
+
cams = x[:n_var * 6].reshape(n_var, 6)
|
|
160
|
+
pts = x[n_var * 6:].reshape(n_pts, 3)
|
|
161
|
+
out = [(project(pts[vc_pt], cams[vc_cam], K) - vc_2d).ravel()]
|
|
162
|
+
if n_fc:
|
|
163
|
+
out.append((project(pts[fc_pt], fc_pose, K) - fc_2d).ravel())
|
|
164
|
+
return np.concatenate(out)
|
|
165
|
+
|
|
166
|
+
# Esparsidad: obs de cámara variable dependen de su cámara (6) + punto (3);
|
|
167
|
+
# obs de cámara fija dependen solo del punto (3).
|
|
168
|
+
m = (n_vc + n_fc) * 2
|
|
169
|
+
n = n_var * 6 + n_pts * 3
|
|
170
|
+
A = lil_matrix((m, n), dtype=int)
|
|
171
|
+
iv = np.arange(n_vc)
|
|
172
|
+
for s in range(6):
|
|
173
|
+
A[2 * iv, vc_cam * 6 + s] = 1
|
|
174
|
+
A[2 * iv + 1, vc_cam * 6 + s] = 1
|
|
175
|
+
for s in range(3):
|
|
176
|
+
A[2 * iv, n_var * 6 + vc_pt * 3 + s] = 1
|
|
177
|
+
A[2 * iv + 1, n_var * 6 + vc_pt * 3 + s] = 1
|
|
178
|
+
if n_fc:
|
|
179
|
+
jf = np.arange(n_fc) + n_vc
|
|
180
|
+
for s in range(3):
|
|
181
|
+
A[2 * jf, n_var * 6 + fc_pt * 3 + s] = 1
|
|
182
|
+
A[2 * jf + 1, n_var * 6 + fc_pt * 3 + s] = 1
|
|
183
|
+
|
|
184
|
+
res = least_squares(resid, x0, jac_sparsity=A, method='trf', loss='huber',
|
|
185
|
+
f_scale=2.0, x_scale='jac', max_nfev=max_nfev, verbose=0)
|
|
186
|
+
|
|
187
|
+
cam_opt = res.x[:n_var * 6].reshape(n_var, 6)
|
|
188
|
+
pts_opt = res.x[n_var * 6:].reshape(n_pts, 3)
|
|
189
|
+
for f in var_frames:
|
|
190
|
+
map3d['poses'][f] = _rt_to_pose(cam_opt[var_pos[f]])
|
|
191
|
+
for p, i in pid_pos.items():
|
|
192
|
+
map3d['points'][p] = pts_opt[i]
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def fuse_map_points(map3d, voxel):
|
|
196
|
+
"""Fusiona puntos 3D duplicados (mismo punto físico triangulado por separado).
|
|
197
|
+
|
|
198
|
+
Agrupa por vóxel y fusiona miembros cuyos conjuntos de frames observadores
|
|
199
|
+
son DISJUNTOS (tracks separados del mismo punto), sin que ningún frame quede
|
|
200
|
+
observando el punto fusionado dos veces. Redirige las observaciones al punto
|
|
201
|
+
superviviente y marca el duplicado como NaN (lo filtra el export).
|
|
202
|
+
|
|
203
|
+
Conservador a propósito: no fusiona puntos co-visibles (distintos puntos
|
|
204
|
+
cercanos en una pared densa), solo tracks separados.
|
|
205
|
+
"""
|
|
206
|
+
from collections import defaultdict
|
|
207
|
+
pts = map3d['points']
|
|
208
|
+
finite = np.where(np.isfinite(pts).all(axis=1))[0]
|
|
209
|
+
if len(finite) == 0:
|
|
210
|
+
return 0
|
|
211
|
+
|
|
212
|
+
rev = defaultdict(list) # pid -> [(frame, kp)]
|
|
213
|
+
for f, d in map3d['obs'].items():
|
|
214
|
+
for kp, pid in d.items():
|
|
215
|
+
rev[pid].append((f, kp))
|
|
216
|
+
|
|
217
|
+
keys = np.floor(pts[finite] / voxel).astype(np.int64)
|
|
218
|
+
groups = defaultdict(list)
|
|
219
|
+
for pid, key in zip(finite, map(tuple, keys)):
|
|
220
|
+
groups[key].append(pid)
|
|
221
|
+
|
|
222
|
+
fused = 0
|
|
223
|
+
for members in groups.values():
|
|
224
|
+
if len(members) < 2:
|
|
225
|
+
continue
|
|
226
|
+
keep = members[0]
|
|
227
|
+
keep_frames = {f for f, _ in rev[keep]}
|
|
228
|
+
coords = [pts[keep]]
|
|
229
|
+
for dup in members[1:]:
|
|
230
|
+
dup_frames = {f for f, _ in rev[dup]}
|
|
231
|
+
if keep_frames & dup_frames: # comparten frame -> no fusionar
|
|
232
|
+
continue
|
|
233
|
+
for (f, kp) in rev[dup]:
|
|
234
|
+
map3d['obs'][f][kp] = keep
|
|
235
|
+
coords.append(pts[dup])
|
|
236
|
+
pts[dup] = np.nan
|
|
237
|
+
keep_frames |= dup_frames
|
|
238
|
+
fused += 1
|
|
239
|
+
map3d['points'][keep] = np.mean(coords, axis=0)
|
|
240
|
+
return fused
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _build_problem(db, map3d):
|
|
244
|
+
"""Empaqueta poses (rvec|tvec), puntos y observaciones (cam_idx, pt_idx, pts2d)."""
|
|
245
|
+
frames = sorted(map3d['poses'].keys())
|
|
246
|
+
fpos = {f: i for i, f in enumerate(frames)}
|
|
247
|
+
n_cam = len(frames)
|
|
248
|
+
|
|
249
|
+
cam_params = np.zeros((n_cam, 6), dtype=np.float64)
|
|
250
|
+
for f in frames:
|
|
251
|
+
P = map3d['poses'][f]
|
|
252
|
+
rvec, _ = cv2.Rodrigues(P[:3, :3])
|
|
253
|
+
cam_params[fpos[f], :3] = rvec.ravel()
|
|
254
|
+
cam_params[fpos[f], 3:] = P[:3, 3]
|
|
255
|
+
|
|
256
|
+
points = np.asarray(map3d['points'], dtype=np.float64).copy()
|
|
257
|
+
|
|
258
|
+
cam_idx, pt_idx, pts2d = [], [], []
|
|
259
|
+
for f in frames:
|
|
260
|
+
feats = db.features[f]
|
|
261
|
+
for kp, pid in map3d['obs'][f].items():
|
|
262
|
+
cam_idx.append(fpos[f]); pt_idx.append(pid); pts2d.append(feats.pts[kp])
|
|
263
|
+
cam_idx = np.asarray(cam_idx, dtype=int)
|
|
264
|
+
pt_idx = np.asarray(pt_idx, dtype=int)
|
|
265
|
+
pts2d = np.asarray(pts2d, dtype=np.float64)
|
|
266
|
+
return frames, fpos, cam_params, points, cam_idx, pt_idx, pts2d
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _compact_finite_points(map3d):
|
|
270
|
+
"""Elimina puntos NaN/inf (p.ej. duplicados marcados por fuse_map_points) y
|
|
271
|
+
re-indexa las observaciones. least_squares no tolera NaN en el vector x0."""
|
|
272
|
+
pts = np.asarray(map3d['points'])
|
|
273
|
+
keep = np.isfinite(pts).all(axis=1)
|
|
274
|
+
if keep.all():
|
|
275
|
+
return 0
|
|
276
|
+
old2new = -np.ones(len(pts), dtype=int)
|
|
277
|
+
old2new[keep] = np.arange(int(keep.sum()))
|
|
278
|
+
map3d['points'] = pts[keep]
|
|
279
|
+
map3d['colors'] = np.asarray(map3d['colors'])[keep]
|
|
280
|
+
for f in list(map3d['obs'].keys()):
|
|
281
|
+
map3d['obs'][f] = {kp: int(old2new[pid]) for kp, pid in map3d['obs'][f].items()
|
|
282
|
+
if 0 <= pid < len(pts) and keep[pid]}
|
|
283
|
+
return int((~keep).sum())
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def run_bundle_adjustment(out_dir, max_reproj_err=4.0, ftol=1e-8, max_nfev=300):
|
|
287
|
+
db = SfMDatabase(); db.load(os.path.join(out_dir, DB_FILE))
|
|
288
|
+
K = db.get_camera().K # intrínsecos persistidos en la DB
|
|
289
|
+
map_path = os.path.join(out_dir, MAP_STATE_FILE)
|
|
290
|
+
map3d = np.load(map_path, allow_pickle=True).item()
|
|
291
|
+
|
|
292
|
+
n_drop = _compact_finite_points(map3d)
|
|
293
|
+
if n_drop:
|
|
294
|
+
print(f"[BA] saneado: {n_drop} puntos NaN/inf eliminados antes de optimizar")
|
|
295
|
+
|
|
296
|
+
frames, fpos, cam_params, points, cam_idx, pt_idx, pts2d = _build_problem(db, map3d)
|
|
297
|
+
n_cam, n_pts = len(frames), len(points)
|
|
298
|
+
|
|
299
|
+
x0 = np.hstack([cam_params.ravel(), points.ravel()])
|
|
300
|
+
r0 = _residuals(x0, n_cam, n_pts, cam_idx, pt_idx, pts2d, K)
|
|
301
|
+
rmse0 = np.sqrt(np.mean(r0 ** 2))
|
|
302
|
+
print(f"[BA] cámaras={n_cam} puntos={n_pts} observaciones={len(pts2d)}")
|
|
303
|
+
print(f"[BA] RMSE reproyección inicial: {rmse0:7.3f} px")
|
|
304
|
+
|
|
305
|
+
A = _jac_sparsity(n_cam, n_pts, cam_idx, pt_idx)
|
|
306
|
+
res = least_squares(
|
|
307
|
+
_residuals, x0, jac_sparsity=A, x_scale='jac', method='trf',
|
|
308
|
+
loss='huber', f_scale=2.0, ftol=ftol, xtol=1e-10, gtol=1e-12,
|
|
309
|
+
max_nfev=max_nfev, verbose=2, args=(n_cam, n_pts, cam_idx, pt_idx, pts2d, K),
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
rf = _residuals(res.x, n_cam, n_pts, cam_idx, pt_idx, pts2d, K)
|
|
313
|
+
rmse1 = np.sqrt(np.mean(rf ** 2))
|
|
314
|
+
print(f"[BA] RMSE reproyección final: {rmse1:7.3f} px (mejora {rmse0 - rmse1:+.3f})")
|
|
315
|
+
|
|
316
|
+
cam_opt = res.x[:n_cam * 6].reshape(n_cam, 6)
|
|
317
|
+
pts_opt = res.x[n_cam * 6:].reshape(n_pts, 3)
|
|
318
|
+
|
|
319
|
+
# Poses optimizadas de vuelta a 4x4 world->cam
|
|
320
|
+
for f in frames:
|
|
321
|
+
rvec = cam_opt[fpos[f], :3]
|
|
322
|
+
tvec = cam_opt[fpos[f], 3:]
|
|
323
|
+
R, _ = cv2.Rodrigues(rvec)
|
|
324
|
+
map3d['poses'][f] = np.vstack((np.hstack((R, tvec.reshape(3, 1))), [0, 0, 0, 1]))
|
|
325
|
+
|
|
326
|
+
# Filtrar puntos con error de reproyección alto tras la optimización
|
|
327
|
+
per_obs = np.sqrt((rf.reshape(-1, 2) ** 2).sum(axis=1))
|
|
328
|
+
err_sum = np.zeros(n_pts); err_cnt = np.zeros(n_pts)
|
|
329
|
+
np.add.at(err_sum, pt_idx, per_obs)
|
|
330
|
+
np.add.at(err_cnt, pt_idx, 1)
|
|
331
|
+
mean_err = np.divide(err_sum, np.maximum(err_cnt, 1))
|
|
332
|
+
keep = (err_cnt > 0) & (mean_err <= max_reproj_err) & np.isfinite(pts_opt).all(axis=1)
|
|
333
|
+
print(f"[BA] puntos conservados: {int(keep.sum())}/{n_pts} "
|
|
334
|
+
f"(filtrados {n_pts - int(keep.sum())} con reproj > {max_reproj_err}px)")
|
|
335
|
+
|
|
336
|
+
# Re-indexar puntos y obs
|
|
337
|
+
old2new = -np.ones(n_pts, dtype=int)
|
|
338
|
+
old2new[keep] = np.arange(int(keep.sum()))
|
|
339
|
+
map3d['points'] = pts_opt[keep]
|
|
340
|
+
map3d['colors'] = np.asarray(map3d['colors'])[keep]
|
|
341
|
+
new_obs = {}
|
|
342
|
+
for f in frames:
|
|
343
|
+
d = {}
|
|
344
|
+
for kp, pid in map3d['obs'][f].items():
|
|
345
|
+
if 0 <= pid < n_pts and keep[pid]:
|
|
346
|
+
d[kp] = int(old2new[pid])
|
|
347
|
+
new_obs[f] = d
|
|
348
|
+
map3d['obs'] = new_obs
|
|
349
|
+
|
|
350
|
+
np.save(map_path, map3d)
|
|
351
|
+
|
|
352
|
+
# Re-exportar la nube centrada (mismo criterio que track_sfm)
|
|
353
|
+
pts = map3d['points']
|
|
354
|
+
valid = np.isfinite(pts).all(axis=1)
|
|
355
|
+
median_pt = np.median(pts[valid], axis=0) if valid.any() else np.zeros(3)
|
|
356
|
+
centered = pts[valid] - median_pt
|
|
357
|
+
cam_centers = [-(P[:3, :3].T @ P[:3, 3:]).ravel() - median_pt for P in map3d['poses'].values()]
|
|
358
|
+
export_ply(os.path.join(out_dir, TRACKED_CLOUD_FILE), centered, map3d['colors'][valid], cam_centers)
|
|
359
|
+
print(f"[BA] map_state.npy + {TRACKED_CLOUD_FILE} actualizados.")
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
if __name__ == "__main__":
|
|
363
|
+
parser = argparse.ArgumentParser(description="Global Bundle Adjustment sobre el mapa SfM")
|
|
364
|
+
parser.add_argument("--out", required=True, help="Output dir con sfm_data.pkl y map_state.npy.")
|
|
365
|
+
parser.add_argument("--max-reproj", type=float, default=4.0,
|
|
366
|
+
help="Descartar puntos con error medio de reproyección > este valor (px, default 4).")
|
|
367
|
+
parser.add_argument("--max-nfev", type=int, default=300,
|
|
368
|
+
help="Máx. evaluaciones del optimizador (default 300).")
|
|
369
|
+
args = parser.parse_args()
|
|
370
|
+
run_bundle_adjustment(args.out, max_reproj_err=args.max_reproj, max_nfev=args.max_nfev)
|