texture3d 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- texture3d/__init__.py +31 -0
- texture3d/bake.py +159 -0
- texture3d/blur.py +54 -0
- texture3d/camera.py +108 -0
- texture3d/cli.py +74 -0
- texture3d/config.py +57 -0
- texture3d/io.py +52 -0
- texture3d/keyframes.py +42 -0
- texture3d/mapping/__init__.py +31 -0
- texture3d/mapping/atlas.py +246 -0
- texture3d/mapping/imageops.py +80 -0
- texture3d/mapping/labeling.py +298 -0
- texture3d/mapping/leveling.py +548 -0
- texture3d/mapping/mapper.py +114 -0
- texture3d/mapping/patches.py +287 -0
- texture3d/mapping/raster.py +185 -0
- texture3d/mapping/scoring.py +229 -0
- texture3d/mapping/topology.py +99 -0
- texture3d/mapping/types.py +199 -0
- texture3d-0.2.0.dist-info/METADATA +362 -0
- texture3d-0.2.0.dist-info/RECORD +24 -0
- texture3d-0.2.0.dist-info/WHEEL +5 -0
- texture3d-0.2.0.dist-info/entry_points.txt +2 -0
- texture3d-0.2.0.dist-info/top_level.txt +1 -0
texture3d/__init__.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""texture3d: texture a triangle mesh from posed RGB-D frames.
|
|
2
|
+
|
|
3
|
+
Each face is textured from one keyframe. Patches of same-keyframe faces are
|
|
4
|
+
cut from that image, seams between patches are levelled, and the patches
|
|
5
|
+
are packed into a single atlas (see `texture3d.mapping`). Built on trimesh.
|
|
6
|
+
|
|
7
|
+
import texture3d
|
|
8
|
+
|
|
9
|
+
frames = texture3d.load_dataset("apt0_seq0") # or build RGBDFrames yourself
|
|
10
|
+
result = texture3d.bake("mesh.ply", frames)
|
|
11
|
+
result.export("textured.obj") # or .glb
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from texture3d.bake import BakeResult, bake
|
|
15
|
+
from texture3d.camera import PinholeCamera, RGBDFrame
|
|
16
|
+
from texture3d.config import BakeConfig, KeyframeConfig, MappingConfig
|
|
17
|
+
from texture3d.io import load_dataset
|
|
18
|
+
|
|
19
|
+
__version__ = "0.2.0"
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"bake",
|
|
23
|
+
"BakeResult",
|
|
24
|
+
"BakeConfig",
|
|
25
|
+
"KeyframeConfig",
|
|
26
|
+
"MappingConfig",
|
|
27
|
+
"PinholeCamera",
|
|
28
|
+
"RGBDFrame",
|
|
29
|
+
"load_dataset",
|
|
30
|
+
"__version__",
|
|
31
|
+
]
|
texture3d/bake.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"""End-to-end texture baking: mesh + posed RGB-D frames -> textured trimesh.
|
|
2
|
+
|
|
3
|
+
keyframes -> views -> texture mapping (score, label, patch, level, pack)
|
|
4
|
+
-> textured trimesh
|
|
5
|
+
|
|
6
|
+
The mapping (`texture3d.mapping`) textures each face from one keyframe, cuts
|
|
7
|
+
patches of same-keyframe faces out of that keyframe, levels the seams and
|
|
8
|
+
packs the patches into one atlas.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import time
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Iterable, List, Optional, Union
|
|
17
|
+
|
|
18
|
+
import cv2
|
|
19
|
+
import numpy as np
|
|
20
|
+
import trimesh
|
|
21
|
+
|
|
22
|
+
from texture3d import mapping
|
|
23
|
+
from texture3d.camera import RGBDFrame
|
|
24
|
+
from texture3d.config import BakeConfig
|
|
25
|
+
from texture3d.keyframes import select_keyframes
|
|
26
|
+
|
|
27
|
+
MeshLike = Union[trimesh.Trimesh, str, Path]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class BakeResult:
|
|
32
|
+
mesh: trimesh.Trimesh # textured mesh (TextureVisuals with baked image)
|
|
33
|
+
texture: np.ndarray # (S, S, 3) uint8 RGB
|
|
34
|
+
stats: dict
|
|
35
|
+
|
|
36
|
+
def export(self, path: Union[str, Path]) -> Path:
|
|
37
|
+
"""Write the textured mesh (format from extension: .obj, .glb, ...)."""
|
|
38
|
+
path = Path(path)
|
|
39
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
40
|
+
self.mesh.export(path)
|
|
41
|
+
return path
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _load_mesh(mesh: MeshLike) -> trimesh.Trimesh:
|
|
45
|
+
if isinstance(mesh, trimesh.Trimesh):
|
|
46
|
+
return mesh
|
|
47
|
+
# process=False preserves vertex/face order and never merges vertices.
|
|
48
|
+
loaded = trimesh.load(mesh, process=False, force="mesh")
|
|
49
|
+
if not isinstance(loaded, trimesh.Trimesh) or len(loaded.faces) == 0:
|
|
50
|
+
raise ValueError(f"Not a triangle mesh: {mesh}")
|
|
51
|
+
return loaded
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _material(texture: np.ndarray):
|
|
55
|
+
from PIL import Image
|
|
56
|
+
|
|
57
|
+
# Ka/Kd white + Ks black: viewers multiply the texture by Kd, and
|
|
58
|
+
# trimesh's default 0.4 grey would darken everything.
|
|
59
|
+
return trimesh.visual.material.SimpleMaterial(
|
|
60
|
+
image=Image.fromarray(texture),
|
|
61
|
+
name="texture3d",
|
|
62
|
+
ambient=[255, 255, 255, 255],
|
|
63
|
+
diffuse=[255, 255, 255, 255],
|
|
64
|
+
specular=[0, 0, 0, 255],
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _views(keyframes: List[RGBDFrame]) -> List[mapping.View]:
|
|
69
|
+
"""The mapping's views: BGR pixels and a camera with R world-to-camera and
|
|
70
|
+
C the centre in world coordinates, from each frame's camera-to-world pose."""
|
|
71
|
+
views = []
|
|
72
|
+
for frame in keyframes:
|
|
73
|
+
w2c = frame.pose_w2c
|
|
74
|
+
cam = frame.camera
|
|
75
|
+
camera = mapping.Camera(
|
|
76
|
+
fx=cam.fx, fy=cam.fy, cx=cam.cx, cy=cam.cy, R=w2c[:3, :3].copy(), C=frame.pose_c2w[:3, 3].copy()
|
|
77
|
+
)
|
|
78
|
+
views.append(mapping.View(bgr=cv2.cvtColor(frame.load_rgb(), cv2.COLOR_RGB2BGR), camera=camera))
|
|
79
|
+
return views
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _output_mesh(mesh: trimesh.Trimesh, result: mapping.Result, texture: np.ndarray) -> trimesh.Trimesh:
|
|
83
|
+
"""The textured result as a trimesh with per-vertex UVs. A vertex has one
|
|
84
|
+
texture coordinate per patch it belongs to (its projection into that
|
|
85
|
+
patch's view), so vertices are split along patch borders."""
|
|
86
|
+
faces = np.asarray(mesh.faces)
|
|
87
|
+
vertices = np.asarray(mesh.vertices, dtype=np.float64)
|
|
88
|
+
size = result.atlas_size
|
|
89
|
+
corner_vertex = faces.ravel()
|
|
90
|
+
corner_patch = np.repeat(result.face_patch, 3)
|
|
91
|
+
key = corner_vertex * (result.patch_count + 1) + corner_patch
|
|
92
|
+
unique_key, new_index = np.unique(key, return_inverse=True)
|
|
93
|
+
new_vertices = vertices[unique_key // (result.patch_count + 1)]
|
|
94
|
+
# Pixel centres to OBJ UVs: u right, v up from the bottom-left.
|
|
95
|
+
tc = result.face_texcoords.reshape(-1, 2).astype(np.float64)
|
|
96
|
+
uv = np.empty((len(unique_key), 2), dtype=np.float64)
|
|
97
|
+
uv[new_index] = np.stack([(tc[:, 0] + 0.5) / size, 1.0 - (tc[:, 1] + 0.5) / size], axis=-1)
|
|
98
|
+
visual = trimesh.visual.TextureVisuals(uv=uv, material=_material(texture))
|
|
99
|
+
return trimesh.Trimesh(vertices=new_vertices, faces=new_index.reshape(-1, 3), visual=visual, process=False)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def bake(
|
|
103
|
+
mesh: MeshLike,
|
|
104
|
+
frames: Iterable[RGBDFrame],
|
|
105
|
+
config: Optional[BakeConfig] = None,
|
|
106
|
+
verbose: bool = True,
|
|
107
|
+
) -> BakeResult:
|
|
108
|
+
"""Texture `mesh` from posed RGB-D `frames`: every face from one keyframe,
|
|
109
|
+
seams levelled. Any UVs the mesh carries are ignored, since the mapping's
|
|
110
|
+
UVs follow the labelling. Returns a `BakeResult` whose `.mesh` is a
|
|
111
|
+
textured `trimesh.Trimesh` ready to `.export()`."""
|
|
112
|
+
cfg = config or BakeConfig()
|
|
113
|
+
t0 = time.time()
|
|
114
|
+
mesh = _load_mesh(mesh)
|
|
115
|
+
|
|
116
|
+
def log(msg: str) -> None:
|
|
117
|
+
if verbose:
|
|
118
|
+
print(msg)
|
|
119
|
+
|
|
120
|
+
vertices = np.asarray(mesh.vertices, dtype=np.float32)
|
|
121
|
+
faces = np.asarray(mesh.faces)
|
|
122
|
+
log(f"mesh: {len(vertices)} verts, {len(faces)} faces")
|
|
123
|
+
frames = list(frames)
|
|
124
|
+
keyframes = select_keyframes(frames, cfg.keyframes)
|
|
125
|
+
log(f"keyframes: {len(keyframes)} / {len(frames)} frames")
|
|
126
|
+
views = _views(keyframes)
|
|
127
|
+
m = cfg.mapping
|
|
128
|
+
empty = m.empty_color
|
|
129
|
+
tuning = mapping.Tuning(
|
|
130
|
+
outlier_threshold=m.outlier_threshold,
|
|
131
|
+
data_smoothness_ratio=m.data_smoothness_ratio,
|
|
132
|
+
global_seam_leveling=m.global_seam_leveling,
|
|
133
|
+
local_seam_leveling=m.local_seam_leveling,
|
|
134
|
+
empty_color_bgr=(int(empty[2]), int(empty[1]), int(empty[0])),
|
|
135
|
+
sharpness_weight=m.sharpness_weight,
|
|
136
|
+
max_texture_size=m.max_texture_size,
|
|
137
|
+
local_leveling_strip_width=m.local_leveling_strip_width,
|
|
138
|
+
scoring_workers=m.scoring_workers,
|
|
139
|
+
exact_packing_limit=m.exact_packing_limit,
|
|
140
|
+
fill_unseen=m.fill_unseen,
|
|
141
|
+
)
|
|
142
|
+
log("mapping: scoring views, labelling faces, levelling seams, packing...")
|
|
143
|
+
result = mapping.run(mapping.Mesh(positions=vertices, faces=faces), views, tuning, log)
|
|
144
|
+
texture = cv2.cvtColor(result.atlas_bgr, cv2.COLOR_BGR2RGB)
|
|
145
|
+
out_mesh = _output_mesh(mesh, result, texture)
|
|
146
|
+
textured = int((result.labels != mapping.types.NO_ID).sum())
|
|
147
|
+
stats = {
|
|
148
|
+
"num_frames": len(frames),
|
|
149
|
+
"num_keyframes": len(keyframes),
|
|
150
|
+
"num_patches": result.patch_count,
|
|
151
|
+
"texture_size": result.atlas_size,
|
|
152
|
+
"num_faces": int(len(faces)),
|
|
153
|
+
"textured_faces": textured,
|
|
154
|
+
"textured_fraction": round(textured / max(1, len(faces)), 4),
|
|
155
|
+
"output_vertices": int(len(out_mesh.vertices)),
|
|
156
|
+
"stage_seconds": result.timings,
|
|
157
|
+
"elapsed_seconds": round(time.time() - t0, 1),
|
|
158
|
+
}
|
|
159
|
+
return BakeResult(mesh=out_mesh, texture=texture, stats=stats)
|
texture3d/blur.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Image blurriness estimation.
|
|
2
|
+
|
|
3
|
+
Crete et al., "The blur effect: perception and estimation with a new
|
|
4
|
+
no-reference perceptual blur metric" (SPIE 2007). The image is blurred with
|
|
5
|
+
a 1x9 box kernel horizontally and vertically; the loss of neighboring-pixel
|
|
6
|
+
variation between the original and the blurred image measures how sharp the
|
|
7
|
+
original was. Returns a value in [0, 1] where higher means blurrier.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import cv2
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _abs_diff_vertical(img: np.ndarray) -> np.ndarray:
|
|
17
|
+
"""|I(y, x) - I(y, x-1)| with a zero first column."""
|
|
18
|
+
out = np.zeros_like(img)
|
|
19
|
+
out[:, 1:] = np.abs(img[:, 1:] - img[:, :-1])
|
|
20
|
+
return out
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _abs_diff_horizontal(img: np.ndarray) -> np.ndarray:
|
|
24
|
+
"""|I(y, x) - I(y-1, x)| with a zero first row."""
|
|
25
|
+
out = np.zeros_like(img)
|
|
26
|
+
out[1:, :] = np.abs(img[1:, :] - img[:-1, :])
|
|
27
|
+
return out
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def estimate_blurriness(rgb: np.ndarray) -> float:
|
|
31
|
+
"""Blurriness of an HxWx3 RGB uint8 image."""
|
|
32
|
+
gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY).astype(np.float32)
|
|
33
|
+
|
|
34
|
+
kernel = np.full((1, 9), 1.0 / 9, dtype=np.float32)
|
|
35
|
+
b_ver = cv2.filter2D(gray, -1, kernel) # blur along rows
|
|
36
|
+
b_hor = cv2.filter2D(gray, -1, kernel.T) # blur along columns
|
|
37
|
+
|
|
38
|
+
d_f_ver = _abs_diff_vertical(gray)
|
|
39
|
+
d_f_hor = _abs_diff_horizontal(gray)
|
|
40
|
+
d_b_ver = _abs_diff_vertical(b_ver)
|
|
41
|
+
d_b_hor = _abs_diff_horizontal(b_hor)
|
|
42
|
+
|
|
43
|
+
v_ver = np.maximum(d_f_ver - d_b_ver, 0.0)
|
|
44
|
+
v_hor = np.maximum(d_f_hor - d_b_hor, 0.0)
|
|
45
|
+
|
|
46
|
+
# Sums exclude the first row and column.
|
|
47
|
+
s_f_ver = float(d_f_ver[1:, 1:].sum())
|
|
48
|
+
s_f_hor = float(d_f_hor[1:, 1:].sum())
|
|
49
|
+
s_v_ver = float(v_ver[1:, 1:].sum())
|
|
50
|
+
s_v_hor = float(v_hor[1:, 1:].sum())
|
|
51
|
+
|
|
52
|
+
b_f_ver = (s_f_ver - s_v_ver) / s_f_ver if s_f_ver > 0 else 0.0
|
|
53
|
+
b_f_hor = (s_f_hor - s_v_hor) / s_f_hor if s_f_hor > 0 else 0.0
|
|
54
|
+
return max(b_f_ver, b_f_hor)
|
texture3d/camera.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Camera and posed-frame primitives.
|
|
2
|
+
|
|
3
|
+
The library is dataset-agnostic: callers hand `bake()` a sequence of
|
|
4
|
+
`RGBDFrame`s, each holding (or lazily pointing at) an RGB image, a metric
|
|
5
|
+
depth image, a camera-to-world pose, and pinhole intrinsics. Convenience
|
|
6
|
+
loaders for on-disk layouts live in `texture3d.io`.
|
|
7
|
+
|
|
8
|
+
Conventions:
|
|
9
|
+
- camera looks down +z, x right, y down (OpenCV pinhole);
|
|
10
|
+
- poses are 4x4 camera-to-world;
|
|
11
|
+
- RGB images are HxWx3 uint8 in RGB channel order;
|
|
12
|
+
- depth images are HxW float32 meters (0 = invalid).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Optional, Union
|
|
20
|
+
|
|
21
|
+
import cv2
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
ImageSource = Union[np.ndarray, str, Path]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class PinholeCamera:
|
|
29
|
+
width: int
|
|
30
|
+
height: int
|
|
31
|
+
fx: float
|
|
32
|
+
fy: float
|
|
33
|
+
cx: float
|
|
34
|
+
cy: float
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def from_matrix(cls, k: np.ndarray, width: int, height: int) -> PinholeCamera:
|
|
38
|
+
k = np.asarray(k, dtype=np.float64)
|
|
39
|
+
return cls(
|
|
40
|
+
width=int(width), height=int(height),
|
|
41
|
+
fx=float(k[0, 0]), fy=float(k[1, 1]),
|
|
42
|
+
cx=float(k[0, 2]), cy=float(k[1, 2]),
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def matrix(self) -> np.ndarray:
|
|
47
|
+
return np.array(
|
|
48
|
+
[[self.fx, 0.0, self.cx], [0.0, self.fy, self.cy], [0.0, 0.0, 1.0]],
|
|
49
|
+
dtype=np.float64,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def project(self, points_cam: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
|
53
|
+
"""Project (N, 3) camera-space points. Returns ((N, 2) pixels, (N,) z)."""
|
|
54
|
+
z = points_cam[:, 2]
|
|
55
|
+
zsafe = np.where(z != 0, z, 1.0)
|
|
56
|
+
u = points_cam[:, 0] / zsafe * self.fx + self.cx
|
|
57
|
+
v = points_cam[:, 1] / zsafe * self.fy + self.cy
|
|
58
|
+
return np.stack([u, v], axis=-1), z
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass
|
|
62
|
+
class RGBDFrame:
|
|
63
|
+
"""One posed RGB-D frame. Images may be in-memory arrays or file paths.
|
|
64
|
+
|
|
65
|
+
`depth` semantics: float arrays are metric meters already; integer arrays
|
|
66
|
+
and integer image files are multiplied by `depth_scale` (meters per unit,
|
|
67
|
+
e.g. 1e-3 for millimeter PNGs). Zeros mean invalid.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
pose_c2w: np.ndarray # (4, 4)
|
|
71
|
+
camera: PinholeCamera
|
|
72
|
+
rgb: ImageSource
|
|
73
|
+
depth: ImageSource
|
|
74
|
+
depth_scale: float = 1.0e-3
|
|
75
|
+
depth_camera: Optional[PinholeCamera] = None
|
|
76
|
+
name: str = ""
|
|
77
|
+
|
|
78
|
+
def __post_init__(self):
|
|
79
|
+
self.pose_c2w = np.asarray(self.pose_c2w, dtype=np.float64)
|
|
80
|
+
if self.pose_c2w.shape != (4, 4):
|
|
81
|
+
raise ValueError(f"pose_c2w must be 4x4, got {self.pose_c2w.shape}")
|
|
82
|
+
if self.depth_camera is None:
|
|
83
|
+
self.depth_camera = self.camera
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def pose_w2c(self) -> np.ndarray:
|
|
87
|
+
return np.linalg.inv(self.pose_c2w)
|
|
88
|
+
|
|
89
|
+
def load_rgb(self) -> np.ndarray:
|
|
90
|
+
"""HxWx3 uint8, RGB order."""
|
|
91
|
+
if isinstance(self.rgb, np.ndarray):
|
|
92
|
+
return self.rgb
|
|
93
|
+
bgr = cv2.imread(str(self.rgb), cv2.IMREAD_COLOR)
|
|
94
|
+
if bgr is None:
|
|
95
|
+
raise OSError(f"Failed to read RGB image {self.rgb}")
|
|
96
|
+
return cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
|
|
97
|
+
|
|
98
|
+
def load_depth_m(self) -> np.ndarray:
|
|
99
|
+
"""HxW float32 meters, 0 = invalid."""
|
|
100
|
+
if isinstance(self.depth, np.ndarray):
|
|
101
|
+
arr = self.depth
|
|
102
|
+
else:
|
|
103
|
+
arr = cv2.imread(str(self.depth), cv2.IMREAD_UNCHANGED)
|
|
104
|
+
if arr is None:
|
|
105
|
+
raise OSError(f"Failed to read depth image {self.depth}")
|
|
106
|
+
if np.issubdtype(arr.dtype, np.floating):
|
|
107
|
+
return arr.astype(np.float32)
|
|
108
|
+
return arr.astype(np.float32) * self.depth_scale
|
texture3d/cli.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Command-line interface.
|
|
2
|
+
|
|
3
|
+
texture3d DATASET_DIR --mesh mesh.ply [--out DIR] [--format obj|glb] ...
|
|
4
|
+
|
|
5
|
+
DATASET_DIR must follow the dataset.json layout understood by
|
|
6
|
+
texture3d.io.load_dataset. When --mesh is omitted, the repo convention
|
|
7
|
+
reconstructions/<dataset>/mesh.ply is tried.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import json
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from texture3d.bake import bake
|
|
17
|
+
from texture3d.config import BakeConfig
|
|
18
|
+
from texture3d.io import load_dataset
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def main(argv=None) -> None:
|
|
22
|
+
p = argparse.ArgumentParser(prog="texture3d", description=__doc__)
|
|
23
|
+
p.add_argument("dataset", help="dataset directory (dataset.json layout)")
|
|
24
|
+
p.add_argument("--mesh", type=Path, default=None, help="mesh to texture (.ply/.obj/.stl)")
|
|
25
|
+
p.add_argument("--out", type=Path, default=None, help="output dir (default textured/<dataset>)")
|
|
26
|
+
p.add_argument("--format", choices=["obj", "glb"], default="obj")
|
|
27
|
+
# Keyframes
|
|
28
|
+
p.add_argument("--gap", type=int, default=5, help="1 keyframe per N frames (1 = all)")
|
|
29
|
+
# Mapping
|
|
30
|
+
p.add_argument("--outlier", type=float, default=0.06,
|
|
31
|
+
help="Gaussian likelihood below which a view is an outlier for a face (0 = off)")
|
|
32
|
+
p.add_argument("--data-ratio", type=float, default=0.1,
|
|
33
|
+
help="data cost relative to the smoothness cost; lower = larger patches")
|
|
34
|
+
p.add_argument("--no-global-leveling", action="store_true", help="skip the global seam leveling")
|
|
35
|
+
p.add_argument("--no-local-leveling", action="store_true", help="skip the local seam leveling")
|
|
36
|
+
p.add_argument("--strip-width", type=int, default=20, help="local leveling strip width in px")
|
|
37
|
+
p.add_argument("--sharpness", type=float, default=0.5, help="unsharp-mask weight on the atlas (0 = off)")
|
|
38
|
+
p.add_argument("--max-texture", type=int, default=8192, help="largest atlas edge in px")
|
|
39
|
+
p.add_argument("--no-fill-unseen", action="store_true",
|
|
40
|
+
help="leave sub-pixel faces no view rasterised in the empty colour")
|
|
41
|
+
args = p.parse_args(argv)
|
|
42
|
+
|
|
43
|
+
dataset_dir = Path(args.dataset)
|
|
44
|
+
mesh_path = args.mesh or Path("reconstructions") / dataset_dir.name / "mesh.ply"
|
|
45
|
+
if not mesh_path.exists():
|
|
46
|
+
raise SystemExit(f"Mesh not found: {mesh_path} (pass --mesh)")
|
|
47
|
+
out_dir = args.out or Path("textured") / dataset_dir.name
|
|
48
|
+
|
|
49
|
+
cfg = BakeConfig()
|
|
50
|
+
cfg.keyframes.rgbd_frame_gap = args.gap
|
|
51
|
+
cfg.mapping.outlier_threshold = args.outlier
|
|
52
|
+
cfg.mapping.data_smoothness_ratio = args.data_ratio
|
|
53
|
+
cfg.mapping.global_seam_leveling = not args.no_global_leveling
|
|
54
|
+
cfg.mapping.local_seam_leveling = not args.no_local_leveling
|
|
55
|
+
cfg.mapping.local_leveling_strip_width = args.strip_width
|
|
56
|
+
cfg.mapping.sharpness_weight = args.sharpness
|
|
57
|
+
cfg.mapping.max_texture_size = args.max_texture
|
|
58
|
+
cfg.mapping.fill_unseen = not args.no_fill_unseen
|
|
59
|
+
|
|
60
|
+
frames = load_dataset(dataset_dir)
|
|
61
|
+
result = bake(mesh_path, frames, cfg)
|
|
62
|
+
|
|
63
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
64
|
+
out_path = result.export(out_dir / f"textured_mesh.{args.format}")
|
|
65
|
+
(out_dir / "config.json").write_text(cfg.to_json() + "\n")
|
|
66
|
+
result.stats["output"] = str(out_path)
|
|
67
|
+
result.stats["mesh_input"] = str(mesh_path)
|
|
68
|
+
result.stats["dataset_dir"] = str(dataset_dir)
|
|
69
|
+
(out_dir / "summary.json").write_text(json.dumps(result.stats, indent=2) + "\n")
|
|
70
|
+
print(f"Done: {out_path}")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
if __name__ == "__main__":
|
|
74
|
+
main()
|
texture3d/config.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Bake configuration."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import asdict, dataclass, field
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class KeyframeConfig:
|
|
11
|
+
"""Keyframe selection: the least blurry frame of every window, skipping
|
|
12
|
+
near-duplicate poses."""
|
|
13
|
+
|
|
14
|
+
# 1 keyframe per window of this many frames (the least blurry). 1 = use all.
|
|
15
|
+
rgbd_frame_gap: int = 5
|
|
16
|
+
# Candidates whose pose is close to the previous keyframe are skipped.
|
|
17
|
+
closest_pose_translation: float = 0.05 # meters
|
|
18
|
+
closest_pose_rotation_angle: float = 0.09 # radians
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class MappingConfig:
|
|
23
|
+
"""The one-keyframe-per-face mapping (`texture3d.mapping`)."""
|
|
24
|
+
|
|
25
|
+
# Views whose mean face colour falls below this Gaussian likelihood against
|
|
26
|
+
# the other views' colours are dropped for that face. 0 disables the test.
|
|
27
|
+
outlier_threshold: float = 0.06
|
|
28
|
+
# Data cost scale relative to the Potts smoothness cost. Lower makes
|
|
29
|
+
# larger, fewer patches.
|
|
30
|
+
data_smoothness_ratio: float = 0.1
|
|
31
|
+
global_seam_leveling: bool = True
|
|
32
|
+
local_seam_leveling: bool = True
|
|
33
|
+
# Width in pixels of the border strip the local leveling blends.
|
|
34
|
+
local_leveling_strip_width: int = 20
|
|
35
|
+
# Unsharp-mask weight applied to the finished atlas. 0 disables it.
|
|
36
|
+
sharpness_weight: float = 0.5
|
|
37
|
+
# The atlas is the smallest power of two that packs the patches, up to this.
|
|
38
|
+
max_texture_size: int = 8192
|
|
39
|
+
# Colour of atlas texels no patch covers (RGB).
|
|
40
|
+
empty_color: tuple = (255, 255, 255)
|
|
41
|
+
# Views scored at once; each holds a face map and a depth map while scored.
|
|
42
|
+
scoring_workers: int = 4
|
|
43
|
+
# Give a face no keyframe rasterised (smaller than a pixel in every view)
|
|
44
|
+
# the view of a textured neighbour instead of leaving it the empty colour.
|
|
45
|
+
fill_unseen: bool = True
|
|
46
|
+
# Patch count up to which the atlas is packed with MaxRects (quadratic in
|
|
47
|
+
# the patch count); beyond it a shelf packer runs. 0 = always MaxRects.
|
|
48
|
+
exact_packing_limit: int = 4000
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class BakeConfig:
|
|
53
|
+
keyframes: KeyframeConfig = field(default_factory=KeyframeConfig)
|
|
54
|
+
mapping: MappingConfig = field(default_factory=MappingConfig)
|
|
55
|
+
|
|
56
|
+
def to_json(self) -> str:
|
|
57
|
+
return json.dumps(asdict(self), indent=2)
|
texture3d/io.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Dataset loaders.
|
|
2
|
+
|
|
3
|
+
`load_dataset` reads the per-frame RGB-D layout produced by
|
|
4
|
+
create_rgbd_dataset.py (dataset.json index + <frame_id>/{rgb.jpg, depth.png,
|
|
5
|
+
pose.txt}), returning plain `RGBDFrame`s. Other layouts can be adapted by
|
|
6
|
+
constructing `RGBDFrame`s directly.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import List
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
from texture3d.camera import PinholeCamera, RGBDFrame
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def load_dataset(dataset_dir: str | Path) -> List[RGBDFrame]:
|
|
21
|
+
root = Path(dataset_dir)
|
|
22
|
+
index_path = root / "dataset.json"
|
|
23
|
+
if not index_path.exists():
|
|
24
|
+
raise FileNotFoundError(f"No dataset.json in {root}")
|
|
25
|
+
index = json.loads(index_path.read_text())
|
|
26
|
+
|
|
27
|
+
size = index["image_size"]
|
|
28
|
+
color = PinholeCamera.from_matrix(
|
|
29
|
+
np.asarray(index["intrinsics"]["color_matrix_3x3"]), size["width"], size["height"]
|
|
30
|
+
)
|
|
31
|
+
depth_k = index["intrinsics"].get("depth_matrix_3x3")
|
|
32
|
+
depth_cam = (
|
|
33
|
+
PinholeCamera.from_matrix(np.asarray(depth_k), size["width"], size["height"])
|
|
34
|
+
if depth_k is not None
|
|
35
|
+
else color
|
|
36
|
+
)
|
|
37
|
+
depth_scale = float(index["depth"]["scale_to_meters"])
|
|
38
|
+
|
|
39
|
+
frames = []
|
|
40
|
+
for rec in index["frames"]:
|
|
41
|
+
frames.append(
|
|
42
|
+
RGBDFrame(
|
|
43
|
+
pose_c2w=np.loadtxt(root / rec["pose"]).reshape(4, 4),
|
|
44
|
+
camera=color,
|
|
45
|
+
rgb=root / rec["rgb"],
|
|
46
|
+
depth=root / rec["depth"],
|
|
47
|
+
depth_scale=depth_scale,
|
|
48
|
+
depth_camera=depth_cam,
|
|
49
|
+
name=rec.get("frame_id", ""),
|
|
50
|
+
)
|
|
51
|
+
)
|
|
52
|
+
return frames
|
texture3d/keyframes.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Keyframe selection.
|
|
2
|
+
|
|
3
|
+
From every window of `rgbd_frame_gap` frames, take the least blurry one
|
|
4
|
+
(Crete metric); drop it if its pose is too close to the previously accepted
|
|
5
|
+
keyframe.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import List, Sequence
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
from scipy.spatial.transform import Rotation
|
|
14
|
+
|
|
15
|
+
from texture3d.blur import estimate_blurriness
|
|
16
|
+
from texture3d.camera import RGBDFrame
|
|
17
|
+
from texture3d.config import KeyframeConfig
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def poses_close(t1: np.ndarray, t2: np.ndarray, cfg: KeyframeConfig) -> bool:
|
|
21
|
+
"""True when translation or relative rotation is under the threshold."""
|
|
22
|
+
if np.linalg.norm(t1[:3, 3] - t2[:3, 3]) < cfg.closest_pose_translation:
|
|
23
|
+
return True
|
|
24
|
+
relative = t1[:3, :3].T @ t2[:3, :3]
|
|
25
|
+
angles = Rotation.from_matrix(relative).as_euler("xyz")
|
|
26
|
+
return bool(np.linalg.norm(angles) < cfg.closest_pose_rotation_angle)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def select_keyframes(frames: Sequence[RGBDFrame], cfg: KeyframeConfig) -> List[RGBDFrame]:
|
|
30
|
+
selected: List[RGBDFrame] = []
|
|
31
|
+
n = len(frames)
|
|
32
|
+
gap = max(1, cfg.rgbd_frame_gap)
|
|
33
|
+
for start in range(0, n, gap):
|
|
34
|
+
window = frames[start : min(start + gap, n)]
|
|
35
|
+
if len(window) == 1:
|
|
36
|
+
best = window[0]
|
|
37
|
+
else:
|
|
38
|
+
best = min(window, key=lambda f: estimate_blurriness(f.load_rgb()))
|
|
39
|
+
if selected and poses_close(selected[-1].pose_c2w, best.pose_c2w, cfg):
|
|
40
|
+
continue
|
|
41
|
+
selected.append(best)
|
|
42
|
+
return selected
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""One-keyframe-per-face texture mapping with seam leveling.
|
|
2
|
+
|
|
3
|
+
See ``texture3d.mapping.types`` for the stage list and conventions.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from texture3d.mapping.mapper import Tuning, run
|
|
7
|
+
from texture3d.mapping.types import (
|
|
8
|
+
Camera,
|
|
9
|
+
EmptyMesh,
|
|
10
|
+
MappingError,
|
|
11
|
+
Mesh,
|
|
12
|
+
NoViews,
|
|
13
|
+
NoVisibleFace,
|
|
14
|
+
PatchTooLarge,
|
|
15
|
+
Result,
|
|
16
|
+
View,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"run",
|
|
21
|
+
"Tuning",
|
|
22
|
+
"Mesh",
|
|
23
|
+
"Camera",
|
|
24
|
+
"View",
|
|
25
|
+
"Result",
|
|
26
|
+
"MappingError",
|
|
27
|
+
"EmptyMesh",
|
|
28
|
+
"NoViews",
|
|
29
|
+
"NoVisibleFace",
|
|
30
|
+
"PatchTooLarge",
|
|
31
|
+
]
|