reverse3d 0.5.2__tar.gz → 0.7.0__tar.gz

{reverse3d-0.5.2/reverse3d.egg-info → reverse3d-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: reverse3d
-Version: 0.5.2
+Version: 0.7.0
 Summary: AI-agent-ready toolkit for 3D reverse engineering: image analysis, photogrammetry, mesh repair, primitive fitting and headless rendering.
 Author-email: Voxel Enterprise <info@voxel.lat>
 Maintainer-email: Voxel Enterprise <info@voxel.lat>
@@ -31,6 +31,7 @@ License-File: LICENSE
 License-File: NOTICE
 Requires-Dist: numpy>=1.26
 Requires-Dist: scipy>=1.11
+Requires-Dist: scikit-image>=0.22
 Requires-Dist: opencv-python-headless>=4.9
 Requires-Dist: pillow>=10.3
 Requires-Dist: pillow-heif>=1.1

{reverse3d-0.5.2 → reverse3d-0.7.0}/pyproject.toml

@@ -39,6 +39,7 @@ classifiers = [
 dependencies = [
     "numpy>=1.26",
     "scipy>=1.11",
+    "scikit-image>=0.22",
     "opencv-python-headless>=4.9",
     "pillow>=10.3",
     "pillow-heif>=1.1",

reverse3d-0.7.0/reverse3d/_version.py

@@ -0,0 +1 @@
+__version__ = "0.7.0"

{reverse3d-0.5.2 → reverse3d-0.7.0}/reverse3d/offline.py

@@ -54,6 +54,7 @@ MODEL_REVISIONS: dict[str, str] = {
     "depth-anything/Depth-Anything-V2-Small-hf": "5426e4f0f36572d16453bbda7a8389317b1bef99",
     "depth-anything/Depth-Anything-V2-Base-hf": "b1958afc87fb45a9e3746cb387596094de553ed8",
     "depth-anything/Depth-Anything-V2-Large-hf": "7581137eff8d4e94f6e796d3baea0e9fa79b22d2",
+    "facebook/sam-vit-base": "70c1a07f894ebb5b307fd9eaaee97b9dfc16068f",
 }
 
 
@@ -100,8 +101,9 @@ def is_offline(settings: Settings) -> bool:
 
 def _allowed_models() -> dict[str, tuple[str, str]]:
     from reverse3d.reconstruction.depth_estimation import ALLOWED_MODELS
+    from reverse3d.vision.segmenter import ALLOWED_SEGMENTERS
 
-    return dict(ALLOWED_MODELS)
+    return {**ALLOWED_MODELS, **ALLOWED_SEGMENTERS}
 
 
 def _default_model() -> str:
@@ -177,7 +179,7 @@ def resolve_model(model_id: str, settings: Settings) -> Path:
         return local
     if is_offline(settings):
         raise DependencyMissingError(
-            "The AI depth model is not on this computer and there is no internet connection.",
+            "The AI model is not on this computer and there is no internet connection.",
             hint="Connect to the internet once and run: reverse3d offline prepare",
         )
     try:
@@ -190,7 +192,7 @@ def resolve_model(model_id: str, settings: Settings) -> Path:
     except Exception as exc:  # network errors come in many shapes; report them plainly
         has_internet(refresh=True)
         raise DependencyMissingError(
-            f"Could not download the AI depth model ({exc.__class__.__name__}).",
+            f"Could not download the AI model ({exc.__class__.__name__}).",
             hint="Check your connection and run: reverse3d offline prepare",
         ) from exc
 
@@ -201,7 +203,10 @@ def prepare(settings: Settings, *, include_non_commercial: bool = False) -> dict
     Safe to run any time: models already on disk are skipped. It never raises
     for a missing optional stack; that is reported in the result instead.
     """
-    models = [_default_model()]
+    from reverse3d.vision.segmenter import DEFAULT_SEGMENTER
+
+    # Depth (reliefs) and segmentation (object silhouettes), both commercial-friendly.
+    models = [_default_model(), DEFAULT_SEGMENTER]
     if include_non_commercial:
         models += [m for m in _allowed_models() if m not in models]
 

{reverse3d-0.5.2 → reverse3d-0.7.0}/reverse3d/recipes.py

@@ -24,7 +24,6 @@ from __future__ import annotations
 import json
 import logging
 import math
-import shutil
 import time
 from collections.abc import Callable
 from dataclasses import dataclass, field
@@ -37,6 +36,7 @@ from reverse3d.errors import (
     DependencyMissingError,
     InputValidationError,
     ProcessingError,
+    Reverse3DError,
 )
 from reverse3d.hardware import torch_device, wants_gpu
 from reverse3d.profiles import profile_values
@@ -195,7 +195,11 @@ _MIN_GROUP = 8
 
 
 def _photos_to_model(run: _Run, src: Path, opt: RecipeOptions) -> MeshStats:
-    from reverse3d.reconstruction.photogrammetry import PhotogrammetryOptions, run_photogrammetry
+    from reverse3d.reconstruction.photogrammetry import (
+        PhotogrammetryOptions,
+        PhotogrammetryResult,
+        run_photogrammetry,
+    )
     from reverse3d.scenes import SCENE_LABEL_ES, SCENE_PRESETS, detect_scene
     from reverse3d.vision.feature_matcher import analyze_image_set
 
@@ -219,21 +223,9 @@ def _photos_to_model(run: _Run, src: Path, opt: RecipeOptions) -> MeshStats:
     analysis = run.step("analyze", lambda: analyze_image_set(src, run.settings))
     run.details["analysis"] = analysis.to_dict()
     if not analysis.suitable_for_photogrammetry and analysis.connected_images >= _MIN_GROUP:
-        # Part of the set overlaps well: reconstruct from that group only.
-        dropped = set(analysis.weak_links)
-        group_dir = run.out / "photos_used"
-        group_dir.mkdir(parents=True, exist_ok=True)
-        for image in sorted(src.iterdir()):
-            if image.is_file() and image.name not in dropped and not image.name.startswith("."):
-                target = group_dir / image.name
-                if not target.exists():
-                    shutil.copy2(image, target)
-        run.warnings.append(
-            f"Used the {analysis.connected_images} photos that overlap; skipped "
-            f"{len(dropped)} that do not match the rest."
-        )
-        run.details["photos_used"] = analysis.connected_images
-        src = group_dir
+        # COLMAP places cameras far better than this quick check (it often uses
+        # the whole room), so every photo goes in; the check is only advice.
+        run.warnings.extend(analysis.recommendations)
     elif not analysis.suitable_for_photogrammetry:
         if not opt.force_analysis:
             raise CaptureQualityError(
@@ -244,26 +236,76 @@ def _photos_to_model(run: _Run, src: Path, opt: RecipeOptions) -> MeshStats:
         run.warnings.extend(analysis.recommendations)
 
     quality = opt.quality or preset.quality or profile_values(run.settings).photo_quality
-    photo = run.step(
-        "photogrammetry",
-        lambda: run_photogrammetry(
+    # Objects and characters: COLMAP only places the cameras (fast, no dense stage)
+    # and the shape is carved from the silhouettes, which also works on plain,
+    # white or glossy surfaces. Open scenes keep classic dense reconstruction.
+    silhouette = scene in ("object", "character")
+
+    def photogrammetry(classic: bool) -> PhotogrammetryResult:
+        return run_photogrammetry(
             src,
             run.out / "reconstruction",
             run.settings,
             PhotogrammetryOptions(
                 quality=quality,  # type: ignore[arg-type]
                 matcher=preset.matcher,
+                dense="auto" if classic else "off",
                 use_gpu=wants_gpu(run.settings),
+                build_mesh=classic,
                 poisson_depth=preset.poisson_depth,
                 density_quantile=preset.density_quantile,
             ),
-            overwrite=opt.overwrite,
-        ),
-    )
+            overwrite=opt.overwrite or not classic,
+        )
+
+    photo = run.step("photogrammetry", lambda: photogrammetry(classic=not silhouette))
+    mesh_path: str | None = photo.mesh_path
+    upright = False
+    if silhouette:
+        from reverse3d.reconstruction.silhouette import visual_hull
+        from reverse3d.vision.segmenter import ObjectSegmenter
+
+        def load_segmenter() -> ObjectSegmenter | None:
+            # The AI segmenter keeps dark caps, handles and thin parts; without it
+            # (no AI extra, or offline before the first download) GrabCut is used.
+            try:
+                return ObjectSegmenter(run.settings, device=torch_device(run.settings))
+            except Reverse3DError as exc:
+                logger.info("AI segmentation unavailable (%s); using GrabCut", exc.message)
+                run.warnings.append(
+                    "AI segmentation unavailable: thin or dark parts may be missing."
+                )
+                return None
+
+        try:
+            hull = run.step(
+                "silhouette",
+                lambda: visual_hull(
+                    Path(photo.images_dir or ""),
+                    Path(photo.sparse_text_dir or ""),
+                    run.out / "reconstruction" / "object_hull.ply",
+                    segmenter=load_segmenter,
+                ),
+            )
+        except ProcessingError as exc:
+            logger.info("Silhouette carving failed (%s); using dense reconstruction", exc.message)
+            run.warnings.append(
+                f"Silhouette carving failed ({exc.message}); used dense reconstruction."
+            )
+            photo = run.step("photogrammetry", lambda: photogrammetry(classic=True))
+            mesh_path = photo.mesh_path
+        else:
+            run.details["silhouette"] = hull.to_dict()
+            mesh_path = hull.mesh_path
+            upright = True
+    if photo.registered_images >= 0.9 * max(photo.total_images, 1):
+        # COLMAP placed (almost) every photo: the quick pre-check's capture advice
+        # would only confuse.
+        run.warnings[:] = [w for w in run.warnings if w not in analysis.recommendations]
     run.warnings.extend(photo.warnings)
     run.details["photogrammetry"] = photo.to_dict()
     run.outputs["point_cloud"] = photo.dense_points_path or photo.sparse_points_path
-    if not photo.mesh_path:
+    if not mesh_path:
         raise ProcessingError(
             "Point cloud created but no mesh could be built.",
             hint='Install the meshing extra: pip install "reverse3d[recon]" (Python ≤ 3.12).',
@@ -271,11 +313,12 @@ def _photos_to_model(run: _Run, src: Path, opt: RecipeOptions) -> MeshStats:
     model = run.out / "model.stl"
     _clean(
         run,
-        Path(photo.mesh_path),
+        Path(mesh_path),
         model,
         scale=opt.scale,
         overwrite=opt.overwrite,
         fill_holes=preset.fill_holes,
+        align="none" if upright else "obb",
         keep_largest=preset.keep_largest,
         repair=opt.repair,
     )

{reverse3d-0.5.2 → reverse3d-0.7.0}/reverse3d/reconstruction/photogrammetry.py

@@ -68,7 +68,15 @@ _MIN_SPARSE_POINTS = 1000
 _EXHAUSTIVE_LIMIT = 150
 
 # Names of the artifacts this pipeline creates; only these are removed on overwrite.
-_ARTIFACTS = ("database.db", "images", "sparse", "dense", "sparse_points.ply", "mesh.ply")
+_ARTIFACTS = (
+    "database.db",
+    "images",
+    "sparse",
+    "sparse_txt",
+    "dense",
+    "sparse_points.ply",
+    "mesh.ply",
+)
 
 
 @dataclass
@@ -122,6 +130,9 @@ class PhotogrammetryResult:
     sparse_points: int
     stages: list[StageTiming] = field(default_factory=list)
     warnings: list[str] = field(default_factory=list)
+    #: the photos as COLMAP saw them, and the chosen model as text (silhouette carving).
+    images_dir: str | None = None
+    sparse_text_dir: str | None = None
 
     def to_dict(self) -> dict[str, object]:
         return asdict(self)
@@ -394,6 +405,23 @@ class ColmapPipeline:
             ],
         )
         result.sparse_points_path = str(sparse_ply)
+        result.images_dir = str(staged)
+        text_dir = out / "sparse_txt"
+        text_dir.mkdir(parents=True, exist_ok=True)
+        self._stage(
+            result,
+            "model_converter_txt",
+            [
+                "model_converter",
+                "--input_path",
+                str(best),
+                "--output_path",
+                str(text_dir),
+                "--output_type",
+                "TXT",
+            ],
+        )
+        result.sparse_text_dir = str(text_dir)
 
         # 4. Dense reconstruction (MVS) — CUDA only
         run_dense = options.dense == "on" or (options.dense == "auto" and options.use_gpu)

reverse3d-0.7.0/reverse3d/reconstruction/silhouette.py

@@ -0,0 +1,376 @@
+"""Shape from silhouettes (visual hull) for objects photographed all around.
+
+Classic multi-view stereo needs texture on the object itself, so plain,
+white, glossy or dark objects (a thermos, a bottle, a vase) come out as holes
+or blobs, while the textured room around them reconstructs fine. A visual
+hull does not need texture on the object: COLMAP only has to place the
+cameras (the room helps with that), then the object is carved as the volume
+that falls inside its silhouette in every photo.
+
+Pipeline:
+
+1. read the COLMAP model (cameras, poses, sparse points) exported as text;
+2. the object is where the cameras' optical axes converge;
+3. per photo, segment the object (GrabCut around its projected position);
+4. carve a voxel grid; re-segment each photo guided by the projected hull
+   and carve again (a few rounds tighten the silhouettes);
+5. cut the support (table) plane, keep the main part, mesh it with marching
+   cubes, smooth it and stand it upright on its base.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import cv2
+import numpy as np
+import trimesh
+
+from reverse3d.errors import DependencyMissingError, ProcessingError
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from reverse3d.vision.segmenter import ObjectSegmenter
+
+logger = logging.getLogger(__name__)
+
+_WORK_SIDE = 640
+_MIN_VIEWS = 8
+
+
+@dataclass
+class _Camera:
+    width: int
+    height: int
+    fx: float
+    fy: float
+    cx: float
+    cy: float
+    k: float
+
+
+@dataclass
+class _View:
+    name: str
+    rotation: np.ndarray
+    translation: np.ndarray
+    camera: _Camera
+
+
+@dataclass
+class HullResult:
+    mesh_path: str
+    views: int
+    voxels: int
+    faces: int
+    support_plane: bool
+    ai_segmentation: bool = False
+
+    def to_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+
+def _rotation(q: np.ndarray) -> np.ndarray:
+    w, x, y, z = q
+    return np.array(
+        [
+            [1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * w * z, 2 * x * z + 2 * w * y],
+            [2 * x * y + 2 * w * z, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * w * x],
+            [2 * x * z - 2 * w * y, 2 * y * z + 2 * w * x, 1 - 2 * x * x - 2 * y * y],
+        ]
+    )
+
+
+def read_colmap_text(model_dir: Path) -> tuple[list[_View], np.ndarray]:
+    """Views (intrinsics + world→camera pose) and sparse points of a TXT model."""
+    cameras: dict[int, _Camera] = {}
+    for line in (model_dir / "cameras.txt").read_text(encoding="utf-8").splitlines():
+        if not line or line.startswith("#"):
+            continue
+        p = line.split()
+        model, params = p[1], [float(v) for v in p[4:]]
+        if model in ("SIMPLE_RADIAL", "SIMPLE_PINHOLE", "RADIAL"):
+            f, cx, cy = params[:3]
+            k = params[3] if model in ("SIMPLE_RADIAL", "RADIAL") else 0.0
+            cam = _Camera(int(p[2]), int(p[3]), f, f, cx, cy, k)
+        else:  # PINHOLE, OPENCV...: fx fy cx cy [distortion ignored]
+            cam = _Camera(int(p[2]), int(p[3]), params[0], params[1], params[2], params[3], 0.0)
+        cameras[int(p[0])] = cam
+    # Each image takes two lines (header, then its 2D points, which may be empty).
+    rows = [
+        line
+        for line in (model_dir / "images.txt").read_text(encoding="utf-8").splitlines()
+        if not line.startswith("#")
+    ]
+    views = []
+    i = 0
+    while i < len(rows):
+        p = rows[i].split()
+        if len(p) < 10:
+            i += 1
+            continue
+        q = np.array([float(v) for v in p[1:5]])
+        t = np.array([float(v) for v in p[5:8]])
+        views.append(_View(" ".join(p[9:]), _rotation(q), t, cameras[int(p[8])]))
+        i += 2
+    points = []
+    for line in (model_dir / "points3D.txt").read_text(encoding="utf-8").splitlines():
+        if line and not line.startswith("#"):
+            points.append([float(v) for v in line.split()[1:4]])
+    return views, np.asarray(points, dtype=np.float64).reshape(-1, 3)
+
+
+def _project(points: np.ndarray, view: _View) -> tuple[np.ndarray, np.ndarray]:
+    cam = view.camera
+    xc = points @ view.rotation.T + view.translation
+    z = xc[:, 2]
+    safe = np.where(np.abs(z) < 1e-9, 1e-9, z)
+    x, y = xc[:, 0] / safe, xc[:, 1] / safe
+    s = 1 + cam.k * (x * x + y * y)
+    return np.stack([cam.fx * x * s + cam.cx, cam.fy * y * s + cam.cy], axis=1), z
+
+
+def convergence_point(views: list[_View]) -> tuple[np.ndarray, float]:
+    """Point closest to every camera's optical axis, and the median camera distance."""
+    a = np.zeros((3, 3))
+    b = np.zeros(3)
+    centers = []
+    for view in views:
+        c = -view.rotation.T @ view.translation
+        d = view.rotation.T @ np.array([0.0, 0.0, 1.0])
+        d /= np.linalg.norm(d)
+        m = np.eye(3) - np.outer(d, d)
+        a += m
+        b += m @ c
+        centers.append(c)
+    point = np.linalg.solve(a, b)
+    return point, float(np.median(np.linalg.norm(np.asarray(centers) - point, axis=1)))
+
+
+def _grabcut(
+    image: np.ndarray, init: np.ndarray, rect: tuple[int, int, int, int] | None
+) -> np.ndarray:
+    bgd, fgd = np.zeros((1, 65)), np.zeros((1, 65))
+    mode = cv2.GC_INIT_WITH_RECT if rect is not None else cv2.GC_INIT_WITH_MASK
+    try:
+        cv2.grabCut(image, init, rect or (0, 0, 0, 0), bgd, fgd, 4, mode)
+    except cv2.error:
+        return np.zeros(image.shape[:2], np.uint8)
+    return np.where((init == cv2.GC_FGD) | (init == cv2.GC_PR_FGD), 255, 0).astype(np.uint8)
+
+
+def _support_plane(
+    points: np.ndarray, center: np.ndarray, radius: float, cameras: np.ndarray
+) -> tuple[np.ndarray, float] | None:
+    """The table under the object: RANSAC plane through nearby sparse points, normal up."""
+    near = points[np.linalg.norm(points - center, axis=1) < 1.2 * radius]
+    if len(near) < 30:
+        return None
+    rng = np.random.default_rng(0)
+    best: tuple[np.ndarray, float, int] | None = None
+    for _ in range(400):
+        s = near[rng.choice(len(near), 3, replace=False)]
+        n = np.cross(s[1] - s[0], s[2] - s[0])
+        norm = np.linalg.norm(n)
+        if norm < 1e-12:
+            continue
+        n /= norm
+        d = -float(n @ s[0])
+        count = int((np.abs(near @ n + d) < 0.01 * radius).sum())
+        if best is None or count > best[2]:
+            best = (n, d, count)
+    if best is None or best[2] < 20:
+        return None
+    n, d, _ = best
+    if np.median(cameras @ n + d) < 0:  # cameras are above the table
+        n, d = -n, -d
+    return n, d
+
+
+def visual_hull(
+    image_dir: Path,
+    model_text_dir: Path,
+    output_path: Path,
+    *,
+    resolution: int = 176,
+    refinements: int = 3,
+    segmenter: Callable[[], ObjectSegmenter | None] | None = None,
+) -> HullResult:
+    """Carve the object seen by every photo; writes a watertight, upright mesh.
+
+    Raises:
+        ProcessingError: too few placed photos, or no object could be isolated.
+    """
+    try:
+        from scipy import ndimage
+        from skimage.measure import marching_cubes
+    except ImportError as exc:  # pragma: no cover - core dependencies
+        raise DependencyMissingError(
+            "scikit-image and scipy are needed for silhouette reconstruction.",
+            hint="pip install scikit-image scipy",
+        ) from exc
+
+    views, points = read_colmap_text(model_text_dir)
+    if len(views) < _MIN_VIEWS:
+        raise ProcessingError(
+            f"Only {len(views)} photos were placed; at least {_MIN_VIEWS} are needed."
+        )
+    center, distance = convergence_point(views)
+    radius = 0.45 * distance
+    cameras = np.asarray([-v.rotation.T @ v.translation for v in views])
+
+    images: list[tuple[np.ndarray, float]] = []
+    for view in views:
+        bgr = cv2.imread(str(image_dir / view.name))
+        if bgr is None:
+            raise ProcessingError(f"Could not read {view.name} for silhouette carving.")
+        h, w = bgr.shape[:2]
+        scale = _WORK_SIDE / max(h, w)
+        images.append(
+            (
+                cv2.resize(bgr, (round(w * scale), round(h * scale)), interpolation=cv2.INTER_AREA),
+                scale,
+            )
+        )
+
+    # First silhouettes: GrabCut in a box around where the object projects.
+    masks: list[np.ndarray] = []
+    for (small, scale), view in zip(images, views, strict=True):
+        uv, z = _project(center[None], view)
+        if z[0] <= 0:
+            masks.append(np.zeros(small.shape[:2], np.uint8))
+            continue
+        cu, cv_ = uv[0] * scale
+        half = view.camera.fx * scale * 0.6 * radius / z[0]
+        x0, y0 = max(0, int(cu - half)), max(0, int(cv_ - 1.6 * half))
+        x1 = min(small.shape[1] - 1, int(cu + half))
+        y1 = min(small.shape[0] - 1, int(cv_ + 1.6 * half))
+        init = np.zeros(small.shape[:2], np.uint8)
+        rect = (x0, y0, max(2, x1 - x0), max(2, y1 - y0))
+        fg = _grabcut(small, init, rect)
+        masks.append(cv2.morphologyEx(fg, cv2.MORPH_CLOSE, np.ones((7, 7), np.uint8)))
+
+    n = resolution
+    axis = np.linspace(-radius, radius, n)
+    gx, gy, gz = np.meshgrid(axis, axis, axis, indexing="ij")
+    grid = np.stack([gx.ravel(), gy.ravel(), gz.ravel()], axis=1) + center
+    projections = []
+    for (small, scale), view in zip(images, views, strict=True):
+        uv, z = _project(grid, view)
+        u = np.round(uv[:, 0] * scale).astype(np.int64)
+        v = np.round(uv[:, 1] * scale).astype(np.int64)
+        inside = (z > 0) & (u >= 0) & (v >= 0) & (u < small.shape[1]) & (v < small.shape[0])
+        projections.append((u, v, inside))
+
+    def carve(mask_list: list[np.ndarray], tolerance: int) -> np.ndarray:
+        votes = np.zeros(len(grid), np.int32)
+        seen = np.zeros(len(grid), np.int32)
+        for fg, (u, v, inside) in zip(mask_list, projections, strict=True):
+            seen += inside
+            hit = np.zeros(len(grid), bool)
+            hit[inside] = fg[v[inside], u[inside]] > 0
+            votes += hit
+        return (seen >= 3) & (votes >= seen - tolerance)
+
+    keep = carve(masks, 2)
+    # The AI segmenter is only loaded (or downloaded) once the photos proved usable.
+    ai = segmenter() if segmenter is not None else None
+    if ai is not None:
+        # AI segmentation: a box around the projected hull (taller upwards, where caps
+        # and handles get lost) and a point on it. Two rounds, each from the new hull.
+        states = [ai.embed(cv2.cvtColor(small, cv2.COLOR_BGR2RGB)) for small, _ in images]
+        for _ in range(2):
+            ai_masks: list[np.ndarray] = []
+            for state, (small, _), (u, v, inside), old in zip(
+                states, images, projections, masks, strict=True
+            ):
+                sel = keep & inside
+                if not sel.any():
+                    ai_masks.append(old)
+                    continue
+                us, vs = u[sel], v[sel]
+                x0, x1 = int(us.min()), int(us.max())
+                y0, y1 = int(vs.min()), int(vs.max())
+                pad_x, pad_y = 0.15 * (x1 - x0 + 1), 0.15 * (y1 - y0 + 1)
+                box = (
+                    max(0, int(x0 - pad_x)),
+                    max(0, int(y0 - 2.5 * pad_y)),
+                    min(small.shape[1] - 1, int(x1 + pad_x)),
+                    min(small.shape[0] - 1, int(y1 + pad_y)),
+                )
+                point = (int(np.median(us)), int(np.median(vs)))
+                ai_masks.append(ai.segment(state, box, point))
+            masks = ai_masks
+            keep = carve(masks, 1)
+        refinements = 0
+    for _ in range(refinements):
+        refined: list[np.ndarray] = []
+        for (small, _), (u, v, inside), old in zip(images, projections, masks, strict=True):
+            hull: np.ndarray = np.zeros(small.shape[:2], np.uint8)
+            sel = keep & inside
+            hull[v[sel], u[sel]] = 255
+            hull = cv2.morphologyEx(hull, cv2.MORPH_CLOSE, np.ones((9, 9), np.uint8))
+            if not hull.any():
+                refined.append(old)
+                continue
+            init = np.full(small.shape[:2], cv2.GC_BGD, np.uint8)
+            init[cv2.dilate(hull, np.ones((25, 25), np.uint8)) > 0] = cv2.GC_PR_BGD
+            init[hull > 0] = cv2.GC_PR_FGD
+            init[cv2.erode(hull, np.ones((21, 21), np.uint8)) > 0] = cv2.GC_FGD
+            fg = _grabcut(small, init, None)
+            refined.append(cv2.morphologyEx(fg, cv2.MORPH_OPEN, np.ones((5, 5), np.uint8)))
+        masks = refined
+        keep = carve(masks, 1)
+
+    volume = keep.reshape(n, n, n).astype(np.float32)
+    plane = _support_plane(points, center, radius, cameras)
+    if plane is not None:
+        normal, offset = plane
+        height = (grid @ normal + offset).reshape(n, n, n)
+        volume[height < 0.015 * radius] = 0
+    labels, count = ndimage.label(volume > 0.5)
+    if count == 0:
+        raise ProcessingError("No object could be isolated in the photos.")
+    sizes = ndimage.sum(np.ones_like(labels), labels, range(1, count + 1))
+    volume = (labels == 1 + int(np.argmax(sizes))).astype(np.float32)
+    voxels = int(volume.sum())
+    if voxels < 500:
+        raise ProcessingError("The isolated object is too small to build a model.")
+    touches_border = bool(
+        volume[0].any()
+        or volume[-1].any()
+        or volume[:, 0].any()
+        or volume[:, -1].any()
+        or volume[:, :, 0].any()
+        or volume[:, :, -1].any()
+    )
+    if touches_border:
+        logger.info("The object reaches the edge of the carving volume; it may be cropped.")
+    # One empty voxel of margin all around: marching cubes then yields a closed surface.
+    volume = np.pad(ndimage.gaussian_filter(volume, 1.4), 2)
+    verts, faces, _, _ = marching_cubes(volume, 0.5)
+    verts = (verts - 2) / (n - 1) * 2 * radius - radius + center
+    mesh = trimesh.Trimesh(verts, faces, process=True)
+    trimesh.smoothing.filter_taubin(mesh, lamb=0.5, nu=-0.53, iterations=15)
+
+    # Stand it upright: +Z along the table normal (or the mean camera "up").
+    up = plane[0] if plane is not None else np.mean(cameras - center, axis=0)
+    up = up / np.linalg.norm(up)
+    side = np.cross([0.0, 1.0, 0.0] if abs(up[1]) < 0.9 else [1.0, 0.0, 0.0], up)
+    side /= np.linalg.norm(side)
+    basis = np.stack([side, np.cross(up, side), up])
+    transform = np.eye(4)
+    transform[:3, :3] = basis
+    mesh.apply_transform(transform)
+    mesh.apply_translation([-mesh.centroid[0], -mesh.centroid[1], -mesh.bounds[0][2]])
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    mesh.export(output_path)
+    logger.info("Silhouette model: %d views, %d faces", len(views), len(mesh.faces))
+    return HullResult(
+        str(output_path), len(views), voxels, len(mesh.faces), plane is not None, ai is not None
+    )

reverse3d-0.7.0/reverse3d/vision/segmenter.py

@@ -0,0 +1,136 @@
+"""AI object segmentation (Segment Anything) for silhouette carving.
+
+GrabCut only compares colours, so a dark cap or handle in front of a dark
+background is cut off. Segment Anything (Meta, Apache-2.0) understands what
+"the object" is: given a box and a point on it, it returns the whole object,
+cap and handle included.
+
+The model is allowlisted, pinned to an exact commit and loaded from
+safetensors only (see :mod:`reverse3d.offline`); after the first download it
+works without internet. Everything degrades gracefully: without the AI extra
+or the model, callers fall back to GrabCut.
+"""
+
+from __future__ import annotations
+
+import logging
+from functools import lru_cache
+from typing import Any
+
+import numpy as np
+
+from reverse3d.config import Settings
+from reverse3d.errors import DependencyMissingError
+
+logger = logging.getLogger(__name__)
+
+#: Allowlisted segmentation models: id -> (kind, license).
+ALLOWED_SEGMENTERS: dict[str, tuple[str, str]] = {
+    "facebook/sam-vit-base": ("sam", "Apache-2.0"),
+}
+DEFAULT_SEGMENTER = "facebook/sam-vit-base"
+
+
+def segmenter_available() -> bool:
+    """True when the AI stack (torch + transformers) is installed."""
+    try:
+        import torch  # noqa: F401
+        import transformers  # noqa: F401
+    except ImportError:
+        return False
+    return True
+
+
+@lru_cache(maxsize=2)
+def _load(model_id: str, device: str, model_dir: str) -> tuple[Any, Any]:
+    from transformers import SamModel, SamProcessor
+
+    try:  # keep CLI/agent output clean: no loading progress bars
+        from transformers.utils import logging as hf_logging
+
+        hf_logging.disable_progress_bar()
+        hf_logging.set_verbosity_error()
+    except ImportError:  # pragma: no cover - very old transformers
+        pass
+    # model_dir is the local snapshot of a commit-pinned, allowlisted model
+    # (reverse3d.offline.resolve_model): nothing is fetched here.
+    model: Any = SamModel.from_pretrained(  # nosec B615
+        model_dir, local_files_only=True, use_safetensors=True, trust_remote_code=False
+    )
+    processor = SamProcessor.from_pretrained(model_dir, local_files_only=True)  # nosec B615
+    model.to(device).eval()
+    logger.info("Loaded %s on %s", model_id, device)
+    return model, processor
+
+
+class ObjectSegmenter:
+    """Segments the object inside a box (plus a point on it) in RGB images.
+
+    Raises:
+        DependencyMissingError: the AI extra or the model is not available.
+    """
+
+    def __init__(self, settings: Settings, device: str = "cpu", model_id: str = DEFAULT_SEGMENTER):
+        if not segmenter_available():
+            raise DependencyMissingError(
+                "The AI segmentation stack is not installed.",
+                hint='pip install "reverse3d[depth]"',
+            )
+        from reverse3d.offline import force_offline_env, is_offline, resolve_model
+
+        self.device = device
+        model_dir = resolve_model(model_id, settings)
+        if is_offline(settings):
+            force_offline_env()
+        self.model, self.processor = _load(model_id, device, str(model_dir))
+
+    def embed(self, rgb: np.ndarray) -> dict[str, Any]:
+        """Run the heavy image encoder once; reuse the result for several prompts."""
+        import torch
+
+        inputs = self.processor(rgb, return_tensors="pt").to(self.device)
+        with torch.no_grad():
+            embeddings = self.model.get_image_embeddings(inputs["pixel_values"])
+        return {
+            "rgb": rgb,
+            "embeddings": embeddings,
+            "original_sizes": inputs["original_sizes"],
+            "reshaped_input_sizes": inputs["reshaped_input_sizes"],
+        }
+
+    def segment(
+        self,
+        image: np.ndarray | dict[str, Any],
+        box: tuple[int, int, int, int],
+        point: tuple[int, int],
+    ) -> np.ndarray:
+        """Object mask (uint8, 255 = object) for a box ``(x0, y0, x1, y1)`` and a point.
+
+        ``image`` is an RGB array or the result of :meth:`embed` (much faster).
+        """
+        import torch
+
+        state = image if isinstance(image, dict) else self.embed(image)
+        prompts = self.processor(
+            state["rgb"],
+            input_boxes=[[[float(v) for v in box]]],
+            input_points=[[[[float(point[0]), float(point[1])]]]],
+            input_labels=[[[1]]],
+            return_tensors="pt",
+        ).to(self.device)
+        with torch.no_grad():
+            outputs = self.model(
+                image_embeddings=state["embeddings"],
+                input_boxes=prompts["input_boxes"],
+                input_points=prompts["input_points"],
+                input_labels=prompts["input_labels"],
+                multimask_output=True,
+            )
+        masks = self.processor.image_processor.post_process_masks(
+            outputs.pred_masks.cpu(),
+            state["original_sizes"].cpu(),
+            state["reshaped_input_sizes"].cpu(),
+        )[0][0]
+        scores = outputs.iou_scores.cpu()[0, 0]
+        best = int(torch.argmax(scores))
+        return np.where(masks[best].numpy(), 255, 0).astype(np.uint8)

{reverse3d-0.5.2 → reverse3d-0.7.0/reverse3d.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: reverse3d
-Version: 0.5.2
+Version: 0.7.0
 Summary: AI-agent-ready toolkit for 3D reverse engineering: image analysis, photogrammetry, mesh repair, primitive fitting and headless rendering.
 Author-email: Voxel Enterprise <info@voxel.lat>
 Maintainer-email: Voxel Enterprise <info@voxel.lat>
@@ -31,6 +31,7 @@ License-File: LICENSE
 License-File: NOTICE
 Requires-Dist: numpy>=1.26
 Requires-Dist: scipy>=1.11
+Requires-Dist: scikit-image>=0.22
 Requires-Dist: opencv-python-headless>=4.9
 Requires-Dist: pillow>=10.3
 Requires-Dist: pillow-heif>=1.1

{reverse3d-0.5.2 → reverse3d-0.7.0}/reverse3d.egg-info/SOURCES.txt

@@ -40,6 +40,7 @@ reverse3d/reconstruction/__init__.py
 reverse3d/reconstruction/depth_estimation.py
 reverse3d/reconstruction/photogrammetry.py
 reverse3d/reconstruction/pointcloud.py
+reverse3d/reconstruction/silhouette.py
 reverse3d/rendering/__init__.py
 reverse3d/rendering/_blender_job.py
 reverse3d/rendering/blender_render.py
@@ -52,6 +53,7 @@ reverse3d/reverse_eng/parametric.py
 reverse3d/vision/__init__.py
 reverse3d/vision/feature_matcher.py
 reverse3d/vision/mask_extractor.py
+reverse3d/vision/segmenter.py
 tests/test_agent_tools.py
 tests/test_authorship.py
 tests/test_automation.py
@@ -67,4 +69,5 @@ tests/test_reconstruction.py
 tests/test_render.py
 tests/test_scenes.py
 tests/test_security.py
+tests/test_silhouette.py
 tests/test_vision.py

{reverse3d-0.5.2 → reverse3d-0.7.0}/reverse3d.egg-info/requires.txt

@@ -1,5 +1,6 @@
 numpy>=1.26
 scipy>=1.11
+scikit-image>=0.22
 opencv-python-headless>=4.9
 pillow>=10.3
 pillow-heif>=1.1

{reverse3d-0.5.2 → reverse3d-0.7.0}/tests/test_automation.py

@@ -442,7 +442,11 @@ def test_colmap_without_cuda_degrades_to_cpu(
         if args[0] == "mapper":
             (Path(args[args.index("--output_path") + 1]) / "0").mkdir()
         if args[0] == "model_converter":
-            Path(args[args.index("--output_path") + 1]).write_text("ply")
+            target = Path(args[args.index("--output_path") + 1])
+            if args[args.index("--output_type") + 1] == "TXT":
+                target.mkdir(parents=True, exist_ok=True)
+            else:
+                target.write_text("ply")
         return subprocess.CompletedProcess(args, 0, "", "Registered images: 6\nPoints: 6000")
 
     monkeypatch.setattr(pg, "run_command", fake)

{reverse3d-0.5.2 → reverse3d-0.7.0}/tests/test_offline_autopilot.py

@@ -125,8 +125,9 @@ class TestModels:
 
     def test_every_allowlisted_model_is_pinned(self, tmp_path: Path) -> None:
         from reverse3d.reconstruction.depth_estimation import ALLOWED_MODELS
+        from reverse3d.vision.segmenter import ALLOWED_SEGMENTERS
 
-        assert set(ALLOWED_MODELS) == set(offline.MODEL_REVISIONS)
+        assert set(ALLOWED_MODELS) | set(ALLOWED_SEGMENTERS) == set(offline.MODEL_REVISIONS)
         assert all(len(sha) == 40 for sha in offline.MODEL_REVISIONS.values())
         other = tmp_path / "_r3d_home" / "models" / ("models--" + SMALL.replace("/", "--"))
         (other / "snapshots" / "unreviewed").mkdir(parents=True)

{reverse3d-0.5.2 → reverse3d-0.7.0}/tests/test_reconstruction.py

@@ -82,7 +82,10 @@ class FakeColmap:
                 args, 0, "", f"Registered images: {n}\nPoints: {n * self.points_per_image}"
             )
         elif command == "model_converter":
-            Path(opt("--output_path")).write_text("ply\n")
+            if opt("--output_type") == "TXT":
+                Path(opt("--output_path")).mkdir(parents=True, exist_ok=True)
+            else:
+                Path(opt("--output_path")).write_text("ply\n")
         return subprocess.CompletedProcess(args, 0, "", "")
 
 

reverse3d-0.7.0/tests/test_silhouette.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import cv2
+import numpy as np
+import pytest
+import trimesh
+
+pytest.importorskip("skimage")
+
+from reverse3d.errors import ProcessingError
+from reverse3d.reconstruction.silhouette import convergence_point, read_colmap_text, visual_hull
+
+SIZE = 400
+F = 420.0
+RADIUS, HEIGHT = 0.35, 1.2  # the object: an upright cylinder standing on z = 0
+
+
+def _look_at(eye: np.ndarray, target: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+    """World→camera rotation and translation (COLMAP convention: +Z forward, +Y down)."""
+    forward = target - eye
+    forward /= np.linalg.norm(forward)
+    right = np.cross(forward, [0.0, 0.0, 1.0])
+    right /= np.linalg.norm(right)
+    down = np.cross(forward, right)
+    rot = np.stack([right, down, forward])
+    return rot, -rot @ eye
+
+
+def _quat(rot: np.ndarray) -> list[float]:
+    return list(
+        trimesh.transformations.quaternion_from_matrix(
+            np.vstack([np.hstack([rot, [[0], [0], [0]]]), [0, 0, 0, 1]])
+        )
+    )
+
+
+def _scene(folder: Path) -> tuple[Path, Path]:
+    images = folder / "images"
+    text = folder / "txt"
+    images.mkdir(parents=True)
+    text.mkdir()
+    rng = np.random.default_rng(0)
+    theta = np.linspace(0, 2 * np.pi, 64, endpoint=False)
+    rim = np.concatenate(
+        [
+            np.stack([RADIUS * np.cos(theta), RADIUS * np.sin(theta), np.full_like(theta, z)], 1)
+            for z in (0.0, HEIGHT)
+        ]
+    )
+    lines = []
+    k = 0
+    for height, count in ((0.6, 12), (2.2, 12), (3.4, 6)):
+        for i in range(count):
+            a = 2 * np.pi * i / count
+            eye = np.array([3.2 * np.cos(a), 3.2 * np.sin(a), height])
+            rot, t = _look_at(eye, np.array([0.0, 0.0, 0.6]))
+            # Textured background, then the white cylinder's silhouette.
+            image = rng.integers(0, 120, (SIZE, SIZE, 3), dtype=np.uint8)
+            image = cv2.GaussianBlur(image, (5, 5), 0)
+            cam = rim @ rot.T + t
+            uv = (cam[:, :2] / cam[:, 2:3]) * F + SIZE / 2
+            cv2.fillConvexPoly(image, cv2.convexHull(uv.astype(np.int32)), (235, 235, 235))
+            name = f"img_{k:02d}.jpg"
+            cv2.imwrite(str(images / name), image)
+            q = _quat(rot)
+            lines.append(f"{k + 1} {q[0]} {q[1]} {q[2]} {q[3]} {t[0]} {t[1]} {t[2]} 1 {name}\n\n")
+            k += 1
+    (text / "cameras.txt").write_text(f"1 PINHOLE {SIZE} {SIZE} {F} {F} {SIZE / 2} {SIZE / 2}\n")
+    (text / "images.txt").write_text("".join(lines))
+    table = np.column_stack([rng.uniform(-1, 1, 400), rng.uniform(-1, 1, 400), np.zeros(400)])
+    table = table[np.hypot(table[:, 0], table[:, 1]) > RADIUS + 0.05]
+    (text / "points3D.txt").write_text(
+        "".join(f"{i + 1} {x} {y} {z} 0 0 0 0\n" for i, (x, y, z) in enumerate(table))
+    )
+    return images, text
+
+
+def test_cameras_converge_on_the_object(tmp_path: Path) -> None:
+    _, text = _scene(tmp_path)
+    views, points = read_colmap_text(text)
+    center, distance = convergence_point(views)
+    assert len(views) == 30 and len(points) > 100
+    assert np.linalg.norm(center - [0, 0, 0.6]) < 0.05
+    assert 3.0 < distance < 3.8
+
+
+def test_visual_hull_rebuilds_an_upright_cylinder(tmp_path: Path) -> None:
+    images, text = _scene(tmp_path)
+    result = visual_hull(images, text, tmp_path / "hull.ply", resolution=80, refinements=1)
+    mesh = trimesh.load(result.mesh_path)
+    assert result.support_plane
+    assert mesh.is_watertight
+    width = (mesh.extents[0] + mesh.extents[1]) / 2
+    assert mesh.bounds[0][2] == pytest.approx(0.0, abs=1e-6)
+    # Height / diameter of the real cylinder is 1.2 / 0.7 ≈ 1.7.
+    assert 1.3 < mesh.extents[2] / width < 2.2
+
+
+def test_too_few_photos(tmp_path: Path) -> None:
+    images, text = _scene(tmp_path)
+    rows = (text / "images.txt").read_text().split("\n\n")
+    (text / "images.txt").write_text("\n\n".join(rows[:5]) + "\n\n")
+    with pytest.raises(ProcessingError, match="at least"):
+        visual_hull(images, text, tmp_path / "hull.ply", resolution=48, refinements=0)
+
+
+class _FakeSegmenter:
+    """Stands in for Segment Anything: the object is the bright part inside the box."""
+
+    def __init__(self) -> None:
+        self.calls = 0
+
+    def embed(self, rgb: np.ndarray) -> dict[str, np.ndarray]:
+        return {"rgb": rgb}
+
+    def segment(
+        self, state: dict[str, np.ndarray], box: tuple[int, int, int, int], point: tuple[int, int]
+    ) -> np.ndarray:
+        self.calls += 1
+        gray = state["rgb"].mean(axis=2)
+        mask = np.zeros(gray.shape, np.uint8)
+        x0, y0, x1, y1 = box
+        mask[y0 : y1 + 1, x0 : x1 + 1] = np.where(gray[y0 : y1 + 1, x0 : x1 + 1] > 200, 255, 0)
+        return mask
+
+
+def test_ai_segmenter_is_loaded_lazily_and_used(tmp_path: Path) -> None:
+    images, text = _scene(tmp_path)
+    fake = _FakeSegmenter()
+    loads: list[int] = []
+
+    def factory() -> _FakeSegmenter:
+        loads.append(1)
+        return fake
+
+    result = visual_hull(
+        images,
+        text,
+        tmp_path / "hull.ply",
+        resolution=64,
+        refinements=1,
+        segmenter=factory,  # type: ignore[arg-type]
+    )
+    assert loads == [1] and fake.calls == 2 * 30  # two AI rounds over the 30 photos
+    assert result.ai_segmentation
+    mesh = trimesh.load(result.mesh_path)
+    width = (mesh.extents[0] + mesh.extents[1]) / 2
+    assert mesh.is_watertight and 1.3 < mesh.extents[2] / width < 2.2
+
+
+def test_without_ai_segmenter_grabcut_is_used(tmp_path: Path) -> None:
+    images, text = _scene(tmp_path)
+    result = visual_hull(
+        images, text, tmp_path / "hull.ply", resolution=48, refinements=1, segmenter=lambda: None
+    )
+    assert not result.ai_segmentation

reverse3d-0.5.2/reverse3d/_version.py

@@ -1 +0,0 @@
-__version__ = "0.5.2"