neuromeka-vfm 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

neuromeka_vfm/generate_mesh.py

@@ -0,0 +1,122 @@
+"""
+Utility to generate simple parametric meshes (currently rectangular box) as binary STL.
+
+Design goals
+- Units: meters
+- Origin: object center at (0, 0, 0)
+- Axes: faces aligned to +/-X, +/-Y, +/-Z
+- Output: binary STL saved to /opt/meshes (docker volume mount)
+
+Usage (programmatic):
+    from backend.generate_mesh import write_box_stl
+    path = write_box_stl("custom_box.stl", width=0.054, depth=0.097, height=0.054)
+
+CLI (optional):
+    python -m backend.generate_mesh box custom_box.stl 0.054 0.097 0.054
+"""
+
+from __future__ import annotations
+
+import struct
+import sys
+from pathlib import Path
+from typing import Iterable, Tuple
+
+MESH_DIR = Path("/opt/meshes")
+
+
+def _ensure_dir(path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def _pack_triangle(normal: Iterable[float], v1: Iterable[float], v2: Iterable[float], v3: Iterable[float]) -> bytes:
+    """Pack one triangle (normal + 3 vertices) into binary STL record."""
+    return struct.pack(
+        "<12fH",
+        *normal,
+        *v1,
+        *v2,
+        *v3,
+        0,  # attribute byte count
+    )
+
+
+def _box_triangles(width: float, depth: float, height: float):
+    """Generate normals and vertices for a box centered at origin."""
+    hx, hy, hz = width / 2.0, depth / 2.0, height / 2.0
+    # 8 vertices
+    p = [
+        (-hx, -hy, -hz),
+        (hx, -hy, -hz),
+        (hx, hy, -hz),
+        (-hx, hy, -hz),
+        (-hx, -hy, hz),
+        (hx, -hy, hz),
+        (hx, hy, hz),
+        (-hx, hy, hz),
+    ]
+    # Each face: two triangles (ccw when looking from outside)
+    faces = [
+        ((-1, 0, 0), (0, 1, 3, 7, 4)),  # -X
+        ((1, 0, 0), (1, 2, 6, 5)),  # +X
+        ((0, -1, 0), (0, 1, 5, 4)),  # -Y
+        ((0, 1, 0), (3, 2, 6, 7)),  # +Y
+        ((0, 0, -1), (0, 1, 2, 3)),  # -Z
+        ((0, 0, 1), (4, 5, 6, 7)),  # +Z
+    ]
+    for normal, idx in faces:
+        if len(idx) == 4:
+            a, b, c, d = idx
+            # two triangles: (a,b,c) and (a,c,d)
+            yield normal, p[a], p[b], p[c]
+            yield normal, p[a], p[c], p[d]
+        else:
+            raise ValueError("Face index must have 4 vertices.")
+
+
+def write_box_stl(filename: str, width: float, depth: float, height: float) -> Path:
+    """
+    Create a rectangular box STL.
+
+    Args:
+        filename: output file name (placed under /opt/meshes). If only a name is
+                  given, it is resolved relative to MESH_DIR.
+        width, depth, height: box dimensions in meters (must be > 0).
+
+    Returns:
+        Path to the written STL file.
+    """
+    if width <= 0 or depth <= 0 or height <= 0:
+        raise ValueError("width, depth, height must be positive.")
+
+    out_path = Path(filename)
+    if not out_path.is_absolute():
+        out_path = MESH_DIR / out_path
+    _ensure_dir(out_path)
+
+    triangles = list(_box_triangles(width, depth, height))
+    header = b"rect_box_stl" + b"\0" * (80 - len("rect_box_stl"))
+    with out_path.open("wb") as f:
+        f.write(header)
+        f.write(struct.pack("<I", len(triangles)))
+        for tri in triangles:
+            f.write(_pack_triangle(*tri))
+    return out_path
+
+
+def _cli(args: list[str]) -> int:
+    if len(args) != 5 or args[0].lower() != "box":
+        print("Usage: python -m backend.generate_mesh box <filename> <width> <depth> <height>")
+        return 1
+    _, fname, w, d, h = args
+    try:
+        path = write_box_stl(fname, float(w), float(d), float(h))
+    except Exception as exc:  # noqa: BLE001
+        print(f"Error: {exc}")
+        return 1
+    print(f"STL written to: {path}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(_cli(sys.argv[1:]))

neuromeka_vfm/point_cloud_utils.py

@@ -0,0 +1,377 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+#
+# NVIDIA CORPORATION and its licensors retain all intellectual property
+# and proprietary rights in and to this software, related documentation
+# and any modifications thereto. Any use, reproduction, disclosure or
+# distribution of this software and related documentation without an express
+# license agreement from NVIDIA CORPORATION is strictly prohibited.
+
+import logging
+from typing import Tuple, Dict
+
+import numpy as np
+import trimesh
+import trimesh.transformations as tra
+from tqdm import tqdm
+
+logger = logging.getLogger(__name__)
+
+
+def _pairwise_distances(X: np.ndarray, Y: np.ndarray, norm: int) -> np.ndarray:
+    if norm == 1:
+        return np.sum(np.abs(X[:, None, :] - Y[None, :, :]), axis=2)
+    if norm == 2:
+        diff = X[:, None, :] - Y[None, :, :]
+        return np.sqrt(np.sum(diff * diff, axis=2))
+    diff = X[:, None, :] - Y[None, :, :]
+    return np.linalg.norm(diff, ord=norm, axis=2)
+
+
+def knn_points(X: np.ndarray, K: int, norm: int):
+    """
+    Computes the K-nearest neighbors for each point in the point cloud X.
+
+    Args:
+        X: (N, 3) array representing the point cloud.
+        K: Number of nearest neighbors.
+
+    Returns:
+        dists: (N, K) array containing distances to the K nearest neighbors.
+        idxs: (N, K) array containing indices of the K nearest neighbors.
+    """
+    X = np.asarray(X, dtype=np.float32)
+    if X.ndim != 2 or X.shape[1] != 3:
+        raise ValueError("X must be a (N, 3) array")
+    if K <= 0:
+        raise ValueError("K must be positive")
+    N, _ = X.shape
+    if K >= N:
+        raise ValueError("K must be smaller than number of points")
+
+    dists_out = np.empty((N, K), dtype=np.float32)
+    idxs_out = np.empty((N, K), dtype=np.int64)
+
+    max_bytes = 64 * 1024 * 1024
+    bytes_per_row = N * X.dtype.itemsize
+    chunk_size = max(1, min(N, max_bytes // max(bytes_per_row, 1)))
+
+    for start in range(0, N, chunk_size):
+        end = min(start + chunk_size, N)
+        chunk = X[start:end]
+        dist_matrix = _pairwise_distances(chunk, X, norm=norm)
+
+        row_idx = np.arange(end - start)
+        col_idx = row_idx + start
+        dist_matrix[row_idx, col_idx] = np.inf
+
+        idx_part = np.argpartition(dist_matrix, K, axis=1)[:, :K]
+        dist_part = np.take_along_axis(dist_matrix, idx_part, axis=1)
+        order = np.argsort(dist_part, axis=1)
+        idxs = np.take_along_axis(idx_part, order, axis=1)
+        dists = np.take_along_axis(dist_part, order, axis=1)
+
+        dists_out[start:end] = dists
+        idxs_out[start:end] = idxs
+
+    return dists_out, idxs_out
+
+
+def point_cloud_outlier_removal(
+    obj_pc: np.ndarray, threshold: float = 0.014, K: int = 20
+) -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Remove outliers from a point cloud. K-nearest neighbors is used to compute
+    the distance to the nearest neighbor for each point. If the distance is
+    greater than a threshold, the point is considered an outlier and removed.
+
+    Args:
+        obj_pc (np.ndarray): (N, 3) array representing the point cloud.
+        threshold (float): Distance threshold for outlier detection. Points with mean distance to
+            K nearest neighbors greater than this threshold are removed.
+        K (int): Number of nearest neighbors to consider for outlier detection.
+
+    Returns:
+        Tuple[np.ndarray, np.ndarray]: Tuple containing filtered and removed point clouds.
+    """
+    obj_pc = np.asarray(obj_pc, dtype=np.float32)
+    if obj_pc.ndim != 2 or obj_pc.shape[1] != 3:
+        raise ValueError("obj_pc must be a (N, 3) array")
+
+    nn_dists, _ = knn_points(obj_pc, K=K, norm=1)
+
+    mask = nn_dists.mean(axis=1) < threshold
+    filtered_pc = obj_pc[mask]
+    removed_pc = obj_pc[~mask]
+
+    logger.info(
+        "Removed %s points from point cloud",
+        obj_pc.shape[0] - filtered_pc.shape[0],
+    )
+    return filtered_pc, removed_pc
+
+
+def point_cloud_outlier_removal_with_color(
+    obj_pc: np.ndarray,
+    obj_pc_color: np.ndarray,
+    threshold: float = 0.014,
+    K: int = 20,
+) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    """
+    Remove outliers from a point cloud with colors.
+
+    Args:
+        obj_pc (np.ndarray): (N, 3) array representing the point cloud.
+        obj_pc_color (np.ndarray): (N, 3) array representing the point cloud color.
+        threshold (float): Distance threshold for outlier detection. Points with mean distance to
+            K nearest neighbors greater than this threshold are removed.
+        K (int): Number of nearest neighbors to consider for outlier detection.
+
+    Returns:
+        Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: Tuple containing filtered and
+            removed point clouds and colors.
+    """
+    obj_pc = np.asarray(obj_pc, dtype=np.float32)
+    obj_pc_color = np.asarray(obj_pc_color, dtype=np.float32)
+    if obj_pc.ndim != 2 or obj_pc.shape[1] != 3:
+        raise ValueError("obj_pc must be a (N, 3) array")
+    if obj_pc_color.shape != obj_pc.shape:
+        raise ValueError("obj_pc_color must match obj_pc shape")
+
+    nn_dists, _ = knn_points(obj_pc, K=K, norm=1)
+
+    mask = nn_dists.mean(axis=1) < threshold
+    filtered_pc = obj_pc[mask]
+    removed_pc = obj_pc[~mask]
+
+    filtered_pc_color = obj_pc_color[mask]
+    removed_pc_color = obj_pc_color[~mask]
+
+    logger.info(
+        "Removed %s points from point cloud",
+        obj_pc.shape[0] - filtered_pc.shape[0],
+    )
+    return filtered_pc, removed_pc, filtered_pc_color, removed_pc_color
+
+
+def depth2points(
+    depth: np.array,
+    fx: int,
+    fy: int,
+    cx: int,
+    cy: int,
+    xmap: np.array = None,
+    ymap: np.array = None,
+    rgb: np.array = None,
+    seg: np.array = None,
+    mask: np.arange = None,
+) -> Dict:
+    """Compute point cloud from a depth image."""
+    if rgb is not None:
+        assert rgb.shape[0] == depth.shape[0] and rgb.shape[1] == depth.shape[1]
+    if xmap is not None:
+        assert xmap.shape[0] == depth.shape[0] and xmap.shape[1] == depth.shape[1]
+    if ymap is not None:
+        assert ymap.shape[0] == depth.shape[0] and ymap.shape[1] == depth.shape[1]
+
+    im_height, im_width = depth.shape[0], depth.shape[1]
+
+    if xmap is None or ymap is None:
+        ww = np.linspace(0, im_width - 1, im_width)
+        hh = np.linspace(0, im_height - 1, im_height)
+        xmap, ymap = np.meshgrid(ww, hh)
+
+    pt2 = depth
+    pt0 = (xmap - cx) * pt2 / fx
+    pt1 = (ymap - cy) * pt2 / fy
+
+    mask_depth = np.ma.getmaskarray(np.ma.masked_greater(pt2, 0))
+    if mask is None:
+        mask = mask_depth
+    else:
+        mask_semantic = np.ma.getmaskarray(np.ma.masked_equal(mask, 1))
+        mask = mask_depth * mask_semantic
+
+    index = mask.flatten().nonzero()[0]
+
+    pt2_valid = pt2.flatten()[:, np.newaxis].astype(np.float32)
+    pt0_valid = pt0.flatten()[:, np.newaxis].astype(np.float32)
+    pt1_valid = pt1.flatten()[:, np.newaxis].astype(np.float32)
+    pc_xyz = np.concatenate((pt0_valid, pt1_valid, pt2_valid), axis=1)
+    if rgb is not None:
+        r = rgb[:, :, 0].flatten()[:, np.newaxis]
+        g = rgb[:, :, 1].flatten()[:, np.newaxis]
+        b = rgb[:, :, 2].flatten()[:, np.newaxis]
+        pc_rgb = np.concatenate((r, g, b), axis=1)
+    else:
+        pc_rgb = None
+
+    if seg is not None:
+        pc_seg = seg.flatten()[:, np.newaxis]
+    else:
+        pc_seg = None
+
+    return {"xyz": pc_xyz, "rgb": pc_rgb, "seg": pc_seg, "index": index}
+
+
+def depth_and_segmentation_to_point_clouds(
+    depth_image: np.ndarray,
+    segmentation_mask: np.ndarray,
+    fx: float,
+    fy: float,
+    cx: float,
+    cy: float,
+    rgb_image: np.ndarray = None,
+    target_object_id: int = 1,
+    remove_object_from_scene: bool = False,
+) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    """
+    Convert depth image and instance segmentation mask to scene and object point clouds.
+
+    Args:
+        depth_image: HxW depth image in meters
+        segmentation_mask: HxW instance segmentation mask with integer labels
+        fx, fy, cx, cy: Camera intrinsic parameters
+        rgb_image: HxWx3 RGB image (optional, for colored point clouds)
+        target_object_id: ID of the target object in the segmentation mask
+        remove_object_from_scene: If True, removes object points from scene point cloud
+
+    Returns:
+        scene_pc: Nx3 point cloud of the entire scene (excluding object if remove_object_from_scene=True)
+        object_pc: Mx3 point cloud of the target object only
+        scene_colors: Nx3 RGB colors for scene points (or None)
+        object_colors: Mx3 RGB colors for object points (or None)
+
+    Raises:
+        ValueError: If no target object found or multiple objects detected
+    """
+    unique_ids = np.unique(segmentation_mask)
+    if target_object_id not in unique_ids:
+        raise ValueError(
+            f"Target object ID {target_object_id} not found in segmentation mask. Available IDs: {unique_ids}"
+        )
+
+    non_background_ids = unique_ids[unique_ids != 0]
+    if len(non_background_ids) > 1:
+        raise ValueError(
+            "Multiple objects detected in segmentation mask: "
+            f"{non_background_ids}. Please ensure only one object is present."
+        )
+
+    pts_data = depth2points(
+        depth=depth_image,
+        fx=int(fx),
+        fy=int(fy),
+        cx=int(cx),
+        cy=int(cy),
+        rgb=rgb_image,
+        seg=segmentation_mask,
+    )
+
+    xyz = pts_data["xyz"]
+    rgb = pts_data["rgb"]
+    seg = pts_data["seg"]
+    index = pts_data["index"]
+
+    xyz_valid = xyz[index]
+    seg_valid = seg[index] if seg is not None else None
+    rgb_valid = rgb[index] if rgb is not None else None
+
+    scene_pc = xyz_valid
+    scene_colors = rgb_valid
+
+    if seg_valid is not None:
+        object_mask = seg_valid.flatten() == target_object_id
+        object_pc = xyz_valid[object_mask]
+        object_colors = rgb_valid[object_mask] if rgb_valid is not None else None
+
+        if remove_object_from_scene:
+            scene_mask = ~object_mask
+            scene_pc = xyz_valid[scene_mask]
+            scene_colors = rgb_valid[scene_mask] if rgb_valid is not None else None
+            logger.info(
+                "Removed %s object points from scene point cloud",
+                np.sum(object_mask),
+            )
+    else:
+        raise ValueError("Segmentation data not available from depth2points")
+
+    if len(object_pc) == 0:
+        raise ValueError(f"No points found for target object ID {target_object_id}")
+
+    logger.info("Scene point cloud: %s points", len(scene_pc))
+    logger.info("Object point cloud: %s points", len(object_pc))
+
+    return scene_pc, object_pc, scene_colors, object_colors
+
+
+def filter_colliding_grasps(
+    scene_pc: np.ndarray,
+    grasp_poses: np.ndarray,
+    gripper_collision_mesh: trimesh.Trimesh,
+    collision_threshold: float = 0.002,
+    num_collision_samples: int = 2000,
+) -> np.ndarray:
+    """
+    Filter grasps based on collision detection with scene point cloud.
+
+    Args:
+        scene_pc: Nx3 scene point cloud
+        grasp_poses: Kx4x4 array of grasp poses
+        gripper_collision_mesh: Trimesh of gripper collision geometry
+        collision_threshold: Distance threshold for collision detection (meters)
+        num_collision_samples: Number of points to sample from gripper mesh surface
+
+    Returns:
+        collision_mask: K-length boolean array, True if grasp is collision-free
+    """
+    gripper_surface_points, _ = trimesh.sample.sample_surface(
+        gripper_collision_mesh, num_collision_samples
+    )
+    gripper_surface_points = np.array(gripper_surface_points)
+
+    scene_pc = np.asarray(scene_pc, dtype=np.float32)
+    collision_free_mask = []
+
+    logger.info(
+        "Checking collision for %s grasps against %s scene points...",
+        len(grasp_poses),
+        len(scene_pc),
+    )
+
+    for _, grasp_pose in tqdm(
+        enumerate(grasp_poses), total=len(grasp_poses), desc="Collision checking"
+    ):
+        gripper_points_transformed = tra.transform_points(
+            gripper_surface_points, grasp_pose
+        ).astype(np.float32, copy=False)
+
+        min_distances_sq = []
+        batch_size = 100
+        for j in range(0, len(gripper_points_transformed), batch_size):
+            batch_gripper_points = gripper_points_transformed[j : j + batch_size]
+            diff = batch_gripper_points[:, None, :] - scene_pc[None, :, :]
+            dist_sq = np.einsum("ijk,ijk->ij", diff, diff)
+            batch_min_dist_sq = np.min(dist_sq, axis=1)
+            min_distances_sq.append(batch_min_dist_sq)
+
+        all_min_distances_sq = np.concatenate(min_distances_sq, axis=0)
+        collision_detected = np.any(
+            all_min_distances_sq < collision_threshold * collision_threshold
+        )
+        collision_free_mask.append(not bool(collision_detected))
+
+    collision_free_mask = np.array(collision_free_mask)
+    num_collision_free = np.sum(collision_free_mask)
+    logger.info("Found %s/%s collision-free grasps", num_collision_free, len(grasp_poses))
+
+    return collision_free_mask
+
+
+__all__ = [
+    "knn_points",
+    "point_cloud_outlier_removal",
+    "point_cloud_outlier_removal_with_color",
+    "depth2points",
+    "depth_and_segmentation_to_point_clouds",
+    "filter_colliding_grasps",
+]

neuromeka_vfm/segmentation.py

@@ -16,6 +16,7 @@ class Segmentation:
         self.client = PickleClient(hostname, port)
         self.tracking_object_ids = []
         self.current_frame_masks = {}
+        self.invisible_object_ids = []
         self.image_prompt_names = set()
         if compression_strategy in STRATEGIES:
             self.compression_strategy_name = compression_strategy
@@ -51,14 +52,28 @@ class Segmentation:
         else:
             raise ValueError(f"Only valid compression strategies are {list(STRATEGIES.keys())}")
 
+    def set_config(self, config):
+        data = {"operation": "set_config", "config": config}
+        return self.client.send_data(data)
+
+    def get_capabilities(self):
+        data = {"operation": "get_capabilities"}
+        return self.client.send_data(data)
+
+    def get_config(self):
+        data = {"operation": "get_config"}
+        return self.client.send_data(data)
+
     def reset(self):
         self.first_frame_registered = False
         self.tracking_object_ids = []
         self.current_frame_masks = {}
+        self.invisible_object_ids = []
         self.encoder = None
         if self.benchmark:
             self.call_time = {"add_image_prompt": 0, "register_first_frame": 0, "get_next": 0}
             self.call_count = {"add_image_prompt": 0, "register_first_frame": 0, "get_next": 0}
+        self.client.send_data({"operation": "reset"})
 
     def add_image_prompt(self, object_name, object_image):
         if self.benchmark:
@@ -100,6 +115,9 @@ class Segmentation:
                 if np.any(mask):
                     masks[obj_id] = mask
             self.current_frame_masks = masks
+            self.invisible_object_ids = [
+                obj_id for obj_id in self.tracking_object_ids if obj_id not in masks
+            ]
             if self.benchmark:
                 self.call_time["register_first_frame"] += time.time() - start
                 self.call_count["register_first_frame"] += 1
@@ -124,21 +142,57 @@ class Segmentation:
                 if np.any(mask):
                     masks[obj_id] = mask
             self.current_frame_masks = masks
+            self.invisible_object_ids = [
+                obj_id for obj_id in self.tracking_object_ids if obj_id not in masks
+            ]
             if self.benchmark:
                 self.call_time["get_next"] += time.time() - start
                 self.call_count["get_next"] += 1
             return masks
+        if isinstance(response, dict) and any(
+            key in response for key in ("result", "status", "success", "message")
+        ):
+            if self.benchmark:
+                self.call_time["get_next"] += time.time() - start
+                self.call_count["get_next"] += 1
+            return response
         if self.benchmark:
             self.call_time["get_next"] += time.time() - start
             self.call_count["get_next"] += 1
         return None
 
+    def remove_object(self, obj_id, strict=False, need_output=False):
+        if not self.first_frame_registered:
+            print("Segmentation: register_first_frame must be called first")
+            return None
+        data = {
+            "operation": "remove_object",
+            "obj_id": obj_id,
+            "strict": strict,
+            "need_output": need_output,
+        }
+        response = self.client.send_data(data)
+        if self._is_success(response):
+            obj_ids = response.get("data", {}).get("obj_ids")
+            if obj_ids is not None:
+                self.tracking_object_ids = obj_ids
+                self.current_frame_masks = {
+                    obj_id: mask
+                    for obj_id, mask in self.current_frame_masks.items()
+                    if obj_id in obj_ids
+                }
+                self.invisible_object_ids = [
+                    obj_id for obj_id in obj_ids if obj_id not in self.current_frame_masks
+                ]
+        return response
+
     def finish(self):
         if not self.first_frame_registered:
             print("Warning: Segmentation: register_first_frame must be called first")
         self.first_frame_registered = False
         self.tracking_object_ids = []
         self.current_frame_masks = {}
+        self.invisible_object_ids = []
 
     def close(self):
         """Close underlying ZeroMQ socket/context."""

neuromeka_vfm-0.1.6.dist-info/METADATA

@@ -0,0 +1,301 @@
+Metadata-Version: 2.4
+Name: neuromeka_vfm
+Version: 0.1.6
+Summary: Client utilities for Neuromeka VFM FoundationPose RPC (upload meshes, call server)
+Author: Neuromeka
+License: MIT License
+        
+        Copyright (c) 2025 Neuromeka Co., Ltd.
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: pyzmq
+Requires-Dist: paramiko
+Requires-Dist: av
+Requires-Dist: trimesh
+Requires-Dist: tqdm
+Dynamic: license-file
+
+# neuromeka_vfm
+
+A lightweight client SDK for communicating with Segmentation (SAM2, Grounding DINO) and Pose Estimation (NVIDIA FoundationPose) servers over RPC/ZeroMQ. It also provides SSH/SFTP utilities to upload mesh files to the host.
+
+- Website: http://www.neuromeka.com
+- PyPI package: https://pypi.org/project/neuromeka_vfm/
+- Documents: https://docs.neuromeka.com
+
+## Installation
+
+```bash
+pip install neuromeka_vfm
+```
+
+## Python API (usage by example)
+
+- Client PC: the machine running your application with this package installed.
+- Host PC: the machine running Segmentation and Pose Estimation Docker servers. If you run Docker locally, use `localhost`.
+
+### Segmentation
+
+```python
+from neuromeka_vfm import Segmentation
+
+seg = Segmentation(
+    hostname="192.168.10.63",
+    port=5432,
+    compression_strategy="png",    # none | png | jpeg | h264
+)
+
+# Register using an image prompt
+seg.add_image_prompt("drug_box", ref_rgb)
+seg.register_first_frame(
+    frame=first_rgb,
+    prompt="drug_box",     # ID string
+    use_image_prompt=True,
+)
+
+# Register using a text prompt
+seg.register_first_frame(
+    frame=first_rgb,
+    prompt="box .",         # Text prompt (must end with " .")
+    use_image_prompt=False,
+)
+
+# SAM2 tracking on the registered mask(s)
+resp = seg.get_next(next_rgb)
+if isinstance(resp, dict) and resp.get("result") == "ERROR":
+    print(f"Tracking error: {resp.get('message')}")
+    seg.reset()
+else:
+    masks = resp
+
+# Segmentation settings / model selection (nrmk_realtime_segmentation v0.2+)
+caps = seg.get_capabilities()["data"]
+current = seg.get_config()["data"]
+seg.set_config(
+    {
+        "grounding_dino": {
+            "backbone": "Swin-B",        # Swin-T | Swin-B
+            "box_threshold": 0.35,
+            "text_threshold": 0.25,
+        },
+        "dino_detection": {
+            "threshold": 0.5,
+            "target_multiplier": 25,
+            "img_multiplier": 50,
+            "background_threshold": -1.0,
+            "final_erosion_count": 10,
+            "segment_min_size": 20,
+        },
+        "sam2": {
+            "model": "facebook/sam2.1-hiera-large",
+            "use_legacy": False,
+            "compile": False,
+            "offload_state_to_cpu": False,
+            "offload_video_to_cpu": False,
+        },
+    }
+)
+
+# Remove an object (v0.2+, only when use_legacy=False)
+seg.remove_object("cup_0")
+
+seg.close()
+```
+
+Additional Segmentation APIs and behaviors
+
+- `benchmark=True` in the constructor enables timing counters (`call_time`, `call_count`) for `add_image_prompt`, `register_first_frame`, and `get_next`.
+- `switch_compression_strategy()` lets you change the compression strategy at runtime.
+- `register_first_frame()` returns `True`/`False` and raises `ValueError` if image prompts are missing when `use_image_prompt=True`.
+- `register_first_frame()` accepts a list of prompt IDs when `use_image_prompt=True`.
+- `get_next()` returns `None` if called before registration; it can also return the server error dict when available.
+- `reset()` performs a server-side reset, while `finish()` clears only local state.
+- Exposed state: `tracking_object_ids`, `current_frame_masks`, `invisible_object_ids`.
+- Backward-compat alias: `NrmkRealtimeSegmentation`.
+
+#### Segmentation v0.2 config summary (defaults/choices)
+`seg.get_capabilities()` can differ depending on server configuration. The following reflects v0.2 defaults.
+
+```yaml
+grounding_dino:
+  backbone:
+    choices:
+      - Swin-B
+      - Swin-T
+    default: Swin-T
+  box_threshold:
+    default: 0.35
+    min: 0.0
+    max: 1.0
+  text_threshold:
+    default: 0.25
+    min: 0.0
+    max: 1.0
+
+dino_detection:
+  threshold:
+    default: 0.5
+  target_multiplier:
+    default: 25
+  img_multiplier:
+    default: 50
+  background_threshold:
+    default: -1.0
+  final_erosion_count:
+    default: 10
+  segment_min_size:
+    default: 20
+
+sam2:
+  model:
+    choices:
+      - facebook/sam2-hiera-base-plus
+      - facebook/sam2-hiera-large
+      - facebook/sam2-hiera-small
+      - facebook/sam2-hiera-tiny
+      - facebook/sam2.1-hiera-base-plus
+      - facebook/sam2.1-hiera-large
+      - facebook/sam2.1-hiera-small
+      - facebook/sam2.1-hiera-tiny
+    default: facebook/sam2.1-hiera-large
+  use_legacy:
+    default: false
+  compile:
+    default: false
+  offload_state_to_cpu:
+    default: false
+  offload_video_to_cpu:
+    default: false
+```
+
+#### Segmentation v0.2 notes and changes
+
+- If SAM2 VRAM estimation fails, `seg.get_next()` may return `{"result":"ERROR"}`. Handle the error and call `reset` before re-registering.
+- `compile=True` can slow down first-frame registration and `reset`.
+- CPU offloading is most effective when both `offload_state_to_cpu=True` and `offload_video_to_cpu=True` are set (legacy mode does not support `offload_video_to_cpu`).
+- `remove_object` is supported only when `use_legacy=False`.
+- GroundingDINO added the Swin-B backbone and fixed prompt-token merge issues.
+
+### Pose Estimation
+
+**Mesh upload**: Upload the mesh file (STL) to `/opt/meshes/` on the host PC. You can also use SSH directly.
+
+```python
+from neuromeka_vfm import upload_mesh
+
+upload_mesh(
+    host="192.168.10.63",
+    user="user",
+    password="pass",
+    local="mesh/my_mesh.stl",         # local mesh path
+    remote="/opt/meshes/my_mesh.stl", # host mesh path (Docker volume)
+)
+```
+
+Initialization
+
+```python
+from neuromeka_vfm import PoseEstimation
+
+pose = PoseEstimation(host="192.168.10.72", port=5557)
+
+pose.init(
+    mesh_path="/app/modules/foundation_pose/mesh/my_mesh.stl",
+    apply_scale=1.0,
+    track_refine_iter=3,
+    min_n_views=40,
+    inplane_step=60,
+)
+```
+
+- mesh_path: path to the mesh file (STL/OBJ). Initialization fails if missing.
+- apply_scale: scalar applied after loading the mesh.
+  - STL in meters: 1.0 (no scaling)
+  - STL in centimeters: 0.01 (1 cm -> 0.01 m)
+  - STL in millimeters: 0.001 (1 mm -> 0.001 m)
+- force_apply_color: if True, forces a solid color when the mesh lacks color data.
+- apply_color: RGB tuple (0-255) used when `force_apply_color=True`.
+- est_refine_iter: number of refinement iterations during registration (higher = more accurate, slower).
+- track_refine_iter: number of refinement iterations per frame during tracking.
+- min_n_views: minimum number of sampled camera views (affects rotation candidates).
+- inplane_step: in-plane rotation step in degrees (smaller = more candidates).
+
+Registration and tracking
+
+```python
+# Registration (server defaults when iteration is omitted, check_vram=True pre-checks VRAM)
+register_resp = pose.register(rgb=rgb0, depth=depth0, mask=mask0, K=cam_K, check_vram=True)
+
+# Tracking (optionally limit search area with bbox_xywh)
+track_resp = pose.track(rgb=rgb1, depth=depth1, K=cam_K, bbox_xywh=bbox_xywh)
+
+pose.close()
+```
+
+- cam_K: camera intrinsics.
+- Large RGB resolution, large `min_n_views`, or small `inplane_step` can cause GPU VRAM errors.
+- `check_vram=True` in `register` performs a pre-check to prevent server shutdown due to OOM.
+- `iteration` in `register`/`track` can override the server default if provided.
+- `reset()` resets the server state; `reset_object()` reuses the cached mesh to rebuild the rotation grid.
+- Default host/port can come from `FPOSE_HOST` and `FPOSE_PORT` environment variables.
+- Backward-compat alias: `FoundationPoseClient`.
+
+<!--
+## Benchmark
+
+Measured on local servers. Empty cells are not yet measured.
+
+**RTX 5060**
+| Task | Prompt | None (s) | JPEG (s) | PNG (s) | h264 (s) |
+| --- | --- | --- | --- | --- | --- |
+| Grounding DINO | text (human . cup .) | 0.86 | 0.35 | 0.50 | 0.52 |
+| DINOv2 | image prompt | 0.85 | 0.49 | 0.65 | 0.63 |
+| SAM2 | - |  |  |  |  |
+| FoundationPose registration | - |  |  |  |  |
+| FoundationPose track | - |  |  |  |  |
+
+**RTX 5090**
+| Task | Prompt | None (s) | JPEG (s) | PNG (s) | h264 (s) |
+| --- | --- | --- | --- | --- | --- |
+| Grounding DINO | text (human . cup .) |  |  |  |  |
+| DINOv2 | image prompt |  |  |  |  |
+| SAM2 | - |  |  |  |  |
+| FoundationPose registration | - | 0.4 | - |  |  |
+| FoundationPose track | - | 0.03 |  |  |  |
+-->
+
+## Release notes
+
+- 0.1.2: Improved success detection for Segmentation responses (`result`/`success`/`status`), fixed image prompt registration/usage, added `check_vram` to PoseEstimation `register`.
+- 0.1.1: Improved resource cleanup in PoseEstimation/Segmentation, use server defaults when iteration is omitted, added pose demo example.
+- 0.1.0: Initial public release. Includes FoundationPose RPC client, real-time segmentation client, SSH-based mesh upload CLI/API.

{neuromeka_vfm-0.1.4.dist-info → neuromeka_vfm-0.1.6.dist-info}/RECORD

@@ -1,14 +1,16 @@
 neuromeka_vfm/__init__.py,sha256=h5ODdWFgN7a9TBzF6Qfdyx5VxUr2hG0pFTwq57jEvDo,422
 neuromeka_vfm/compression.py,sha256=d2xOz4XBJZ60pPSXwQ5LPYwhpsaNORvNoY_0CUiAvt0,5191
+neuromeka_vfm/generate_mesh.py,sha256=HV2dUfVXROPQ9kDDPmkwdn5E5gelWIQDsPsZuvZxI6E,3634
 neuromeka_vfm/pickle_client.py,sha256=Iw2fpxdnKB20oEUgsd0rJlvzOd5JhetphpKkF9qQcX0,591
+neuromeka_vfm/point_cloud_utils.py,sha256=ZnCh8Xg6pLGoyi5ufZkz59HzE9RuRdihE8z-XNYT1PA,13261
 neuromeka_vfm/pose_estimation.py,sha256=3MUVhL0nMcpHApZDAzutS7fINPHcb-tu_WoXvNGU33E,2625
-neuromeka_vfm/segmentation.py,sha256=wae0_m225DUMD8Nm2A7iQm49QWeUas17B8PaoGGFt5w,6311
+neuromeka_vfm/segmentation.py,sha256=8kmMut_gNJ3wa9F0l7iEYFNqHJzHJ5KPBzs7vSiwjqg,8464
 neuromeka_vfm/upload_mesh.py,sha256=aW5G9aE5OeiDN5pEVKDzMeV538U-I2iRYZvVZTfGsr4,2728
 neuromeka_vfm/examples/__init__.py,sha256=dEhb0FqhpEGNmg0pMunmrTlViIcxvd95fYEjZ49IOTQ,37
 neuromeka_vfm/examples/pose_demo.py,sha256=zq1Z0_kxQc4CB-ltfwm_oMoC7JLoN5GyeE3C6jKGQKw,13658
-neuromeka_vfm-0.1.4.dist-info/licenses/LICENSE,sha256=40cBWxFahhu0p_EB0GhU8oVIifVNmH1o2fZtx0bIif8,1076
-neuromeka_vfm-0.1.4.dist-info/METADATA,sha256=EPZ5Y6uuSz00whHesar6bKAqrUDNpGAfquZq9Pvd24U,7829
-neuromeka_vfm-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-neuromeka_vfm-0.1.4.dist-info/entry_points.txt,sha256=Wl4XqiUt_GLQ08oTJtsYjLW0iYxZ52ysVd1-cN0kYP4,72
-neuromeka_vfm-0.1.4.dist-info/top_level.txt,sha256=uAH_yXikUvxXTSEnUC0M8Zl5ggxbnkYtXlmTfEG8MUk,14
-neuromeka_vfm-0.1.4.dist-info/RECORD,,
+neuromeka_vfm-0.1.6.dist-info/licenses/LICENSE,sha256=40cBWxFahhu0p_EB0GhU8oVIifVNmH1o2fZtx0bIif8,1076
+neuromeka_vfm-0.1.6.dist-info/METADATA,sha256=zHBuQ5sHDJjOdY96dmxLiwtE8OHIRJg0VcyzUB0rV48,10628
+neuromeka_vfm-0.1.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+neuromeka_vfm-0.1.6.dist-info/entry_points.txt,sha256=Wl4XqiUt_GLQ08oTJtsYjLW0iYxZ52ysVd1-cN0kYP4,72
+neuromeka_vfm-0.1.6.dist-info/top_level.txt,sha256=uAH_yXikUvxXTSEnUC0M8Zl5ggxbnkYtXlmTfEG8MUk,14
+neuromeka_vfm-0.1.6.dist-info/RECORD,,

{neuromeka_vfm-0.1.4.dist-info → neuromeka_vfm-0.1.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

neuromeka_vfm-0.1.4.dist-info/METADATA

@@ -1,186 +0,0 @@
-Metadata-Version: 2.4
-Name: neuromeka_vfm
-Version: 0.1.4
-Summary: Client utilities for Neuromeka VFM FoundationPose RPC (upload meshes, call server)
-Author: Neuromeka
-License: MIT License
-        
-        Copyright (c) 2025 Neuromeka Co., Ltd.
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
-        
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: numpy
-Requires-Dist: pyzmq
-Requires-Dist: paramiko
-Requires-Dist: av
-Dynamic: license-file
-
-# neuromeka_vfm
-
-클라이언트 PC에서 Segmentation (SAM2, Grounding DINO), Pose Estimation (NVIDIA FoundationPose) 서버(RPC, ZeroMQ)와 통신하고, SSH/SFTP로 호스트에 mesh를 업로드하는 간단한 유틸 패키지입니다.
-
-- Website: http://www.neuromeka.com
-- Source code: https://github.com/neuromeka-robotics/neuromeka_vfm
-- PyPI package: https://pypi.org/project/neuromeka_vfm/
-- Documents: https://docs.neuromeka.com
-
-## Web UI (VFM Tester)를 통해 사용 가능
-
-- VFM Tester (Web UI): https://gitlab.com/neuromeka-group/nrmkq/nrmk_vfm_tester
-
-
-## Installation
-```bash
-pip install neuromeka_vfm
-```
-
-## Python API (예제로 보는 사용법)
-
-* 내 PC: 어플리케이션을 구현하고 이 패키지 (neuromeka_vfm)이 설치된 PC
-* 서버PC (Host): Segmentation, Pose Estimation 도커 서버가 설치된 PC. 내 PC에 도커를 설치할 경우 localhost 사용.
-
-### Segmentation
-```python
-from neuromeka_vfm import Segmentation
-
-seg = Segmentation(
-    hostname="192.168.10.63", 
-    port=5432,
-    compression_strategy="png",    # none | png | jpeg | h264
-)
-
-# Image Prompt를 이용한 등록
-seg.add_image_prompt("drug_box", ref_rgb)
-seg.register_first_frame(frame=first_rgb, 
-                        prompt="drug_box", # ID str
-                        use_image_prompt=True)
-
-# Text Prompt를 이용한 등록
-seg.register_first_frame(frame=first_rgb, 
-                        prompt="box .",   # Text prompt (끝에 띄어쓰기 . 필수)
-                        use_image_prompt=False)
-
-# 등록된 mask에 대한 SAM2 tracking
-masks = seg.get_next(next_rgb)
-
-
-seg.close()
-```
-
-### Pose Estimation
-
-**Mesh 파일 업로드**: 등록/인식하고자 하는 mesh 파일 (stl)을 호스트PC의 '/opt/meshes/' 경로에 업로드 (직접 SSH 통해 파일을 옮겨도 됨)
-```python
-from neuromeka_vfm import upload_mesh
-upload_mesh(
-    host="192.168.10.63",
-    user="user",
-    password="pass",                  
-    local="mesh/my_mesh.stl",         # 내 PC mesh 경로
-    remote="/opt/meshes/my_mesh.stl", # 호스트PC mesh 경로 (도커 볼륨마운트)
-)
-```
-
-초기화
-```python
-from neuromeka_vfm import PoseEstimation
-
-pose = PoseEstimation(host="192.168.10.72", port=5557)  
-
-pose.init(
-    mesh_path="/app/modules/foundation_pose/mesh/my_mesh.stl",
-    apply_scale=1.0,  
-    track_refine_iter=3,
-    min_n_views=40,
-    inplane_step=60
-)
-```
-- mesh_path: 사용할 물체 메시 파일(STL/OBJ 등) 경로. 없으면 초기화 실패.
-- apply_scale: 메시를 로드한 뒤 전체를 배율 조정하는 스케일 값. 단위 없는 곱셈 계수.
-    - STL 모델이 미터 단위라면 1.0 (스케일 없음)
-    - STL 모델이 센티미터 단위라면 0.01 (1 cm → 0.01 m)
-    - STL 모델이 밀리미터 단위라면 0.001 (1 mm → 0.001 m)
-- force_apply_color: True일 때 메시에 단색 텍스처를 강제로 입힘. 메시가 색상을 안 가졌을 때 시각화 안정성을 위해 사용.
-- apply_color: force_apply_color가 True일 때 적용할 RGB 색상값(0~255) 튜플.
-- est_refine_iter: 초기 등록(register) 단계에서 포즈를 반복 정련하는 횟수. 값이 클수록 정확도 ↑, 연산 시간 ↑.
-- track_refine_iter: 추적(track) 단계에서 한 프레임당 포즈 정련 반복 횟수.
-- min_n_views: 초기 뷰 샘플링 시 생성할 최소 카메라 뷰 수(회전 후보 수에 영향).
-- inplane_step: in-plane 회전 샘플링 간격(도 단위). 값이 작을수록 더 많은 회전 후보를 생성.
-
-
-인식 및 추적
-```python
-# 초기 등록 (iteration 생략 시 서버 기본값, check_vram=True로 VRAM 사전 체크)
-register_resp = pose.register(rgb=rgb0, depth=depth0, mask=mask0, K=cam_K, check_vram=True)
-
-# 추적 (bbox_xywh로 탐색 범위 제한 가능)
-track_resp = pose.track(rgb=rgb1, depth=depth1, K=cam_K, bbox_xywh=bbox_xywh)
-pose.close()
-```
-- cam_K: camera intrinsic
-- RGB resolution이 크거나, min_n_views 값이 크거나, inplane_step이 작을 경우 GPU VRAM 초과 에러 발생. 
-- register check_vram=True 일 경우 VRAM 초과 사전 검사하여 shutdown 방지.
-
-
-## VFM (Vision Foundation Model) latency benchmark
-로컬 서버 구동 시 측정. 빈칸은 아직 미측정 항목입니다. 
-
-**RTX 5060**
-| Task | Prompt | None (s) | JPEG (s) | PNG (s) | h264 (s) |
-| --- | --- | --- | --- | --- | --- |
-| Grounding DINO | text (human . cup .) | 0.86 | 0.35 | 0.50 | 0.52 |
-| DINOv2 | image prompt | 0.85 | 0.49 | 0.65 | 0.63 |
-| SAM2 | - |  |  |  |  |
-| FoundationPose registration | - |  |  |  |  |
-| FoundationPose track | - |  |  |  |  |
-
-**RTX 5090**
-| Task | Prompt | None (s) | JPEG (s) | PNG (s) | h264 (s) |
-| --- | --- | --- | --- | --- | --- |
-| Grounding DINO | text (human . cup .) |  |  |  |  |
-| DINOv2 | image prompt |  |  |  |  |
-| SAM2 | - |  |  |  |  |
-| FoundationPose registration | - | 0.4 | - |  |  |
-| FoundationPose track | - | 0.03 |  |  |  |
-
-**Jetson Orin**
-| Task | Prompt | None (s) | JPEG (s) | PNG (s) | h264 (s) |
-| --- | --- | --- | --- | --- | --- |
-| Grounding DINO | text (human . cup .) |  |  |  |  |
-| DINOv2 | image prompt |  |  |  |  |
-| SAM2 | - |  |  |  |  |
-| FoundationPose registration | - | 0.4 | - |  |  |
-| FoundationPose track | - | 0.03 |  |  |  |
-
-## 릴리스 노트
-- 0.1.2: Segmentation 응답 성공 판정 개선(`result`/`success`/`status` 모두 지원), image prompt 등록/사용 오류 수정, PoseEstimation `register`에 `check_vram` 옵션 반영.
-- 0.1.1: PoseEstimation/Segmentation에서 리소스 정리 개선, iteration 미전달 시 서버 기본값 사용, pose 데모 예제 추가.
-- 0.1.0: 초기 공개 버전. FoundationPose RPC 클라이언트, 실시간 세그멘테이션 클라이언트, SSH 기반 mesh 업로드 CLI/API 포함.