stereo-charuco-pipeline 0.1.0__py3-none-any.whl

recorder/__init__.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Optional
+
+from .config import RecorderConfig
+from .ffmpeg import FFmpegRunner, python_executable
+
+# Calibration UI exports
+from .calibration_ui import CalibrationUI, CalibConfig
+
+# Pipeline exports
+from .auto_calibrate import AutoCalibConfig, run_auto_calibration, run_auto_reconstruction
+from .pipeline_ui import PipelineUI
+
+# Project management exports
+from .project_manager import ProjectManagerUI, ProjectContext
+
+# Smart recorder exports
+from .smart_recorder import SmartRecorder, SmartRecorderConfig
+
+
+EventCallback = Callable[[dict], None]
+
+
+@dataclass
+class PipelineResult:
+    returncode: int
+    script_path: str
+    session_hint: str
+    cmd: list[str]
+
+
+def _project_root() -> Path:
+    """Find the calib_record_tool root directory.
+
+    Works in dev mode (running from repo) and after pip install
+    (falls back to cwd).
+    """
+    from .paths import _dev_root
+    dev = _dev_root()
+    if dev:
+        return dev
+    return Path.cwd()
+
+
+def run_pipeline(cfg: RecorderConfig, on_event: Optional[EventCallback] = None) -> PipelineResult:
+    """
+    Stable API entrypoint. Runs the existing scripts/record_and_split.py pipeline.
+
+    on_event receives dicts like:
+      {"stage": "run", "message": "..."}
+      {"stage": "log", "message": "..."}
+      {"stage": "done", "message": "..."}
+      {"stage": "error", "message": "..."}
+    """
+    root = _project_root()
+    script = root / "scripts" / "record_and_split.py"
+    if not script.exists():
+        raise FileNotFoundError(f"Pipeline script not found: {script}")
+
+    runner = FFmpegRunner()
+
+    cmd = [python_executable(), str(script)] + cfg.to_args()
+
+    def emit(stage: str, message: str) -> None:
+        if on_event:
+            on_event({"stage": stage, "message": message})
+
+    emit("run", "Starting pipeline...")
+    emit("run", f"CMD: {' '.join(cmd)}")
+
+    # Stream output for UI
+    rc = runner.run_stream(cmd, on_log=lambda line: emit("log", line), cwd=str(root))
+
+    if rc == 0:
+        emit("done", "Pipeline finished successfully.")
+    else:
+        emit("error", f"Pipeline failed. returncode={rc}")
+
+    # session_hint：你现在脚本里 session 目录可能由 outroot 或默认规则决定，这里先给一个“提示”
+    session_hint = cfg.outroot if cfg.outroot else str(Path(cfg.outdir).resolve())
+
+    return PipelineResult(
+        returncode=rc,
+        script_path=str(script),
+        session_hint=session_hint,
+        cmd=cmd,
+    )

recorder/auto_calibrate.py

@@ -0,0 +1,493 @@
+"""Headless auto-calibration pipeline for caliscope.
+
+Calls caliscope's core APIs directly (no GUI) to run the full
+calibration workflow: intrinsic calibration, 2D extraction,
+extrinsic calibration (bundle adjustment), and origin alignment.
+
+Usage:
+  from recorder.auto_calibrate import AutoCalibConfig, run_auto_calibration
+  config = AutoCalibConfig(project_dir=Path("path/to/caliscope_project"))
+  result = run_auto_calibration(config, on_progress=print)
+"""
+
+from __future__ import annotations
+
+import logging
+from copy import deepcopy
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable, Optional
+
+import numpy as np
+import pandas as pd
+
+from caliscope.cameras.camera_array import CameraArray, CameraData
+from caliscope.core.calibrate_intrinsics import run_intrinsic_calibration
+from caliscope.core.charuco import Charuco
+from caliscope.core.point_data import ImagePoints
+from caliscope.core.point_data_bundle import PointDataBundle
+from caliscope.core.bootstrap_pose.build_paired_pose_network import build_paired_pose_network
+from caliscope.core.process_synchronized_recording import process_synchronized_recording
+from caliscope.persistence import (
+  save_camera_array,
+  save_charuco,
+  save_image_points_csv,
+)
+from caliscope.recording.frame_source import FrameSource
+from caliscope.repositories.point_data_bundle_repository import PointDataBundleRepository
+from caliscope.trackers.charuco_tracker import CharucoTracker
+
+logger = logging.getLogger(__name__)
+
+# Type alias for progress callbacks
+# (stage_name, message, percent_0_to_100)
+ProgressCallback = Callable[[str, str, int], None]
+
+FILTERED_FRACTION = 0.025  # 2.5% outlier removal
+
+
+@dataclass
+class AutoCalibConfig:
+  """Configuration for headless auto-calibration pipeline."""
+  project_dir: Path
+  # Charuco board parameters (must match the physical board)
+  charuco_columns: int = 10
+  charuco_rows: int = 16
+  board_height_cm: float = 80.0
+  board_width_cm: float = 50.0
+  dictionary: str = "DICT_4X4_100"
+  aruco_scale: float = 0.7
+  square_size_cm: float = 5.0
+  units: str = "cm"
+  # Camera setup
+  ports: list[int] = field(default_factory=lambda: [1, 2])
+  image_size: tuple[int, int] = (1600, 1200)
+  # Processing parameters
+  intrinsic_subsample: int = 10  # process every Nth frame for intrinsic
+  extrinsic_subsample: int = 3   # process every Nth sync index for extrinsic
+
+  @classmethod
+  def from_yaml(cls, yaml_data: dict, project_dir: Path) -> "AutoCalibConfig":
+    """Create config from YAML dictionary (default.yaml charuco section)."""
+    charuco = yaml_data.get("charuco", {})
+    split = yaml_data.get("split", {})
+    # Derive image size from split config
+    xsplit = split.get("xsplit", 1600)
+    full_h = split.get("full_h", 1200)
+    return cls(
+      project_dir=project_dir,
+      charuco_columns=charuco.get("columns", 10),
+      charuco_rows=charuco.get("rows", 16),
+      board_height_cm=charuco.get("board_height_cm", 80.0),
+      board_width_cm=charuco.get("board_width_cm", 50.0),
+      dictionary=charuco.get("dictionary", "DICT_4X4_100"),
+      aruco_scale=charuco.get("aruco_scale", 0.7),
+      square_size_cm=charuco.get("square_size_cm", 5.0),
+      image_size=(xsplit, full_h),
+    )
+
+
+@dataclass
+class CalibrationResult:
+  """Result of the auto-calibration pipeline."""
+  camera_array: CameraArray
+  bundle: PointDataBundle
+  origin_sync_index: int
+  intrinsic_rmse: dict[int, float]  # port -> RMSE
+  extrinsic_cost: float  # final_cost from bundle adjustment
+  success: bool = True
+  error_message: str = ""
+
+
+def _emit(on_progress: Optional[ProgressCallback], stage: str, msg: str, pct: int):
+  """Helper to safely emit progress."""
+  if on_progress:
+    on_progress(stage, msg, pct)
+  logger.info(f"[{stage}] {msg} ({pct}%)")
+
+
+def _collect_charuco_points_from_video(
+  video_dir: Path,
+  port: int,
+  tracker: CharucoTracker,
+  subsample: int = 10,
+  on_progress: Optional[ProgressCallback] = None,
+) -> list[tuple[int, object]]:
+  """Iterate video frames and collect charuco corner detections.
+
+  Returns list of (frame_index, PointPacket) tuples.
+  """
+  source = FrameSource(video_dir, port)
+  collected = []
+  total_frames = source.frame_count
+
+  try:
+    for frame_idx in range(0, total_frames, subsample):
+      frame = source.get_frame(frame_idx)
+      if frame is None:
+        continue
+
+      points = tracker.get_points(frame, port, 0)
+      if points is not None and len(points.point_id) > 0:
+        collected.append((frame_idx, points))
+
+      if on_progress and frame_idx % (subsample * 20) == 0:
+        pct = int(frame_idx / total_frames * 100)
+        _emit(on_progress, "intrinsic", f"Port {port}: scanning frame {frame_idx}/{total_frames}", pct)
+  finally:
+    source.close()
+
+  logger.info(f"Port {port}: collected {len(collected)} frames with charuco detections out of {total_frames // subsample} sampled")
+  return collected
+
+
+def _build_image_points_from_packets(
+  collected_points: list[tuple[int, object]],
+  port: int,
+) -> ImagePoints:
+  """Convert (frame_index, PointPacket) list to ImagePoints DataFrame.
+
+  Replicates the pattern from IntrinsicCalibrationPresenter._build_image_points().
+  """
+  rows = []
+  for frame_index, points in collected_points:
+    point_count = len(points.point_id)
+    if point_count == 0:
+      continue
+
+    obj_loc_x, obj_loc_y, obj_loc_z = points.obj_loc_list
+
+    for i in range(point_count):
+      rows.append({
+        "sync_index": frame_index,
+        "port": port,
+        "frame_index": frame_index,
+        "frame_time": 0.0,
+        "point_id": int(points.point_id[i]),
+        "img_loc_x": float(points.img_loc[i, 0]),
+        "img_loc_y": float(points.img_loc[i, 1]),
+        "obj_loc_x": obj_loc_x[i],
+        "obj_loc_y": obj_loc_y[i],
+        "obj_loc_z": obj_loc_z[i],
+      })
+
+  if not rows:
+    df = pd.DataFrame(columns=[
+      "sync_index", "port", "frame_index", "frame_time",
+      "point_id", "img_loc_x", "img_loc_y",
+      "obj_loc_x", "obj_loc_y", "obj_loc_z",
+    ])
+    return ImagePoints(df)
+
+  df = pd.DataFrame(rows)
+  return ImagePoints(df)
+
+
+def _generate_frame_timestamps(
+  recording_dir: Path,
+  ports: list[int],
+) -> None:
+  """Generate synthetic frame_timestamps.csv for perfectly synchronized stereo videos.
+
+  Since both views come from a single stereo USB camera,
+  frames are inherently synchronized. We create a CSV where
+  both ports share identical timestamps based on video FPS.
+  """
+  rows = []
+  for port in ports:
+    source = FrameSource(recording_dir, port)
+    fps = source.fps
+    frame_count = source.frame_count
+    source.close()
+
+    for i in range(frame_count):
+      rows.append({"port": port, "frame_time": i / fps})
+
+  df = pd.DataFrame(rows)
+  csv_path = recording_dir / "frame_timestamps.csv"
+  df.to_csv(csv_path, index=False)
+  logger.info(f"Generated frame_timestamps.csv at {csv_path} ({len(rows)} rows, {len(ports)} ports)")
+
+
+def _find_best_origin_sync_index(bundle: PointDataBundle) -> int:
+  """Find the sync_index with the most world points for origin alignment.
+
+  Picks the frame where the charuco board is most visible across all cameras.
+  """
+  world_df = bundle.world_points.df
+  counts = world_df.groupby("sync_index").size()
+  best_sync_index = int(counts.idxmax())
+  logger.info(f"Selected sync_index {best_sync_index} for origin alignment ({counts[best_sync_index]} world points)")
+  return best_sync_index
+
+
+def run_auto_calibration(
+  config: AutoCalibConfig,
+  on_progress: Optional[ProgressCallback] = None,
+) -> CalibrationResult:
+  """Execute complete auto-calibration pipeline.
+
+  Steps:
+    1. Create charuco board and tracker
+    2. Intrinsic calibration for each camera
+    3. Generate frame_timestamps.csv for extrinsic videos
+    4. 2D extraction from extrinsic videos
+    5. Bootstrap extrinsic poses (PnP)
+    6. Bundle adjustment optimization
+    7. Outlier filtering and re-optimization
+    8. Origin alignment
+    9. Save all results
+
+  Args:
+    config: Pipeline configuration
+    on_progress: Optional callback for progress updates
+
+  Returns:
+    CalibrationResult with calibrated camera array and bundle
+  """
+  project = config.project_dir
+  intrinsic_dir = project / "calibration" / "intrinsic"
+  extrinsic_dir = project / "calibration" / "extrinsic"
+  charuco_dir = extrinsic_dir / "CHARUCO"
+  charuco_dir.mkdir(parents=True, exist_ok=True)
+
+  intrinsic_rmse = {}
+
+  try:
+    # ── Step 1: Create charuco and tracker ──────────────────────────
+    _emit(on_progress, "init", "Creating charuco board and tracker...", 0)
+
+    charuco = Charuco(
+      columns=config.charuco_columns,
+      rows=config.charuco_rows,
+      board_height=config.board_height_cm,
+      board_width=config.board_width_cm,
+      dictionary=config.dictionary,
+      units=config.units,
+      aruco_scale=config.aruco_scale,
+      square_size_overide_cm=config.square_size_cm,
+    )
+    tracker = CharucoTracker(charuco)
+
+    # Save charuco.toml
+    save_charuco(charuco, project / "charuco.toml")
+    _emit(on_progress, "init", "Charuco board configured", 5)
+
+    # ── Step 2: Build initial camera array ──────────────────────────
+    cameras = {}
+    for port in config.ports:
+      cameras[port] = CameraData(
+        port=port,
+        size=config.image_size,
+        rotation_count=0,
+      )
+    camera_array = CameraArray(cameras)
+
+    # ── Step 3: Intrinsic calibration ───────────────────────────────
+    total_ports = len(config.ports)
+    for idx, port in enumerate(config.ports):
+      _emit(on_progress, "intrinsic",
+            f"Calibrating camera {port} ({idx + 1}/{total_ports})...",
+            10 + idx * 15)
+
+      # Verify video exists
+      video_path = intrinsic_dir / f"port_{port}.mp4"
+      if not video_path.exists():
+        raise FileNotFoundError(f"Intrinsic video not found: {video_path}")
+
+      # Collect charuco points from video
+      collected = _collect_charuco_points_from_video(
+        intrinsic_dir, port, tracker,
+        subsample=config.intrinsic_subsample,
+        on_progress=on_progress,
+      )
+
+      if not collected:
+        raise ValueError(f"No charuco corners detected in intrinsic video for port {port}")
+
+      # Build ImagePoints from collected packets
+      image_points = _build_image_points_from_packets(collected, port)
+
+      # Run intrinsic calibration
+      camera = camera_array.cameras[port]
+      output = run_intrinsic_calibration(camera, image_points)
+
+      # Update camera array
+      camera_array.cameras[port] = output.camera
+      intrinsic_rmse[port] = output.report.rmse
+
+      _emit(on_progress, "intrinsic",
+            f"Camera {port} calibrated: RMSE={output.report.rmse:.3f}px, "
+            f"frames={output.report.frames_used}",
+            10 + (idx + 1) * 15)
+
+    # Save intermediate camera array
+    save_camera_array(camera_array, project / "camera_array.toml")
+    _emit(on_progress, "intrinsic", "All intrinsic calibrations complete", 40)
+
+    # ── Step 4: Generate frame_timestamps.csv ───────────────────────
+    _emit(on_progress, "extrinsic_2d", "Generating synchronization timestamps...", 42)
+    _generate_frame_timestamps(extrinsic_dir, config.ports)
+
+    # ── Step 5: 2D extraction from extrinsic videos ─────────────────
+    _emit(on_progress, "extrinsic_2d", "Extracting 2D charuco points from extrinsic videos...", 45)
+
+    image_points = process_synchronized_recording(
+      recording_dir=extrinsic_dir,
+      cameras=camera_array.cameras,
+      tracker=tracker,
+      subsample=config.extrinsic_subsample,
+      on_progress=lambda cur, total: _emit(
+        on_progress, "extrinsic_2d",
+        f"Processing sync index {cur}/{total}",
+        45 + int(cur / total * 15),
+      ),
+    )
+
+    # Save image points
+    save_image_points_csv(image_points, charuco_dir / "image_points.csv")
+    _emit(on_progress, "extrinsic_2d", f"2D extraction complete: {len(image_points.df)} observations", 60)
+
+    # ── Step 6: Bootstrap extrinsic poses ───────────────────────────
+    _emit(on_progress, "extrinsic_3d", "Bootstrapping camera poses (PnP)...", 62)
+
+    pose_network = build_paired_pose_network(image_points, camera_array, method="pnp")
+    pose_network.apply_to(camera_array)
+
+    # ── Step 7: Triangulate initial 3D points ───────────────────────
+    _emit(on_progress, "extrinsic_3d", "Triangulating 3D points...", 65)
+
+    world_points = image_points.triangulate(camera_array)
+
+    # ── Step 8: Bundle adjustment ───────────────────────────────────
+    _emit(on_progress, "extrinsic_3d", "Running bundle adjustment (first pass)...", 70)
+
+    bundle = PointDataBundle(camera_array, image_points, world_points)
+    optimized = bundle.optimize(ftol=1e-8, verbose=0)
+
+    initial_cost = optimized.optimization_status.final_cost if optimized.optimization_status else 0.0
+    _emit(on_progress, "extrinsic_3d",
+          f"First pass cost: {initial_cost:.4f}", 75)
+
+    # ── Step 9: Filter outliers and re-optimize ─────────────────────
+    _emit(on_progress, "extrinsic_3d", "Filtering outliers and re-optimizing...", 78)
+
+    filtered = optimized.filter_by_percentile_error(FILTERED_FRACTION * 100)
+    final_bundle = filtered.optimize(ftol=1e-8, verbose=0)
+
+    final_cost = final_bundle.optimization_status.final_cost if final_bundle.optimization_status else 0.0
+    _emit(on_progress, "extrinsic_3d",
+          f"Final cost: {final_cost:.4f}", 85)
+
+    # ── Step 10: Origin alignment ───────────────────────────────────
+    _emit(on_progress, "extrinsic_3d", "Aligning coordinate origin...", 88)
+
+    origin_sync_index = _find_best_origin_sync_index(final_bundle)
+    aligned_bundle = final_bundle.align_to_object(origin_sync_index)
+
+    # ── Step 11: Save everything ────────────────────────────────────
+    _emit(on_progress, "save", "Saving calibration results...", 92)
+
+    # Save final camera array to project root
+    save_camera_array(aligned_bundle.camera_array, project / "camera_array.toml")
+
+    # Save bundle to extrinsic/CHARUCO/
+    bundle_repo = PointDataBundleRepository(charuco_dir)
+    bundle_repo.save(aligned_bundle)
+
+    # Save project settings
+    _save_project_settings(project)
+
+    _emit(on_progress, "done", "Calibration complete!", 100)
+
+    return CalibrationResult(
+      camera_array=aligned_bundle.camera_array,
+      bundle=aligned_bundle,
+      origin_sync_index=origin_sync_index,
+      intrinsic_rmse=intrinsic_rmse,
+      extrinsic_cost=final_cost,
+      success=True,
+    )
+
+  except Exception as e:
+    logger.error(f"Auto-calibration failed: {e}", exc_info=True)
+    _emit(on_progress, "error", f"Calibration failed: {e}", -1)
+    return CalibrationResult(
+      camera_array=CameraArray({}),
+      bundle=None,
+      origin_sync_index=-1,
+      intrinsic_rmse=intrinsic_rmse,
+      extrinsic_cost=0.0,
+      success=False,
+      error_message=str(e),
+    )
+
+
+def _save_project_settings(project_dir: Path):
+  """Save minimal project settings for caliscope compatibility."""
+  import rtoml
+  from datetime import datetime
+
+  settings_path = project_dir / "project_settings.toml"
+  settings = {
+    "creation_date": datetime.now().isoformat(),
+    "save_tracked_points_video": True,
+    "fps_sync_stream_processing": 100,
+  }
+
+  # Only create if doesn't exist (don't overwrite user settings)
+  if not settings_path.exists():
+    settings_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(settings_path, "w") as f:
+      rtoml.dump(settings, f)
+    logger.info(f"Created project settings at {settings_path}")
+
+
+def run_auto_reconstruction(
+  project_dir: Path,
+  recording_name: str,
+  tracker_name: str = "HOLISTIC",
+  on_progress: Optional[ProgressCallback] = None,
+) -> Path:
+  """Run 3D reconstruction on a recording session.
+
+  Args:
+    project_dir: Path to caliscope project directory
+    recording_name: Name of recording subdirectory under recordings/
+    tracker_name: Tracker to use (HOLISTIC, POSE, HAND, etc.)
+    on_progress: Optional callback for progress updates
+
+  Returns:
+    Path to output xyz CSV file
+  """
+  from caliscope.persistence import load_camera_array
+  from caliscope.reconstruction.reconstructor import Reconstructor
+  from caliscope.trackers.tracker_enum import TrackerEnum
+
+  _emit(on_progress, "reconstruction", "Loading calibration data...", 0)
+
+  camera_array = load_camera_array(project_dir / "camera_array.toml")
+  recording_path = project_dir / "recordings" / recording_name
+  tracker_enum = TrackerEnum[tracker_name]
+
+  if not recording_path.exists():
+    raise FileNotFoundError(f"Recording not found: {recording_path}")
+
+  _emit(on_progress, "reconstruction", "Starting 2D landmark detection...", 5)
+
+  reconstructor = Reconstructor(camera_array, recording_path, tracker_enum)
+
+  # Stage 1: 2D detection
+  completed = reconstructor.create_xy(include_video=False)
+  if not completed:
+    raise RuntimeError("2D landmark detection was cancelled or failed")
+
+  _emit(on_progress, "reconstruction", "Starting 3D triangulation...", 80)
+
+  # Stage 2: 3D triangulation
+  reconstructor.create_xyz()
+
+  output_path = recording_path / tracker_enum.name / f"xyz_{tracker_enum.name}.csv"
+  _emit(on_progress, "reconstruction", f"Reconstruction complete: {output_path}", 100)
+
+  return output_path