robopark 2.8.35 → 3.0.0

package/vision/app_pi_clean.py

@@ -1,6 +1,7 @@
 from flask import Flask, Response, jsonify, request
 from flask_cors import CORS
 import argparse
+import glob
 import cv2
 import os
 import sys
@@ -9,13 +10,40 @@ import numpy as np
 import threading
 import requests
 import base64
+import struct
 
 app = Flask(__name__)
 CORS(app)
 
-# Camera management
-current_camera_index = 0
+# Camera management. Production installs create /dev/robopark-camera from
+# the primary USB capture interface, avoiding /dev/videoN renumbering.
+def _normalize_camera_device(value):
+    configured = str(value or "").strip()
+    if configured.lower() in ("", "auto", "default", "first"):
+        if sys.platform.startswith("linux"):
+            return "/dev/robopark-camera" if os.path.exists("/dev/robopark-camera") else "/dev/video0"
+        return 0
+    if configured.isdigit():
+        return int(configured)
+    return configured
+
+
+_configured_camera = os.getenv("ROBOPARK_CAMERA_DEVICE", "")
+current_camera_index = _normalize_camera_device(_configured_camera)
 camera = None
+camera_lock = threading.Lock()
+frame_condition = threading.Condition()
+latest_frame_bytes = None
+latest_frame_sequence = 0
+camera_worker_started = False
+camera_read_started_at = 0.0
+camera_last_frame_at = 0.0
+audio_input_device = "default"
+audio_output_device = "default"
+ROBOVISION_AUDIO_URL = os.getenv("ROBOVISION_AUDIO_URL", "http://127.0.0.1:8000")
+_camera_inventory_cache = []
+_camera_inventory_cache_at = 0.0
+CAMERA_INVENTORY_CACHE_SECONDS = 30.0
 
 def _open_camera(index):
     # On Windows, cv2.VideoCapture(index) with no explicit backend can
@@ -26,6 +54,14 @@ def _open_camera(index):
     # enumerates standard Windows webcams.
     if sys.platform == 'win32':
         return cv2.VideoCapture(index, cv2.CAP_DSHOW)
+    if sys.platform.startswith('linux'):
+        cap = cv2.VideoCapture(index, cv2.CAP_V4L2)
+        if cap.isOpened():
+            cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
+            cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
+            cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
+            cap.set(cv2.CAP_PROP_FPS, 15)
+        return cap
     return cv2.VideoCapture(index)
 
 
@@ -216,22 +252,39 @@ def send_webhook(frame_data):
     except Exception as e:
         print(f"Error preparing webhook: {e}")
 
-def generate_frames():
+def camera_worker():
     global latest_detections, motion_detection_active, motion_detected_state
     global last_motion_time, motion_frame_buffer
+    global latest_frame_bytes, latest_frame_sequence
+    global camera_read_started_at, camera_last_frame_at, camera
 
     prev_gray = None
 
     while True:
-        cam = get_camera()
+        with camera_lock:
+            cam = get_camera()
         if cam is None or not cam.isOpened():
             time.sleep(1)
             continue
 
-        success, frame = cam.read()
+        camera_read_started_at = time.monotonic()
+        try:
+            # This is the only camera reader. Do not hold camera_lock here:
+            # the watchdog must be able to release a wedged V4L2 handle.
+            success, frame = cam.read()
+        except Exception as exc:
+            print(f"Camera read error: {exc}")
+            success, frame = False, None
+        finally:
+            camera_read_started_at = 0.0
         if not success:
+            with camera_lock:
+                if camera is cam:
+                    camera.release()
+                    camera = None
             time.sleep(0.1)
             continue
+        camera_last_frame_at = time.monotonic()
 
         # Run object detection
         processed_frame, detections = detect_objects(frame, conf_threshold=0.5)
@@ -281,7 +334,55 @@ def generate_frames():
             prev_gray = gray
 
         ret, buffer = cv2.imencode('.jpg', processed_frame, [cv2.IMWRITE_JPEG_QUALITY, 80])
-        frame_bytes = buffer.tobytes()
+        if not ret:
+            continue
+        with frame_condition:
+            latest_frame_bytes = buffer.tobytes()
+            latest_frame_sequence += 1
+            frame_condition.notify_all()
+
+
+def _ensure_camera_worker():
+    global camera_worker_started
+    with frame_condition:
+        if camera_worker_started:
+            return
+        camera_worker_started = True
+    threading.Thread(target=camera_worker, name='robovision-camera', daemon=True).start()
+    threading.Thread(target=_camera_watchdog, name='robovision-camera-watchdog', daemon=True).start()
+
+
+def _camera_watchdog():
+    global camera, camera_read_started_at
+    while True:
+        time.sleep(2.0)
+        started = camera_read_started_at
+        if not started or time.monotonic() - started < 8.0:
+            continue
+        print("Camera read stalled for 8s; releasing V4L2 handle")
+        with camera_lock:
+            if camera is not None:
+                try:
+                    camera.release()
+                except Exception:
+                    pass
+                camera = None
+        camera_read_started_at = 0.0
+
+
+def generate_frames():
+    _ensure_camera_worker()
+    sequence = -1
+    while True:
+        with frame_condition:
+            frame_condition.wait_for(
+                lambda: latest_frame_bytes is not None and latest_frame_sequence != sequence,
+                timeout=5.0,
+            )
+            if latest_frame_bytes is None or latest_frame_sequence == sequence:
+                continue
+            frame_bytes = latest_frame_bytes
+            sequence = latest_frame_sequence
 
         yield (b'--frame\r\n'
                b'Content-Type: image/jpeg\r\n\r\n' + frame_bytes + b'\r\n')
@@ -299,6 +400,19 @@ def get_detections():
     with lock:
         return jsonify(latest_detections)
 
+
+@app.route('/api/camera/status')
+def camera_status():
+    age = None if not camera_last_frame_at else round(time.monotonic() - camera_last_frame_at, 3)
+    return jsonify({
+        "device": str(current_camera_index),
+        "open": bool(camera is not None and camera.isOpened()),
+        "worker_started": camera_worker_started,
+        "frame_sequence": latest_frame_sequence,
+        "last_frame_age_seconds": age,
+        "read_stalled": bool(camera_read_started_at and time.monotonic() - camera_read_started_at >= 8.0),
+    })
+
 @app.route('/api/caption')
 def get_caption():
     return jsonify({"caption": "Awaiting caption..."})
@@ -314,28 +428,148 @@ def caption_mode():
 
 @app.route('/api/cameras', methods=['GET'])
 def list_cameras():
+    global _camera_inventory_cache, _camera_inventory_cache_at
+    now = time.monotonic()
+    if now - _camera_inventory_cache_at < CAMERA_INVENTORY_CACHE_SECONDS:
+        return jsonify({"cameras": _camera_inventory_cache, "current": current_camera_index})
+
     available = []
-    for i in range(4):
-        cap = _open_camera(i)
-        if cap.isOpened():
-            available.append({"index": i, "name": f"Camera {i}"})
+    if sys.platform.startswith('linux'):
+        # Query capabilities without starting a stream. Opening every V4L2
+        # node through OpenCV also opens metadata/output nodes and can contend
+        # with the camera stream already owned by RoboVision.
+        import fcntl
+        vidioc_querycap = 0x80685600
+        video_capture = 0x00000001
+        video_capture_mplane = 0x00001000
+        device_caps_flag = 0x80000000
+        for candidate in sorted(glob.glob('/dev/video*')):
+            fd = None
+            try:
+                fd = os.open(candidate, os.O_RDONLY | os.O_NONBLOCK)
+                capability = bytearray(104)
+                fcntl.ioctl(fd, vidioc_querycap, capability, True)
+                capabilities = struct.unpack_from('=I', capability, 84)[0]
+                device_caps = struct.unpack_from('=I', capability, 88)[0]
+                effective = device_caps if capabilities & device_caps_flag else capabilities
+                if not effective & (video_capture | video_capture_mplane):
+                    continue
+                card = bytes(capability[16:48]).split(b'\0', 1)[0].decode('utf-8', 'replace')
+                available.append({
+                    "index": candidate,
+                    "id": candidate,
+                    "name": card or f"Camera {candidate}",
+                    "backend": "v4l2",
+                })
+            except (OSError, ValueError):
+                continue
+            finally:
+                if fd is not None:
+                    os.close(fd)
+    else:
+        for candidate in range(4):
+            cap = _open_camera(candidate)
+            if cap.isOpened():
+                available.append({"index": candidate, "id": str(candidate), "name": f"Camera {candidate}", "backend": "dshow"})
             cap.release()
+
+    _camera_inventory_cache = available
+    _camera_inventory_cache_at = now
     return jsonify({"cameras": available, "current": current_camera_index})
 
 @app.route('/api/camera/switch', methods=['POST'])
 def switch_camera():
     global camera, current_camera_index
     data = request.json or {}
-    new_index = int(data.get('index', 0))
+    new_index = data.get('device', data.get('index', 0))
+    if isinstance(new_index, str) and new_index.isdigit():
+        new_index = int(new_index)
 
-    if camera:
-        camera.release()
-
-    current_camera_index = new_index
-    camera = None
+    with camera_lock:
+        if camera:
+            camera.release()
+        current_camera_index = _normalize_camera_device(new_index)
+        camera = None
 
     return jsonify({"status": "ok", "camera_index": current_camera_index})
 
+
+def _audio_inventory():
+    """Adapt RoboVision's existing audio_server_pi /devices response."""
+    try:
+        response = requests.get(f"{ROBOVISION_AUDIO_URL.rstrip('/')}/devices", timeout=0.8)
+        response.raise_for_status()
+        payload = response.json()
+        inputs, outputs = [], []
+        for device in payload.get("devices", []):
+            item = {
+                "id": str(device["index"]),
+                "name": str(device.get("name", f"Audio device {device['index']}")),
+                "backend": "robovision_audio",
+                "sample_rate": device.get("default_samplerate"),
+            }
+            if device.get("max_input_channels", 0) > 0:
+                inputs.append(item.copy())
+            if device.get("max_output_channels", 0) > 0:
+                outputs.append(item.copy())
+        return inputs, outputs, {
+            "input": payload.get("bluetooth_input"),
+            "output": payload.get("bluetooth_output"),
+            "online": True,
+        }
+    except Exception as exc:
+        return [], [], {"online": False, "error": str(exc)}
+
+
+@app.route('/api/media/inventory', methods=['GET'])
+def media_inventory():
+    cameras = list_cameras().get_json().get("cameras", [])
+    inputs, outputs, audio_state = _audio_inventory()
+    return jsonify({
+        "video": [{"id": "auto", "name": "Auto detect"}, {"id": "none", "name": "Disable camera"}] + cameras,
+        "audio_input": inputs,
+        "audio_output": outputs,
+        "selected": {
+            "video_device": str(current_camera_index),
+            "audio_device": str(audio_state.get("input") if audio_state.get("input") is not None else audio_input_device),
+            "audio_output_device": str(audio_state.get("output") if audio_state.get("output") is not None else audio_output_device),
+        },
+        "audio_server": audio_state,
+        "source": "robovision_pi",
+    })
+
+
+@app.route('/api/media/config', methods=['GET', 'POST'])
+def media_config():
+    global audio_input_device, audio_output_device, camera, current_camera_index
+    if request.method == 'POST':
+        data = request.json or {}
+        if "video_device" in data:
+            value = str(data["video_device"])
+            if value not in ("", "auto", "none"):
+                switch_camera_value = value
+                if camera:
+                    camera.release()
+                current_camera_index = _normalize_camera_device(switch_camera_value)
+                camera = None
+        if "audio_device" in data:
+            audio_input_device = str(data["audio_device"])
+        if "audio_output_device" in data:
+            audio_output_device = str(data["audio_output_device"])
+        if "audio_device" in data or "audio_output_device" in data:
+            # audio_server_pi.py is RoboVision's authoritative selector.
+            # It accepts its original sounddevice indices via input/output.
+            payload = {}
+            if "audio_device" in data:
+                payload["input"] = int(audio_input_device) if audio_input_device.isdigit() else audio_input_device
+            if "audio_output_device" in data:
+                payload["output"] = int(audio_output_device) if audio_output_device.isdigit() else audio_output_device
+            try:
+                requests.post(f"{ROBOVISION_AUDIO_URL.rstrip('/')}/set-device", json=payload, timeout=0.8).raise_for_status()
+            except Exception:
+                pass
+    return jsonify({"video_device": str(current_camera_index), "audio_device": audio_input_device, "audio_output_device": audio_output_device, "source": "robovision_pi"})
+
 @app.route('/api/motion/status', methods=['GET'])
 def motion_status():
     global motion_detection_active, motion_detected_state, last_motion_time
@@ -416,4 +650,7 @@ if __name__ == '__main__':
         print(f"Motion webhook: {webhook_url}")
     print(f"Motion detection: {'ARMED' if motion_detection_active else 'off (POST /api/motion/toggle to arm)'}")
     print("=" * 60)
+    # Start the single camera owner at boot. Production motion and MJPEG
+    # readiness must not depend on an operator opening the dashboard first.
+    _ensure_camera_worker()
     app.run(host='0.0.0.0', port=args.port, debug=False, threaded=True)

package/vision/audio_server_pi.py

@@ -18,12 +18,29 @@ from datetime import datetime
 import logging
 import subprocess
 import base64
+import functools
+import sys
+from pathlib import Path
 from groq import Groq
 
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scheduler"))
+from media_lock import media_lock
+
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(message)s')
 logger = logging.getLogger(__name__)
 
+
+def exclusive_media(kind: str):
+    """Prevent diagnostics from stealing ALSA devices from production."""
+    def decorate(func):
+        @functools.wraps(func)
+        def wrapped(*args, **kwargs):
+            with media_lock(kind, timeout=3.0):
+                return func(*args, **kwargs)
+        return wrapped
+    return decorate
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """Lifespan event handler for startup and shutdown"""
@@ -173,6 +190,7 @@ def find_bluetooth_devices():
 # Find Bluetooth devices on startup
 find_bluetooth_devices()
 
+@exclusive_media("speaker")
 def play_audio_file(file_path: str):
     """Play audio file using sounddevice (supports Bluetooth)"""
     try:
@@ -231,6 +249,7 @@ def get_available_input_devices():
             input_devices.append(i)
     return input_devices
 
+@exclusive_media("microphone")
 def record_audio_with_vad():
     """
     Record audio from Bluetooth microphone with Voice Activity Detection

package/vision/install.sh

@@ -1,34 +1,34 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-VENV_DIR="$ROOT_DIR/venv"
-REQ_FILE="$ROOT_DIR/requirements_pi_unified.txt"
-
-if ! command -v python3 >/dev/null 2>&1; then
-  echo "python3 not found"
-  exit 1
-fi
-
-if [ ! -f "$REQ_FILE" ]; then
-  echo "Missing $REQ_FILE"
-  exit 1
-fi
-
-echo "System packages (run once):"
-echo "sudo apt-get update"
-echo "sudo apt-get install -y python3-opencv python3-numpy python3-pil libatlas-base-dev libopenblas-dev libhdf5-dev libhdf5-serial-dev libhdf5-103 libqt5gui5 libqt5widgets5 libqt5test5 portaudio19-dev libsndfile1 alsa-utils bluetooth bluez bluez-tools pulseaudio pulseaudio-module-bluetooth python3-pyaudio"
-echo ""
-
-echo "Creating venv: $VENV_DIR"
-if [ ! -d "$VENV_DIR" ]; then
-  python3 -m venv "$VENV_DIR" --system-site-packages
-fi
-
-source "$VENV_DIR/bin/activate"
-python -m pip install --upgrade pip
-python -m pip install -r "$REQ_FILE"
-
-echo ""
-echo "Done"
-echo "Next: ./run.sh start-term"
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+VENV_DIR="$ROOT_DIR/venv"
+REQ_FILE="$ROOT_DIR/requirements_pi_unified.txt"
+
+if ! command -v python3 >/dev/null 2>&1; then
+  echo "python3 not found"
+  exit 1
+fi
+
+if [ ! -f "$REQ_FILE" ]; then
+  echo "Missing $REQ_FILE"
+  exit 1
+fi
+
+echo "System packages (run once):"
+echo "sudo apt-get update"
+echo "sudo apt-get install -y python3-opencv python3-numpy python3-pil libatlas-base-dev libopenblas-dev libhdf5-dev libhdf5-serial-dev libhdf5-103 libqt5gui5 libqt5widgets5 libqt5test5 portaudio19-dev libsndfile1 alsa-utils bluetooth bluez bluez-tools pulseaudio pulseaudio-module-bluetooth python3-pyaudio"
+echo ""
+
+echo "Creating venv: $VENV_DIR"
+if [ ! -d "$VENV_DIR" ]; then
+  python3 -m venv "$VENV_DIR" --system-site-packages
+fi
+
+source "$VENV_DIR/bin/activate"
+python -m pip install --upgrade pip
+python -m pip install -r "$REQ_FILE"
+
+echo ""
+echo "Done"
+echo "Next: ./run.sh start-term"