plexus-python 0.1.0__py3-none-any.whl

plexus/buffer.py

@@ -0,0 +1,257 @@
+"""
+Pluggable buffer backends for local point storage.
+
+Two implementations:
+  - MemoryBuffer: In-memory list (default, matches original behavior)
+  - SqliteBuffer: WAL-mode SQLite for persistence across restarts
+
+Store-and-forward:
+  Both backends support drain(batch_size) for incremental backlog upload.
+  SqliteBuffer can run uncapped (max_size=None) for intermittently connected
+  devices like satellites or field robots that buffer for hours/days.
+"""
+
+import json
+import logging
+import os
+import sqlite3
+import threading
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class BufferBackend(ABC):
+    """Abstract buffer backend for storing telemetry points locally."""
+
+    @abstractmethod
+    def add(self, points: List[Dict[str, Any]]) -> None:
+        """Add points to the buffer, evicting oldest if over capacity."""
+
+    @abstractmethod
+    def get_all(self) -> List[Dict[str, Any]]:
+        """Return a copy of all buffered points without clearing."""
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Remove all buffered points."""
+
+    @abstractmethod
+    def size(self) -> int:
+        """Return current number of buffered points."""
+
+    def drain(self, batch_size: int = 5000) -> Tuple[List[Dict[str, Any]], int]:
+        """Remove and return the oldest batch_size points atomically.
+
+        Returns (points, remaining_count). Points are deleted from the buffer
+        only by this call — if the caller fails to upload them, they are lost.
+        Use this for backlog drain where each batch is uploaded then confirmed.
+
+        Default implementation uses get_all/clear (non-atomic). Subclasses
+        should override with atomic implementations.
+        """
+        all_pts = self.get_all()
+        batch = all_pts[:batch_size]
+        if not batch:
+            return [], 0
+        # Replace buffer with remaining points
+        self.clear()
+        remaining = all_pts[batch_size:]
+        if remaining:
+            self.add(remaining)
+        return batch, len(remaining)
+
+
+class MemoryBuffer(BufferBackend):
+    """In-memory buffer with FIFO eviction. Thread-safe.
+
+    This extracts the original behavior from Plexus client._failed_buffer.
+    """
+
+    def __init__(self, max_size: int = 10_000, on_overflow: Optional[Callable[[int], None]] = None):
+        self._max_size = max_size
+        self._on_overflow = on_overflow
+        self._buffer: List[Dict[str, Any]] = []
+        self._lock = threading.Lock()
+
+    def add(self, points: List[Dict[str, Any]]) -> None:
+        with self._lock:
+            self._buffer.extend(points)
+            if len(self._buffer) > self._max_size:
+                overflow = len(self._buffer) - self._max_size
+                logger.warning("Buffer full, dropped %d oldest points", overflow)
+                self._buffer = self._buffer[overflow:]
+                if self._on_overflow:
+                    self._on_overflow(overflow)
+
+    def get_all(self) -> List[Dict[str, Any]]:
+        with self._lock:
+            return list(self._buffer)
+
+    def clear(self) -> None:
+        with self._lock:
+            self._buffer.clear()
+
+    def size(self) -> int:
+        with self._lock:
+            return len(self._buffer)
+
+    def drain(self, batch_size: int = 5000) -> Tuple[List[Dict[str, Any]], int]:
+        with self._lock:
+            batch = self._buffer[:batch_size]
+            self._buffer = self._buffer[batch_size:]
+            return batch, len(self._buffer)
+
+
+class SqliteBuffer(BufferBackend):
+    """SQLite-backed persistent buffer using WAL mode. Thread-safe.
+
+    Survives process restarts. Points are stored as JSON blobs in a single
+    table with auto-incrementing rowid for FIFO ordering.
+
+    Args:
+        path: Path to the SQLite database file.
+              Defaults to ~/.plexus/buffer.db
+        max_size: Maximum number of points to retain. None = unlimited
+                  (disk-bound, suitable for store-and-forward).
+        max_bytes: Maximum database file size in bytes. None = no limit.
+                   Safety valve to prevent filling the disk.
+    """
+
+    def __init__(
+        self,
+        path: Optional[str] = None,
+        max_size: Optional[int] = 100_000,
+        max_bytes: Optional[int] = None,
+        on_overflow: Optional[Callable[[int], None]] = None,
+    ):
+        self._max_size = max_size
+        self._max_bytes = max_bytes
+        self._on_overflow = on_overflow
+        self._lock = threading.Lock()
+
+        if path is None:
+            plexus_dir = os.path.join(os.path.expanduser("~"), ".plexus")
+            os.makedirs(plexus_dir, exist_ok=True)
+            try:
+                os.chmod(plexus_dir, 0o700)
+            except OSError:
+                pass  # Windows or restricted filesystem
+            path = os.path.join(plexus_dir, "buffer.db")
+
+        self._path = path
+        self._conn = sqlite3.connect(path, check_same_thread=False)
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute(
+            "CREATE TABLE IF NOT EXISTS points ("
+            "  id INTEGER PRIMARY KEY AUTOINCREMENT,"
+            "  data TEXT NOT NULL"
+            ")"
+        )
+        self._conn.commit()
+
+    def add(self, points: List[Dict[str, Any]]) -> None:
+        if not points:
+            return
+        with self._lock:
+            # Disk-space safety valve
+            if self._max_bytes is not None:
+                try:
+                    if os.path.getsize(self._path) >= self._max_bytes:
+                        logger.warning("Buffer at disk limit (%d bytes), dropping oldest", self._max_bytes)
+                        self._evict_pct(10)  # Drop 10% to make room
+                except OSError:
+                    pass
+            self._conn.executemany(
+                "INSERT INTO points (data) VALUES (?)",
+                [(json.dumps(p),) for p in points],
+            )
+            self._conn.commit()
+            self._evict()
+
+    def get_all(self) -> List[Dict[str, Any]]:
+        with self._lock:
+            cursor = self._conn.execute("SELECT data FROM points ORDER BY id")
+            return [json.loads(row[0]) for row in cursor.fetchall()]
+
+    def clear(self) -> None:
+        with self._lock:
+            self._conn.execute("DELETE FROM points")
+            self._conn.commit()
+
+    def size(self) -> int:
+        with self._lock:
+            cursor = self._conn.execute("SELECT COUNT(*) FROM points")
+            return cursor.fetchone()[0]
+
+    def drain(self, batch_size: int = 5000) -> Tuple[List[Dict[str, Any]], int]:
+        """Remove and return the oldest batch_size points atomically.
+
+        Uses a single transaction: SELECT then DELETE by rowid. If the process
+        crashes after drain() but before the caller uploads, points are gone —
+        this is the intended trade-off for simplicity. For crash-safe drain,
+        the caller should use smaller batch sizes.
+        """
+        with self._lock:
+            cursor = self._conn.execute(
+                "SELECT id, data FROM points ORDER BY id LIMIT ?",
+                (batch_size,),
+            )
+            rows = cursor.fetchall()
+            if not rows:
+                return [], 0
+
+            points = [json.loads(row[1]) for row in rows]
+            ids = [row[0] for row in rows]
+
+            # Delete the exact rows we read
+            placeholders = ",".join("?" * len(ids))
+            self._conn.execute(
+                f"DELETE FROM points WHERE id IN ({placeholders})",
+                ids,
+            )
+            self._conn.commit()
+
+            remaining = self._conn.execute("SELECT COUNT(*) FROM points").fetchone()[0]
+            return points, remaining
+
+    def close(self) -> None:
+        """Close the underlying database connection."""
+        with self._lock:
+            self._conn.close()
+
+    def _evict(self) -> None:
+        """Remove oldest rows if over max_size. Must be called with lock held."""
+        if self._max_size is None:
+            return
+        cursor = self._conn.execute("SELECT COUNT(*) FROM points")
+        count = cursor.fetchone()[0]
+        if count > self._max_size:
+            overflow = count - self._max_size
+            self._conn.execute(
+                "DELETE FROM points WHERE id IN ("
+                "  SELECT id FROM points ORDER BY id LIMIT ?"
+                ")",
+                (overflow,),
+            )
+            self._conn.commit()
+            logger.warning("Buffer full, dropped %d oldest points", overflow)
+            if self._on_overflow:
+                self._on_overflow(overflow)
+
+    def _evict_pct(self, pct: int) -> None:
+        """Evict a percentage of buffered points. Must be called with lock held."""
+        cursor = self._conn.execute("SELECT COUNT(*) FROM points")
+        count = cursor.fetchone()[0]
+        to_drop = max(1, count * pct // 100)
+        self._conn.execute(
+            "DELETE FROM points WHERE id IN ("
+            "  SELECT id FROM points ORDER BY id LIMIT ?"
+            ")",
+            (to_drop,),
+        )
+        self._conn.commit()
+        logger.warning("Disk safety: dropped %d oldest points", to_drop)
+        if self._on_overflow:
+            self._on_overflow(to_drop)

plexus/cameras/__init__.py

@@ -0,0 +1,57 @@
+"""
+Plexus camera drivers for streaming video from hardware devices.
+
+Supports USB webcams and Raspberry Pi Camera Modules.
+
+Usage:
+    # USB webcam
+    from plexus.cameras import USBCamera
+    camera = USBCamera(device_index=0)
+    frame = camera.capture()
+
+    # Auto-detect cameras
+    from plexus.cameras import scan_cameras, auto_cameras
+    detected = scan_cameras()
+    hub = auto_cameras()
+
+Install camera support:
+    pip install plexus-python[camera]      # USB webcams (OpenCV)
+    pip install plexus-python[picamera]    # Raspberry Pi Camera
+"""
+
+from plexus.cameras.base import BaseCamera, CameraFrame, CameraHub
+from plexus.cameras.auto import (
+    DetectedCamera,
+    scan_cameras,
+    scan_usb_cameras,
+    scan_pi_cameras,
+    auto_cameras,
+)
+
+# Optional imports - only available if dependencies installed
+try:
+    from plexus.cameras.usb import USBCamera
+except ImportError:
+    USBCamera = None
+
+try:
+    from plexus.cameras.picamera import PiCamera
+except ImportError:
+    PiCamera = None
+
+
+__all__ = [
+    # Base classes
+    "BaseCamera",
+    "CameraFrame",
+    "CameraHub",
+    # Drivers (may be None if dependencies not installed)
+    "USBCamera",
+    "PiCamera",
+    # Auto-detection
+    "DetectedCamera",
+    "scan_cameras",
+    "scan_usb_cameras",
+    "scan_pi_cameras",
+    "auto_cameras",
+]

plexus/cameras/auto.py

@@ -0,0 +1,239 @@
+"""
+Camera auto-detection utilities.
+
+Scans for available cameras and creates appropriate drivers.
+"""
+
+import logging
+import os
+from dataclasses import dataclass
+from typing import List, Optional, Set, Tuple, Type
+
+from plexus.cameras.base import BaseCamera, CameraHub
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DetectedCamera:
+    """Information about a detected camera."""
+    name: str
+    device_id: str
+    driver: Type[BaseCamera]
+    description: str
+
+
+def _get_pi_camera_v4l2_indices() -> Set[int]:
+    """Get V4L2 device indices that belong to the Pi Camera pipeline.
+
+    On Raspberry Pi, the camera module exposes multiple /dev/video* devices
+    through the Unicam/ISP pipeline. These should not be opened by OpenCV
+    as they conflict with picamera2/libcamera access.
+    """
+    pi_indices: Set[int] = set()
+    sysfs = "/sys/class/video4linux"
+    if not os.path.isdir(sysfs):
+        return pi_indices
+    for entry in os.listdir(sysfs):
+        if not entry.startswith("video"):
+            continue
+        try:
+            with open(os.path.join(sysfs, entry, "name")) as f:
+                name = f.read().strip().lower()
+            if "unicam" in name or "bcm2835" in name or "rp1" in name:
+                pi_indices.add(int(entry[5:]))  # strip "video" prefix
+        except (IOError, OSError, ValueError):
+            pass
+    return pi_indices
+
+
+def scan_usb_cameras(
+    max_cameras: int = 10, skip_indices: Optional[Set[int]] = None
+) -> List[DetectedCamera]:
+    """
+    Scan for USB cameras using OpenCV.
+
+    Args:
+        max_cameras: Maximum number of device indices to check.
+        skip_indices: V4L2 device indices to skip (e.g. Pi Camera devices).
+
+    Returns:
+        List of detected USB cameras.
+    """
+    try:
+        import cv2
+    except ImportError:
+        return []
+
+    from plexus.cameras.usb import USBCamera
+
+    detected = []
+
+    for i in range(max_cameras):
+        if skip_indices and i in skip_indices:
+            continue
+        cap = cv2.VideoCapture(i)
+        if cap.isOpened():
+            # Get camera info if available
+            width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+            height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+            cap.release()
+
+            detected.append(DetectedCamera(
+                name=f"USB Camera {i}",
+                device_id=f"usb:{i}",
+                driver=USBCamera,
+                description=f"USB webcam at index {i} ({width}x{height})",
+            ))
+        else:
+            cap.release()
+            # Stop after first failed index (cameras are typically sequential)
+            if i > 0:
+                break
+
+    return detected
+
+
+def scan_pi_cameras() -> List[DetectedCamera]:
+    """
+    Scan for Raspberry Pi cameras using picamera2.
+
+    Returns:
+        List of detected Pi cameras.
+    """
+    try:
+        from picamera2 import Picamera2
+    except ImportError:
+        return []
+
+    from plexus.cameras.picamera import PiCamera
+
+    detected = []
+
+    try:
+        # Get list of available cameras
+        camera_info = Picamera2.global_camera_info()
+
+        for i, info in enumerate(camera_info):
+            model = info.get("Model", "Unknown")
+            detected.append(DetectedCamera(
+                name=f"Pi Camera {i}",
+                device_id=f"picam:{i}",
+                driver=PiCamera,
+                description=f"Raspberry Pi Camera: {model}",
+            ))
+    except Exception as e:
+        logger.debug(f"Pi camera scan failed: {e}")
+
+    return detected
+
+
+def scan_cameras() -> List[DetectedCamera]:
+    """
+    Scan for all available cameras (USB and Pi).
+
+    Pi cameras are scanned first so their V4L2 devices can be excluded
+    from the USB scan, preventing duplicate detection and device conflicts.
+
+    Returns:
+        List of all detected cameras.
+
+    Raises:
+        ImportError: If neither picamera2 nor opencv-python is installed.
+    """
+    has_picamera2 = True
+    has_cv2 = True
+
+    try:
+        import picamera2  # noqa: F401
+    except ImportError:
+        has_picamera2 = False
+
+    try:
+        import cv2  # noqa: F401
+    except ImportError:
+        has_cv2 = False
+
+    if not has_picamera2 and not has_cv2:
+        raise ImportError("No camera library installed (picamera2 or opencv-python)")
+
+    cameras = []
+
+    pi_cameras = scan_pi_cameras()
+    cameras.extend(pi_cameras)
+
+    # If Pi cameras exist, skip their V4L2 device indices so OpenCV
+    # doesn't open them (which blocks picamera2/libcamera access).
+    skip = _get_pi_camera_v4l2_indices() if pi_cameras else set()
+    cameras.extend(scan_usb_cameras(skip_indices=skip))
+
+    return cameras
+
+
+def auto_cameras(
+    frame_rate: Optional[float] = None,
+    resolution: Optional[Tuple[int, int]] = None,
+    quality: Optional[int] = None,
+) -> CameraHub:
+    """
+    Auto-detect cameras and create a CameraHub.
+
+    Args:
+        frame_rate: Override frame rate for all cameras.
+        resolution: Override resolution for all cameras.
+        quality: Override JPEG quality for all cameras.
+
+    Returns:
+        CameraHub with detected cameras added.
+    """
+    hub = CameraHub()
+    detected = scan_cameras()
+
+    for camera_info in detected:
+        kwargs = {"camera_id": camera_info.device_id}
+
+        if frame_rate is not None:
+            kwargs["frame_rate"] = frame_rate
+        if resolution is not None:
+            kwargs["resolution"] = resolution
+        if quality is not None:
+            kwargs["quality"] = quality
+
+        # Create camera instance based on driver type
+        if camera_info.device_id.startswith("usb:"):
+            device_index = int(camera_info.device_id.split(":")[1])
+            kwargs["device_index"] = device_index
+        elif camera_info.device_id.startswith("picam:"):
+            camera_num = int(camera_info.device_id.split(":")[1])
+            kwargs["camera_num"] = camera_num
+
+        try:
+            camera = camera_info.driver(**kwargs)
+            hub.add(camera)
+        except Exception as e:
+            logger.debug(f"Failed to create {camera_info.name}: {e}")
+
+    return hub
+
+
+def get_camera_info() -> List[dict]:
+    """
+    Get information about supported camera types.
+
+    Returns:
+        List of camera type info dicts.
+    """
+    return [
+        {
+            "name": "USB Camera",
+            "description": "USB webcams and built-in cameras via OpenCV",
+            "requires": "opencv-python",
+            "install": "pip install plexus-python[camera]",
+        },
+        {
+            "name": "Pi Camera",
+            "description": "Raspberry Pi Camera Module via picamera2",
+            "requires": "picamera2",
+            "install": "pip install plexus-python[picamera]",
+        },
+    ]