homesec 0.1.0__py3-none-any.whl

homesec/sources/base.py

@@ -0,0 +1,224 @@
+"""Shared helpers for threaded and async clip sources."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from abc import ABC, abstractmethod
+from threading import Event, Thread
+from typing import Callable
+
+from homesec.models.clip import Clip
+from homesec.interfaces import ClipSource
+
+logger = logging.getLogger(__name__)
+
+
+class ThreadedClipSource(ClipSource, ABC):
+    """Base class for clip sources that run in a background thread."""
+
+    def __init__(self) -> None:
+        self._callback: Callable[[Clip], None] | None = None
+        self._thread: Thread | None = None
+        self._stop_event = Event()
+        self._last_heartbeat = time.monotonic()
+
+    def register_callback(self, callback: Callable[[Clip], None]) -> None:
+        """Register callback to be invoked when a new clip is ready."""
+        self._callback = callback
+
+    async def start(self) -> None:
+        """Start producing clips in a background thread."""
+        if self._thread is not None:
+            logger.warning("%s already started", self.__class__.__name__)
+            return
+
+        self._stop_event.clear()
+        self._on_start()
+        self._thread = Thread(target=self._run_wrapper, daemon=True)
+        self._thread.start()
+        self._on_started()
+
+    def stop(self, timeout: float | None = None) -> None:
+        """Stop the background thread and cleanup resources."""
+        thread = self._thread
+        if thread is None:
+            return
+
+        self._stop_event.set()
+        self._on_stop()
+
+        if thread.is_alive():
+            thread.join(timeout=timeout or self._stop_timeout())
+
+        if self._thread is thread:
+            self._thread = None
+        self._on_stopped()
+
+    async def shutdown(self, timeout: float | None = None) -> None:
+        """Async wrapper for stopping the background thread."""
+        await asyncio.to_thread(self.stop, timeout)
+
+    def is_healthy(self) -> bool:
+        """Default health check: thread is alive (if started)."""
+        return self._thread_is_healthy()
+
+    def last_heartbeat(self) -> float:
+        """Return monotonic timestamp of last heartbeat update."""
+        return self._last_heartbeat
+
+    def _touch_heartbeat(self) -> None:
+        self._last_heartbeat = time.monotonic()
+
+    def _thread_is_healthy(self) -> bool:
+        if self._thread is None:
+            return True
+        return self._thread.is_alive()
+
+    def _emit_clip(self, clip: Clip) -> None:
+        if not self._callback:
+            return
+        try:
+            self._callback(clip)
+        except Exception as exc:
+            logger.error(
+                "Callback failed for %s: %s",
+                clip.clip_id,
+                exc,
+                exc_info=True,
+            )
+
+    def _run_wrapper(self) -> None:
+        try:
+            self._run()
+        except Exception:
+            logger.exception("%s stopped unexpectedly", self.__class__.__name__)
+        finally:
+            self._thread = None
+
+    def _stop_timeout(self) -> float:
+        return 5.0
+
+    def _on_start(self) -> None:
+        """Hook called before starting the background thread."""
+
+    def _on_started(self) -> None:
+        """Hook called after starting the background thread."""
+
+    def _on_stop(self) -> None:
+        """Hook called before stopping the background thread."""
+
+    def _on_stopped(self) -> None:
+        """Hook called after stopping the background thread."""
+
+    @abstractmethod
+    def _run(self) -> None:
+        """Thread entrypoint (blocking)."""
+        raise NotImplementedError
+
+
+class AsyncClipSource(ClipSource, ABC):
+    """Base class for clip sources that run as async tasks."""
+
+    def __init__(self) -> None:
+        self._callback: Callable[[Clip], None] | None = None
+        self._task: asyncio.Task[None] | None = None
+        self._stop_event = asyncio.Event()
+        self._last_heartbeat = time.monotonic()
+
+    def register_callback(self, callback: Callable[[Clip], None]) -> None:
+        """Register callback to be invoked when a new clip is ready."""
+        self._callback = callback
+
+    async def start(self) -> None:
+        """Start producing clips in a background task."""
+        if self._task is not None:
+            logger.warning("%s already started", self.__class__.__name__)
+            return
+
+        self._stop_event.clear()
+        self._on_start()
+        self._task = asyncio.create_task(self._run_wrapper())
+        self._on_started()
+
+    async def shutdown(self, timeout: float | None = None) -> None:
+        """Stop the background task and cleanup resources."""
+        task = self._task
+        if task is None:
+            return
+
+        self._stop_event.set()
+        self._on_stop()
+
+        if not task.done():
+            try:
+                await asyncio.wait_for(task, timeout=timeout or self._stop_timeout())
+            except asyncio.TimeoutError:
+                logger.warning("%s shutdown timed out, cancelling task", self.__class__.__name__)
+                task.cancel()
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+
+        if self._task is task:
+            self._task = None
+        self._on_stopped()
+
+    def is_healthy(self) -> bool:
+        """Default health check: task is running (if started)."""
+        return self._task_is_healthy()
+
+    def last_heartbeat(self) -> float:
+        """Return timestamp (monotonic) of last successful operation."""
+        return self._last_heartbeat
+
+    def _touch_heartbeat(self) -> None:
+        self._last_heartbeat = time.monotonic()
+
+    def _task_is_healthy(self) -> bool:
+        if self._task is None:
+            return True
+        return not self._task.done()
+
+    def _emit_clip(self, clip: Clip) -> None:
+        if not self._callback:
+            return
+        try:
+            self._callback(clip)
+        except Exception as exc:
+            logger.error(
+                "Callback failed for %s: %s",
+                clip.clip_id,
+                exc,
+                exc_info=True,
+            )
+
+    async def _run_wrapper(self) -> None:
+        try:
+            await self._run()
+        except Exception:
+            logger.exception("%s stopped unexpectedly", self.__class__.__name__)
+        finally:
+            self._task = None
+
+    def _stop_timeout(self) -> float:
+        return 5.0
+
+    def _on_start(self) -> None:
+        """Hook called before starting the background task."""
+
+    def _on_started(self) -> None:
+        """Hook called after starting the background task."""
+
+    def _on_stop(self) -> None:
+        """Hook called before stopping the background task."""
+
+    def _on_stopped(self) -> None:
+        """Hook called after stopping the background task."""
+
+    @abstractmethod
+    async def _run(self) -> None:
+        """Async task entrypoint."""
+        raise NotImplementedError

homesec/sources/ftp.py

@@ -0,0 +1,209 @@
+"""FTP clip source for camera uploads."""
+
+from __future__ import annotations
+
+import logging
+import os
+from datetime import datetime, timedelta
+from pathlib import Path
+from threading import Thread
+from typing import Any
+
+from homesec.models.clip import Clip
+from homesec.models.source import FtpSourceConfig
+from homesec.sources.base import ThreadedClipSource
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_passive_ports(spec: str | None) -> list[int] | None:
+    if not spec:
+        return None
+    spec = str(spec).strip()
+    if not spec:
+        return None
+
+    if "-" in spec:
+        start_s, end_s = spec.split("-", 1)
+        start = int(start_s)
+        end = int(end_s)
+        if end < start:
+            raise ValueError(f"Invalid passive_ports range: {spec!r}")
+        return list(range(start, end + 1))
+
+    ports: list[int] = []
+    for part in spec.split(","):
+        part = part.strip()
+        if part:
+            ports.append(int(part))
+    return ports or None
+
+
+class FtpSource(ThreadedClipSource):
+    """FTP server clip source."""
+
+    def __init__(self, config: FtpSourceConfig, camera_name: str) -> None:
+        super().__init__()
+        self._config = config
+        self.camera_name = camera_name
+        self.root_dir = Path(config.root_dir).expanduser().resolve()
+        if config.ftp_subdir:
+            self.root_dir = self.root_dir / config.ftp_subdir
+        self._allowed_extensions = set(config.allowed_extensions)
+        self._username = self._resolve_env(config.username_env)
+        self._password = self._resolve_env(config.password_env)
+
+        if not config.anonymous and (not self._username or not self._password):
+            raise RuntimeError(
+                "FTP auth requires username/password env vars when anonymous is False"
+            )
+
+        self._server: Any | None = None
+        self._heartbeat_thread: Thread | None = None
+
+    def is_healthy(self) -> bool:
+        """Check if source is healthy."""
+        return self._thread_is_healthy()
+
+    def _heartbeat_loop(self) -> None:
+        while not self._stop_event.wait(self._config.heartbeat_s):
+            self._touch_heartbeat()
+
+    def _run(self) -> None:
+        if self._server is None:
+            return
+        self._touch_heartbeat()
+        try:
+            self._server.serve_forever()
+        except Exception:
+            logger.exception("FTP server stopped unexpectedly")
+
+    def _create_server(self) -> Any:
+        try:
+            from pyftpdlib.authorizers import DummyAuthorizer  # type: ignore[import-untyped]
+            from pyftpdlib.handlers import FTPHandler  # type: ignore[import-untyped]
+            from pyftpdlib.servers import FTPServer  # type: ignore[import-untyped]
+        except ImportError as exc:
+            raise RuntimeError(
+                "Missing dependency: pyftpdlib. Install with: uv pip install pyftpdlib"
+            ) from exc
+
+        authorizer = DummyAuthorizer()
+        if self._config.anonymous:
+            authorizer.add_anonymous(str(self.root_dir), perm=self._config.perms)
+        else:
+            assert self._username is not None
+            assert self._password is not None
+            authorizer.add_user(
+                self._username, self._password, str(self.root_dir), perm=self._config.perms
+            )
+
+        source = self
+
+        class UploadHandler(FTPHandler):  # type: ignore[misc]
+            def on_connect(self) -> None:
+                source._touch_heartbeat()
+
+            def on_login(self, username: str) -> None:
+                source._touch_heartbeat()
+
+            def on_file_received(self, file: str) -> None:
+                source._handle_file_received(Path(file))
+
+            def on_incomplete_file_received(self, file: str) -> None:
+                source._handle_incomplete_file(Path(file))
+
+        UploadHandler.authorizer = authorizer
+
+        parsed_passive_ports = _parse_passive_ports(self._config.passive_ports)
+        if parsed_passive_ports is not None:
+            UploadHandler.passive_ports = parsed_passive_ports
+        if self._config.masquerade_address:
+            UploadHandler.masquerade_address = self._config.masquerade_address
+
+        return FTPServer((self._config.host, int(self._config.port)), UploadHandler)
+
+    def _is_extension_allowed(self, file_path: Path) -> bool:
+        if not self._allowed_extensions:
+            return True
+        return file_path.suffix.lower() in self._allowed_extensions
+
+    def _handle_file_received(self, file_path: Path) -> None:
+        self._touch_heartbeat()
+        logger.info("Received upload: %s", file_path)
+        if not self._is_extension_allowed(file_path):
+            logger.info("Rejecting upload with unsupported extension: %s", file_path)
+            if self._config.delete_non_matching:
+                try:
+                    file_path.unlink(missing_ok=True)
+                except Exception:
+                    logger.exception("Failed to delete non-matching upload: %s", file_path)
+            return
+
+        clip = self._create_clip(file_path)
+        self._emit_clip(clip)
+
+    def _handle_incomplete_file(self, file_path: Path) -> None:
+        self._touch_heartbeat()
+        logger.warning("Incomplete upload (deleting): %s", file_path)
+        if not self._config.delete_incomplete:
+            return
+        try:
+            file_path.unlink(missing_ok=True)
+        except Exception:
+            logger.exception("Failed to delete incomplete upload: %s", file_path)
+
+    def _create_clip(self, file_path: Path) -> Clip:
+        stat = file_path.stat()
+        mtime = datetime.fromtimestamp(stat.st_mtime)
+        duration_s = float(self._config.default_duration_s)
+        return Clip(
+            clip_id=file_path.stem,
+            camera_name=self.camera_name,
+            local_path=file_path,
+            start_ts=mtime - timedelta(seconds=duration_s),
+            end_ts=mtime,
+            duration_s=duration_s,
+            source_type="ftp",
+        )
+
+    def _resolve_env(self, name: str | None) -> str | None:
+        if not name:
+            return None
+        value = os.getenv(name)
+        if value:
+            return value
+        return None
+
+    def _on_start(self) -> None:
+        logger.setLevel(getattr(logging, str(self._config.log_level).upper(), logging.INFO))
+        self.root_dir.mkdir(parents=True, exist_ok=True)
+        self._server = self._create_server()
+
+        if self._config.heartbeat_s > 0:
+            self._heartbeat_thread = Thread(target=self._heartbeat_loop, daemon=True)
+            self._heartbeat_thread.start()
+
+        logger.info(
+            "FTP source started: %s:%s (root=%s)",
+            self._config.host,
+            self._config.port,
+            self.root_dir,
+        )
+
+    def _on_stop(self) -> None:
+        logger.info("Stopping FtpSource...")
+        if self._server is not None:
+            try:
+                self._server.close_all()
+            except Exception:
+                logger.exception("Failed to close FTP server")
+
+        if self._heartbeat_thread and self._heartbeat_thread.is_alive():
+            self._heartbeat_thread.join(timeout=5.0)
+
+        self._heartbeat_thread = None
+        self._server = None
+
+    def _on_stopped(self) -> None:
+        logger.info("FtpSource stopped")

homesec/sources/local_folder.py

@@ -0,0 +1,238 @@
+"""Local folder clip source for production and development."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from collections import OrderedDict
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Callable, TYPE_CHECKING
+
+from anyio import Path as AsyncPath
+
+from homesec.models.clip import Clip
+from homesec.models.source import LocalFolderSourceConfig
+from homesec.sources.base import AsyncClipSource
+
+if TYPE_CHECKING:
+    from homesec.interfaces import StateStore
+
+logger = logging.getLogger(__name__)
+
+
+class LocalFolderSource(AsyncClipSource):
+    """Watches a local folder for new video clips.
+
+    Production-ready async clip source that monitors a directory for new .mp4 files.
+    Uses anyio for non-blocking filesystem operations (glob, stat).
+    Suitable for both testing and production use with local camera storage.
+    """
+
+    def __init__(
+        self,
+        config: LocalFolderSourceConfig,
+        camera_name: str = "local",
+        state_store: "StateStore | None" = None,
+    ) -> None:
+        """Initialize folder watcher.
+
+        Args:
+            config: LocalFolder source configuration
+            camera_name: Name of the camera (used in Clip objects)
+            state_store: Optional state store for deduplication via clip_states table.
+                If None, falls back to in-memory cache only (may reprocess files after restart).
+        """
+        super().__init__()
+        self.watch_dir = Path(config.watch_dir)
+        self.camera_name = camera_name
+        self.poll_interval = float(config.poll_interval)
+        self.stability_threshold_s = float(config.stability_threshold_s)
+        self._state_store = state_store
+
+        # Ensure watch dir exists
+        self.watch_dir.mkdir(parents=True, exist_ok=True)
+
+        # Bounded in-memory cache for performance (avoid DB query on every scan)
+        # 10,000 files ≈ 100 days for 1 camera @ 100 clips/day (≈500 KB memory)
+        # When limit exceeded, oldest half are removed (FIFO eviction)
+        # This is just an optimization - clip_states table is source of truth
+        self._seen_files: OrderedDict[str, None] = OrderedDict()
+        self._max_seen_files = 10000
+
+        logger.info("LocalFolderSource initialized: watch_dir=%s, has_state_store=%s",
+                   self.watch_dir, state_store is not None)
+
+    def register_callback(self, callback: Callable[[Clip], None]) -> None:
+        """Register callback to be invoked when new clip is ready."""
+        super().register_callback(callback)
+        logger.debug("Callback registered for %s", self.camera_name)
+
+    def is_healthy(self) -> bool:
+        """Check if source is healthy.
+
+        Returns True if:
+        - Watch directory exists and is readable
+        - Watch task is alive (if started)
+        """
+        if not self.watch_dir.exists():
+            return False
+
+        if not self.watch_dir.is_dir():
+            return False
+
+        # If started, task should be running
+        if not self._task_is_healthy():
+            return False
+
+        return True
+
+    def _on_start(self) -> None:
+        logger.info("Starting LocalFolderSource: %s", self.watch_dir)
+
+    def _on_stop(self) -> None:
+        logger.info("Stopping LocalFolderSource...")
+
+    def _on_stopped(self) -> None:
+        logger.info("LocalFolderSource stopped")
+
+    async def _run(self) -> None:
+        """Background task that polls for new files.
+
+        Uses anyio.Path for non-blocking filesystem operations to avoid
+        stalling the event loop on slow/network filesystems.
+        """
+        logger.info("Watch loop started")
+
+        # Create async path wrapper for watch directory
+        async_watch_dir = AsyncPath(self.watch_dir)
+
+        while not self._stop_event.is_set():
+            try:
+                # Update heartbeat
+                self._touch_heartbeat()
+
+                # Scan for new .mp4 files (async to avoid blocking event loop)
+                async for async_file_path in async_watch_dir.glob("*.mp4"):
+                    file_path = Path(async_file_path)  # Convert to regular Path for Clip
+                    file_id = str(file_path)
+                    clip_id = file_path.stem
+
+                    # Check in-memory cache first (fast path)
+                    if file_id in self._seen_files:
+                        continue
+
+                    # Check file stability (avoid processing while still being written)
+                    # Use async stat to avoid blocking event loop
+                    try:
+                        stat_info = await async_file_path.stat()
+                        mtime = stat_info.st_mtime
+                        age_s = time.time() - mtime
+                        if age_s < self.stability_threshold_s:
+                            logger.debug(
+                                "Skipping unstable file: %s (modified %.1fs ago)",
+                                file_path.name,
+                                age_s,
+                            )
+                            continue
+                    except OSError as e:
+                        logger.warning("Failed to stat file %s: %s", file_path, e, exc_info=True)
+                        continue
+
+                    # Check clip_states table (source of truth for deduplication)
+                    # This prevents reprocessing files even after cache eviction or restart
+                    if await self._has_clip_state(clip_id):
+                        # File was already processed - add to cache and skip
+                        self._add_to_cache(file_id)
+                        logger.debug("Skipping already-processed file: %s", file_path.name)
+                        continue
+
+                    # Mark as seen in cache
+                    self._add_to_cache(file_id)
+
+                    # Create Clip object (reuse mtime from async stat to avoid blocking)
+                    clip = self._create_clip(file_path, mtime=mtime)
+
+                    # Invoke callback
+                    logger.info("New clip detected: %s", file_path.name)
+                    self._emit_clip(clip)
+
+            except Exception as e:
+                logger.error("Error in watch loop: %s", e, exc_info=True)
+
+            # Sleep before next poll
+            try:
+                await asyncio.wait_for(
+                    self._stop_event.wait(), timeout=self.poll_interval
+                )
+            except asyncio.TimeoutError:
+                pass  # Normal - just means poll_interval elapsed
+
+        logger.info("Watch loop exited")
+
+    def _add_to_cache(self, file_id: str) -> None:
+        """Add file to in-memory cache with FIFO eviction."""
+        self._seen_files[file_id] = None
+        if len(self._seen_files) > self._max_seen_files:
+            # Remove oldest half (FIFO eviction)
+            evict_count = self._max_seen_files // 2
+            for _ in range(evict_count):
+                self._seen_files.popitem(last=False)
+            logger.debug("Evicted %d old entries from seen files cache", evict_count)
+
+    async def _has_clip_state(self, clip_id: str) -> bool:
+        """Check if clip_id exists in clip_states table.
+
+        Returns:
+            True if clip_state exists (any status including 'deleted'), False otherwise
+        """
+        if self._state_store is None:
+            return False
+
+        try:
+            # Direct async DB access - no threading complexity!
+            state = await asyncio.wait_for(
+                self._state_store.get(clip_id),
+                timeout=5.0,
+            )
+            return state is not None
+        except asyncio.TimeoutError:
+            logger.warning(
+                "DB query timeout for clip_states check: %s (assuming not seen)",
+                clip_id,
+            )
+            return False
+        except Exception as e:
+            logger.warning(
+                "Error checking clip_states for %s: %s (assuming not seen)",
+                clip_id,
+                e,
+                exc_info=True,
+            )
+            return False
+
+    def _create_clip(self, file_path: Path, mtime: float) -> Clip:
+        """Create Clip object from file path.
+
+        Args:
+            file_path: Path to the video file
+            mtime: File modification timestamp (from async stat)
+
+        Estimates timestamps based on file modification time.
+        """
+        mtime_dt = datetime.fromtimestamp(mtime)
+
+        # Estimate clip duration (assume 10s if we can't determine)
+        # In production, would parse from filename or video metadata
+        duration_s = 10.0
+
+        return Clip(
+            clip_id=file_path.stem,
+            camera_name=self.camera_name,
+            local_path=file_path,
+            start_ts=mtime_dt - timedelta(seconds=duration_s),
+            end_ts=mtime_dt,
+            duration_s=duration_s,
+            source_type="local_folder",
+        )