homesec 1.2.1__py3-none-any.whl → 1.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
homesec/sources/rtsp.py DELETED
@@ -1,1304 +0,0 @@
1
- """RTSP motion-detecting clip source.
2
-
3
- Ported from motion_recorder.py to keep functionality within homesec.
4
- """
5
-
6
- from __future__ import annotations
7
-
8
- import json
9
- import logging
10
- import os
11
- import platform
12
- import random
13
- import shlex
14
- import subprocess
15
- import time
16
- from dataclasses import dataclass
17
- from datetime import datetime, timedelta
18
- from pathlib import Path
19
- from queue import Empty, Full, Queue
20
- from threading import Event, Thread
21
- from typing import Any, cast
22
-
23
- import cv2
24
- import numpy as np
25
- import numpy.typing as npt
26
-
27
- from homesec.models.clip import Clip
28
- from homesec.models.source import RTSPSourceConfig
29
- from homesec.sources.base import ThreadedClipSource
30
-
31
- logger = logging.getLogger(__name__)
32
-
33
-
34
- @dataclass
35
- class HardwareAccelConfig:
36
- """Configuration for hardware-accelerated video decoding."""
37
-
38
- hwaccel: str | None
39
- hwaccel_device: str | None = None
40
-
41
- @property
42
- def is_available(self) -> bool:
43
- """Check if hardware acceleration is available."""
44
- return self.hwaccel is not None
45
-
46
-
47
- class HardwareAccelDetector:
48
- """Detect available hardware acceleration options for ffmpeg."""
49
-
50
- @staticmethod
51
- def detect(rtsp_url: str) -> HardwareAccelConfig:
52
- """Detect the best available hardware acceleration method."""
53
- if Path("/dev/dri/renderD128").exists():
54
- if HardwareAccelDetector._test_hwaccel("vaapi"):
55
- config = HardwareAccelConfig(
56
- hwaccel="vaapi",
57
- hwaccel_device="/dev/dri/renderD128",
58
- )
59
- if HardwareAccelDetector._test_decode(rtsp_url, config):
60
- return config
61
- logger.warning("VAAPI detected but failed to decode stream - disabling")
62
-
63
- if HardwareAccelDetector._check_nvidia():
64
- if HardwareAccelDetector._test_hwaccel("cuda"):
65
- config = HardwareAccelConfig(hwaccel="cuda")
66
- if HardwareAccelDetector._test_decode(rtsp_url, config):
67
- return config
68
- logger.warning("CUDA detected but failed to decode stream - disabling")
69
-
70
- if platform.system() == "Darwin":
71
- if HardwareAccelDetector._test_hwaccel("videotoolbox"):
72
- config = HardwareAccelConfig(hwaccel="videotoolbox")
73
- if HardwareAccelDetector._test_decode(rtsp_url, config):
74
- return config
75
- logger.warning("VideoToolbox detected but failed to decode stream - disabling")
76
-
77
- if HardwareAccelDetector._test_hwaccel("qsv"):
78
- config = HardwareAccelConfig(hwaccel="qsv")
79
- if HardwareAccelDetector._test_decode(rtsp_url, config):
80
- return config
81
- logger.warning("QSV detected but failed to decode stream - disabling")
82
-
83
- logger.info("Using software decoding (no working hardware acceleration found)")
84
- return HardwareAccelConfig(hwaccel=None)
85
-
86
- @staticmethod
87
- def _test_hwaccel(method: str) -> bool:
88
- """Test if a hardware acceleration method is available in ffmpeg."""
89
- try:
90
- result = subprocess.run(
91
- ["ffmpeg", "-hwaccels"],
92
- capture_output=True,
93
- text=True,
94
- timeout=2,
95
- check=False,
96
- )
97
- return method in result.stdout
98
- except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
99
- return False
100
-
101
- @staticmethod
102
- def _test_decode(rtsp_url: str, config: HardwareAccelConfig) -> bool:
103
- """Test if hardware acceleration works by decoding a few frames."""
104
- cmd = ["ffmpeg"]
105
-
106
- if config.hwaccel:
107
- cmd.extend(["-hwaccel", config.hwaccel])
108
- if config.hwaccel_device:
109
- cmd.extend(["-hwaccel_device", config.hwaccel_device])
110
-
111
- cmd.extend(
112
- [
113
- "-rtsp_transport",
114
- "tcp",
115
- "-i",
116
- rtsp_url,
117
- "-frames:v",
118
- "5",
119
- "-f",
120
- "null",
121
- "-",
122
- ]
123
- )
124
-
125
- try:
126
- result = subprocess.run(
127
- cmd,
128
- capture_output=True,
129
- timeout=10,
130
- text=True,
131
- check=False,
132
- )
133
- if result.returncode == 0:
134
- return True
135
- if any(
136
- err in result.stderr
137
- for err in (
138
- "No VA display found",
139
- "Device creation failed",
140
- "No device available for decoder",
141
- "Failed to initialise VAAPI",
142
- "Cannot load",
143
- )
144
- ):
145
- return False
146
- return result.returncode == 0
147
- except subprocess.TimeoutExpired:
148
- return True
149
- except Exception as exc:
150
- logger.warning("VAAPI check failed: %s", exc, exc_info=True)
151
- return False
152
-
153
- @staticmethod
154
- def _check_nvidia() -> bool:
155
- """Check if NVIDIA GPU is available."""
156
- try:
157
- subprocess.run(
158
- ["nvidia-smi"],
159
- capture_output=True,
160
- check=True,
161
- timeout=2,
162
- )
163
- return True
164
- except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired):
165
- return False
166
-
167
-
168
- class RTSPSource(ThreadedClipSource):
169
- """RTSP clip source with motion detection.
170
-
171
- Uses ffmpeg for frame extraction and recording; detects motion from
172
- downscaled grayscale frames and emits clips when recordings finish.
173
- """
174
-
175
- def __init__(self, config: RTSPSourceConfig, camera_name: str) -> None:
176
- """Initialize RTSP source."""
177
- super().__init__()
178
- rtsp_url = config.rtsp_url
179
- if config.rtsp_url_env:
180
- env_value = os.getenv(config.rtsp_url_env)
181
- if env_value:
182
- rtsp_url = env_value
183
- if not rtsp_url:
184
- raise ValueError("rtsp_url_env or rtsp_url required for RTSP source")
185
-
186
- self.rtsp_url = rtsp_url
187
-
188
- detect_rtsp_url = config.detect_rtsp_url
189
- if config.detect_rtsp_url_env:
190
- env_value = os.getenv(config.detect_rtsp_url_env)
191
- if env_value:
192
- detect_rtsp_url = env_value
193
-
194
- derived_detect = self._derive_detect_rtsp_url(self.rtsp_url)
195
- if detect_rtsp_url:
196
- self.detect_rtsp_url = detect_rtsp_url
197
- self._detect_rtsp_url_source = "explicit"
198
- elif derived_detect:
199
- self.detect_rtsp_url = derived_detect
200
- self._detect_rtsp_url_source = "derived_subtype=1"
201
- else:
202
- self.detect_rtsp_url = self.rtsp_url
203
- self._detect_rtsp_url_source = "same_as_rtsp_url"
204
-
205
- self.output_dir = Path(config.output_dir)
206
- sanitized_name = self._sanitize_camera_name(camera_name)
207
- self.camera_name = sanitized_name or camera_name
208
-
209
- self.pixel_threshold = int(config.pixel_threshold)
210
- self.min_changed_pct = float(config.min_changed_pct)
211
- self.blur_kernel = self._normalize_blur_kernel(config.blur_kernel)
212
- self.motion_stop_delay = float(config.stop_delay)
213
- self.max_recording_s = float(config.max_recording_s)
214
- self.max_reconnect_attempts = int(config.max_reconnect_attempts)
215
- self.debug_motion = bool(config.debug_motion)
216
- self.heartbeat_s = float(config.heartbeat_s)
217
- self.frame_timeout_s = float(config.frame_timeout_s)
218
- self.frame_queue_size = int(config.frame_queue_size)
219
- self.reconnect_backoff_s = float(config.reconnect_backoff_s)
220
- self.rtsp_connect_timeout_s = float(config.rtsp_connect_timeout_s)
221
- self.rtsp_io_timeout_s = float(config.rtsp_io_timeout_s)
222
- self.ffmpeg_flags = list(config.ffmpeg_flags)
223
-
224
- if config.disable_hwaccel:
225
- logger.info("Hardware acceleration manually disabled")
226
- self.hwaccel_config = HardwareAccelConfig(hwaccel=None)
227
- self._hwaccel_failed = True
228
- else:
229
- logger.info("Testing hardware acceleration with camera stream...")
230
- self.hwaccel_config = HardwareAccelDetector.detect(self.detect_rtsp_url)
231
- self._hwaccel_failed = not self.hwaccel_config.is_available
232
-
233
- self.output_dir.mkdir(parents=True, exist_ok=True)
234
-
235
- self.recording_process: subprocess.Popen[bytes] | None = None
236
- self.last_motion_time: float | None = None
237
- self.recording_start_time: float | None = None
238
- self.recording_start_wall: datetime | None = None
239
- self.output_file: Path | None = None
240
- self._stderr_log: Path | None = None
241
- self._recording_id: str | None = None
242
- self._stall_grace_until: float | None = None
243
-
244
- self._prev_motion_frame: npt.NDArray[np.uint8] | None = None
245
- self._last_changed_pct = 0.0
246
- self._last_changed_pixels = 0
247
- self._debug_frame_count = 0
248
-
249
- self.frame_pipe: subprocess.Popen[bytes] | None = None
250
- self._frame_pipe_stderr: Any | None = None
251
- self._frame_reader_thread: Thread | None = None
252
- self._frame_reader_stop: Event | None = None
253
- self._frame_queue: Queue[bytes] | None = None
254
- self._frame_width: int | None = None
255
- self._frame_height: int | None = None
256
- self._frame_size: int | None = None
257
- self.reconnect_count = 0
258
- self.last_successful_frame = self._last_heartbeat
259
-
260
- logger.info(
261
- "RTSPSource initialized: camera=%s, output_dir=%s",
262
- self.camera_name,
263
- self.output_dir,
264
- )
265
-
266
- def is_healthy(self) -> bool:
267
- """Check if source is healthy."""
268
- if not self._thread_is_healthy():
269
- return False
270
-
271
- age = time.monotonic() - self.last_successful_frame
272
- return age < (self.frame_timeout_s * 3)
273
-
274
- def _touch_heartbeat(self) -> None:
275
- self.last_successful_frame = time.monotonic()
276
- super()._touch_heartbeat()
277
-
278
- def _stop_timeout(self) -> float:
279
- return 10.0
280
-
281
- def _on_start(self) -> None:
282
- logger.info("Starting RTSPSource: %s", self.camera_name)
283
-
284
- def _on_stop(self) -> None:
285
- logger.info("Stopping RTSPSource...")
286
-
287
- def _on_stopped(self) -> None:
288
- logger.info("RTSPSource stopped")
289
-
290
- def _derive_detect_rtsp_url(self, rtsp_url: str) -> str | None:
291
- if "subtype=0" in rtsp_url:
292
- return rtsp_url.replace("subtype=0", "subtype=1")
293
- return None
294
-
295
- def _sanitize_camera_name(self, name: str | None) -> str | None:
296
- if not name:
297
- return None
298
- raw = str(name).strip()
299
- if not raw:
300
- return None
301
- out: list[str] = []
302
- for ch in raw:
303
- if ch.isalnum() or ch in ("-", "_"):
304
- out.append(ch)
305
- elif ch.isspace():
306
- out.append("_")
307
- else:
308
- out.append("_")
309
- cleaned = "".join(out).strip("_")
310
- while "__" in cleaned:
311
- cleaned = cleaned.replace("__", "_")
312
- return cleaned or None
313
-
314
- def _recording_prefix(self) -> str:
315
- if not self.camera_name:
316
- return ""
317
- return f"{self.camera_name}_"
318
-
319
- def _make_recording_paths(self, timestamp: str) -> tuple[Path, Path]:
320
- prefix = self._recording_prefix()
321
- output_file = self.output_dir / f"{prefix}motion_{timestamp}.mp4"
322
- stderr_log = self.output_dir / f"{prefix}recording_{timestamp}.log"
323
- return output_file, stderr_log
324
-
325
- def _telemetry_common_fields(self) -> dict[str, object]:
326
- return {
327
- "pixel_threshold": self.pixel_threshold,
328
- "min_changed_pct": self.min_changed_pct,
329
- "blur_kernel": self.blur_kernel,
330
- "stop_delay": self.motion_stop_delay,
331
- "max_recording_s": self.max_recording_s,
332
- "hwaccel": self.hwaccel_config.hwaccel,
333
- "hwaccel_device": self.hwaccel_config.hwaccel_device,
334
- }
335
-
336
- def _event_extra(self, event_type: str, **fields: object) -> dict[str, object]:
337
- extra: dict[str, object] = {
338
- "kind": "event",
339
- "event_type": event_type,
340
- "camera_name": self.camera_name,
341
- "recording_id": self._recording_id,
342
- }
343
- extra.update(self._telemetry_common_fields())
344
- extra.update(fields)
345
- return extra
346
-
347
- def _probe_stream_info(self, rtsp_url: str) -> dict[str, object] | None:
348
- cmd = [
349
- "ffprobe",
350
- "-v",
351
- "error",
352
- "-select_streams",
353
- "v:0",
354
- "-show_entries",
355
- "stream=codec_name,width,height,avg_frame_rate",
356
- "-of",
357
- "json",
358
- "-rtsp_transport",
359
- "tcp",
360
- rtsp_url,
361
- ]
362
- try:
363
- result = subprocess.run(cmd, capture_output=True, text=True, timeout=10, check=False)
364
- data = json.loads(result.stdout)
365
- if not data.get("streams"):
366
- return None
367
- stream = data["streams"][0]
368
- return {
369
- "codec_name": stream.get("codec_name"),
370
- "width": stream.get("width"),
371
- "height": stream.get("height"),
372
- "avg_frame_rate": stream.get("avg_frame_rate"),
373
- }
374
- except Exception as exc:
375
- logger.warning("Failed to probe stream info: %s", exc, exc_info=True)
376
- return None
377
-
378
- def _redact_rtsp_url(self, url: str) -> str:
379
- if "://" not in url:
380
- return url
381
- scheme, rest = url.split("://", 1)
382
- if "@" not in rest:
383
- return url
384
- _creds, host = rest.split("@", 1)
385
- return f"{scheme}://***:***@{host}"
386
-
387
- def _format_cmd(self, cmd: list[str]) -> str:
388
- try:
389
- return shlex.join([str(x) for x in cmd])
390
- except Exception as exc:
391
- logger.warning("Failed to format command with shlex.join: %s", exc, exc_info=True)
392
- return " ".join([str(x) for x in cmd])
393
-
394
- def detect_motion(self, frame: npt.NDArray[np.uint8]) -> bool:
395
- """Return True if motion detected in frame."""
396
- if frame.ndim == 3:
397
- gray = cast(npt.NDArray[np.uint8], cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY))
398
- else:
399
- gray = frame
400
-
401
- if self.blur_kernel > 1:
402
- gray = cast(
403
- npt.NDArray[np.uint8],
404
- cv2.GaussianBlur(gray, (self.blur_kernel, self.blur_kernel), 0),
405
- )
406
-
407
- if self._prev_motion_frame is None:
408
- self._prev_motion_frame = gray
409
- self._last_changed_pct = 0.0
410
- self._last_changed_pixels = 0
411
- return False
412
-
413
- diff = cv2.absdiff(self._prev_motion_frame, gray)
414
- _, mask = cv2.threshold(diff, self.pixel_threshold, 255, cv2.THRESH_BINARY)
415
- changed_pixels = int(cv2.countNonZero(mask))
416
-
417
- total_pixels = int(gray.shape[0]) * int(gray.shape[1])
418
- changed_pct = (changed_pixels / total_pixels * 100.0) if total_pixels else 0.0
419
-
420
- self._prev_motion_frame = gray
421
- self._last_changed_pct = changed_pct
422
- self._last_changed_pixels = changed_pixels
423
-
424
- motion = changed_pct >= self.min_changed_pct
425
-
426
- if self.debug_motion:
427
- self._debug_frame_count += 1
428
- if self._debug_frame_count % 100 == 0:
429
- logger.debug(
430
- "Motion check: changed_pct=%.3f%% changed_px=%s pixel_threshold=%s min_changed_pct=%.3f%% blur=%s",
431
- changed_pct,
432
- changed_pixels,
433
- self.pixel_threshold,
434
- self.min_changed_pct,
435
- self.blur_kernel,
436
- )
437
-
438
- return motion
439
-
440
- def check_recording_health(self) -> bool:
441
- """Check if recording process is still alive."""
442
- if self.recording_process and self.recording_process.poll() is not None:
443
- proc = self.recording_process
444
- output_file = self.output_file
445
- exit_code = proc.returncode
446
- logger.warning("Recording process died unexpectedly (exit code: %s)", exit_code)
447
- if output_file:
448
- logger.error(
449
- "Recording process died",
450
- extra=self._event_extra(
451
- "recording_process_died",
452
- recording_id=output_file.name,
453
- recording_path=str(output_file),
454
- exit_code=exit_code,
455
- pid=proc.pid,
456
- ),
457
- )
458
-
459
- log_file: Path | None = None
460
- if self._stderr_log and self._stderr_log.exists():
461
- log_file = self._stderr_log
462
- elif output_file:
463
- stem = output_file.stem
464
- if "motion_" in stem:
465
- prefix_part, timestamp = stem.split("motion_", 1)
466
- candidate = output_file.parent / f"{prefix_part}recording_{timestamp}.log"
467
- if candidate.exists():
468
- log_file = candidate
469
-
470
- if log_file:
471
- try:
472
- with open(log_file) as f:
473
- error_lines = f.read()
474
- if error_lines:
475
- logger.warning("Recording error log:\n%s", error_lines[-1000:])
476
- except Exception as e:
477
- logger.warning("Could not read log file: %s", e, exc_info=True)
478
-
479
- start_wall = self.recording_start_wall
480
- start_mono = self.recording_start_time
481
- self.recording_process = None
482
- self.output_file = None
483
- self._stderr_log = None
484
- self.recording_start_time = None
485
- self.recording_start_wall = None
486
- self._recording_id = None
487
-
488
- self._stop_recording_process(proc, output_file)
489
- self._finalize_clip(output_file, start_wall, start_mono)
490
- return False
491
- return True
492
-
493
- def start_recording(self) -> None:
494
- """Start ffmpeg recording process with audio."""
495
- if self.recording_process:
496
- return
497
-
498
- self.output_dir.mkdir(parents=True, exist_ok=True)
499
- timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
500
- output_file, stderr_log = self._make_recording_paths(timestamp)
501
-
502
- proc = self._spawn_recording_process(output_file, stderr_log)
503
- if not proc:
504
- logger.error(
505
- "Recording failed to start",
506
- extra=self._event_extra(
507
- "recording_start_error",
508
- recording_id=output_file.name,
509
- recording_path=str(output_file),
510
- stderr_log=str(stderr_log),
511
- ),
512
- )
513
- return
514
-
515
- self.recording_process = proc
516
- self.output_file = output_file
517
- self._stderr_log = stderr_log
518
- self.recording_start_time = time.monotonic()
519
- self.recording_start_wall = datetime.now()
520
- self._recording_id = output_file.name
521
-
522
- logger.info("Started recording: %s (PID: %s)", output_file, proc.pid)
523
- logger.debug("Recording logs: %s", stderr_log)
524
- logger.info(
525
- "Recording started",
526
- extra=self._event_extra(
527
- "recording_start",
528
- recording_id=output_file.name,
529
- recording_path=str(output_file),
530
- stderr_log=str(stderr_log),
531
- pid=proc.pid,
532
- detect_stream_source=getattr(self, "_detect_rtsp_url_source", None),
533
- detect_stream_is_same=(self.detect_rtsp_url == self.rtsp_url),
534
- ),
535
- )
536
-
537
- def _spawn_recording_process(
538
- self, output_file: Path, stderr_log: Path
539
- ) -> subprocess.Popen[bytes] | None:
540
- cmd = [
541
- "ffmpeg",
542
- "-rtsp_transport",
543
- "tcp",
544
- "-rtsp_flags",
545
- "prefer_tcp",
546
- "-user_agent",
547
- "Lavf",
548
- "-i",
549
- self.rtsp_url,
550
- "-c",
551
- "copy",
552
- "-f",
553
- "mp4",
554
- "-y",
555
- ]
556
-
557
- user_flags = self.ffmpeg_flags
558
-
559
- # Naive check to see if user overrode defaults
560
- # If user supplies ANY -loglevel, we don't add ours.
561
- # If user supplies ANY -fflags, we don't add ours (to avoid concatenation complexity).
562
- # This allows full user control.
563
- has_loglevel = any(x == "-loglevel" for x in user_flags)
564
- if not has_loglevel:
565
- cmd.extend(["-loglevel", "warning"])
566
-
567
- has_fflags = any(x == "-fflags" for x in user_flags)
568
- if not has_fflags:
569
- cmd.extend(["-fflags", "+genpts+igndts"])
570
-
571
- has_fps_mode = any(x == "-fps_mode" or x == "-vsync" for x in user_flags)
572
- if not has_fps_mode:
573
- cmd.extend(["-vsync", "0"])
574
-
575
- # Add user flags last so they can potentially override or add to the above
576
- cmd.extend(user_flags)
577
-
578
- cmd.extend([str(output_file)])
579
-
580
- safe_cmd = list(cmd)
581
- try:
582
- idx = safe_cmd.index("-i")
583
- safe_cmd[idx + 1] = self._redact_rtsp_url(str(safe_cmd[idx + 1]))
584
- except Exception as exc:
585
- logger.warning("Failed to redact recording RTSP URL: %s", exc, exc_info=True)
586
- logger.debug("Recording ffmpeg: %s", self._format_cmd(safe_cmd))
587
-
588
- try:
589
- with open(stderr_log, "w") as stderr_file:
590
- proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=stderr_file)
591
-
592
- time.sleep(0.5)
593
- if proc.poll() is not None:
594
- logger.error("Recording process died immediately (exit code: %s)", proc.returncode)
595
- logger.error("Check logs at: %s", stderr_log)
596
- try:
597
- with open(stderr_log) as f:
598
- error_lines = f.read()
599
- if error_lines:
600
- logger.error("Error output: %s", error_lines[:500])
601
- except Exception:
602
- logger.exception("Failed reading recording log: %s", stderr_log)
603
- return None
604
-
605
- return proc
606
- except Exception:
607
- logger.exception("Failed to start recording")
608
- return None
609
-
610
- def stop_recording(self) -> None:
611
- """Stop ffmpeg recording process."""
612
- if not self.recording_process:
613
- return
614
-
615
- proc = self.recording_process
616
- output_file = self.output_file
617
- started_at = self.recording_start_time
618
- started_wall = self.recording_start_wall
619
- self.recording_process = None
620
- self.output_file = None
621
- self._stderr_log = None
622
- self.recording_start_time = None
623
- self.recording_start_wall = None
624
- self._recording_id = None
625
-
626
- self._stop_recording_process(proc, output_file)
627
- self._finalize_clip(output_file, started_wall, started_at)
628
-
629
- if output_file:
630
- duration_s = (time.monotonic() - started_at) if started_at else None
631
- logger.info(
632
- "Recording stopped",
633
- extra=self._event_extra(
634
- "recording_stop",
635
- recording_id=output_file.name,
636
- recording_path=str(output_file),
637
- duration_s=duration_s,
638
- last_changed_pct=getattr(self, "_last_changed_pct", None),
639
- last_changed_pixels=getattr(self, "_last_changed_pixels", None),
640
- ),
641
- )
642
-
643
- def _stop_recording_process(
644
- self, proc: subprocess.Popen[bytes], output_file: Path | None
645
- ) -> None:
646
- try:
647
- if proc.poll() is None:
648
- proc.terminate()
649
- proc.wait(timeout=5)
650
- except subprocess.TimeoutExpired:
651
- logger.warning("Recording process did not terminate, killing (PID: %s)", proc.pid)
652
- try:
653
- proc.kill()
654
- proc.wait(timeout=2)
655
- except Exception:
656
- logger.exception("Failed to kill recording process (PID: %s)", proc.pid)
657
- except Exception:
658
- logger.exception("Failed while stopping recording process (PID: %s)", proc.pid)
659
-
660
- logger.debug(
661
- "Stopped recording: %s",
662
- output_file,
663
- extra={"recording_id": output_file.name if output_file else None},
664
- )
665
-
666
- def _rotate_recording_if_needed(self) -> None:
667
- if not self.recording_process or not self.recording_start_time:
668
- return
669
-
670
- now = time.monotonic()
671
- if (now - self.recording_start_time) < self.max_recording_s:
672
- return
673
-
674
- if not self.last_motion_time:
675
- return
676
-
677
- if (now - self.last_motion_time) > self.motion_stop_delay:
678
- return
679
-
680
- old_proc = self.recording_process
681
- old_output = self.output_file
682
- old_started_wall = self.recording_start_wall
683
- old_started_mono = self.recording_start_time
684
-
685
- timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
686
- new_output, new_log = self._make_recording_paths(timestamp)
687
-
688
- logger.info(
689
- "Max recording length reached (%.1fs), rotating to: %s",
690
- self.max_recording_s,
691
- new_output,
692
- )
693
-
694
- new_proc = self._spawn_recording_process(new_output, new_log)
695
- if new_proc:
696
- self.last_motion_time = now
697
- self.recording_process = new_proc
698
- self.output_file = new_output
699
- self._stderr_log = new_log
700
- self.recording_start_time = now
701
- self.recording_start_wall = datetime.now()
702
- self._recording_id = new_output.name
703
-
704
- if old_output:
705
- logger.info(
706
- "Recording rotated",
707
- extra=self._event_extra(
708
- "recording_rotate",
709
- recording_id=old_output.name,
710
- recording_path=str(old_output),
711
- new_recording_id=new_output.name,
712
- new_recording_path=str(new_output),
713
- ),
714
- )
715
-
716
- self._stop_recording_process(old_proc, old_output)
717
- self._finalize_clip(old_output, old_started_wall, old_started_mono)
718
- return
719
-
720
- logger.warning("Rotation start failed; stopping current recording and retrying")
721
- self.recording_process = None
722
- self.output_file = None
723
- self.recording_start_time = None
724
- self.recording_start_wall = None
725
- self._recording_id = None
726
- self._stop_recording_process(old_proc, old_output)
727
- self._finalize_clip(old_output, old_started_wall, old_started_mono)
728
- self.start_recording()
729
-
730
- def probe_stream_resolution(self) -> tuple[int, int]:
731
- """Probe RTSP stream to get native resolution."""
732
- cmd = [
733
- "ffprobe",
734
- "-v",
735
- "error",
736
- "-select_streams",
737
- "v:0",
738
- "-show_entries",
739
- "stream=width,height",
740
- "-of",
741
- "json",
742
- "-rtsp_transport",
743
- "tcp",
744
- self.rtsp_url,
745
- ]
746
- try:
747
- result = subprocess.run(cmd, capture_output=True, text=True, timeout=10, check=False)
748
- data = json.loads(result.stdout)
749
- width = data["streams"][0]["width"]
750
- height = data["streams"][0]["height"]
751
- return int(width), int(height)
752
- except Exception as e:
753
- logger.warning(
754
- "Failed to probe stream, using default 1920x1080: %s",
755
- e,
756
- exc_info=True,
757
- )
758
- return 1920, 1080
759
-
760
- def get_frame_pipe(self) -> tuple[subprocess.Popen[bytes], Any, int, int]:
761
- """Create ffmpeg process to get frames for motion detection."""
762
- detect_width, detect_height = 320, 240
763
-
764
- stderr_log = self.output_dir / "frame_pipeline.log"
765
- self.output_dir.mkdir(parents=True, exist_ok=True)
766
-
767
- def _read_tail(path: Path, max_bytes: int = 4000) -> str:
768
- try:
769
- data = path.read_bytes()
770
- except Exception as exc:
771
- logger.warning("Failed to read stderr tail: %s", exc, exc_info=True)
772
- return ""
773
- if len(data) <= max_bytes:
774
- return data.decode(errors="replace")
775
- return data[-max_bytes:].decode(errors="replace")
776
-
777
- cmd = ["ffmpeg"]
778
-
779
- # 1. Global Flags (Hardware Acceleration)
780
- if self.hwaccel_config.is_available and not self._hwaccel_failed:
781
- hwaccel = self.hwaccel_config.hwaccel
782
- if hwaccel is not None:
783
- cmd.extend(["-hwaccel", hwaccel])
784
- if self.hwaccel_config.hwaccel_device:
785
- cmd.extend(["-hwaccel_device", self.hwaccel_config.hwaccel_device])
786
- elif self._hwaccel_failed:
787
- logger.info("Hardware acceleration disabled due to previous failures")
788
-
789
- # 2. Global Flags (Robustness & Logging)
790
- user_flags = self.ffmpeg_flags
791
-
792
- has_loglevel = any(x == "-loglevel" for x in user_flags)
793
- if not has_loglevel:
794
- cmd.extend(["-loglevel", "warning"])
795
-
796
- has_fflags = any(x == "-fflags" for x in user_flags)
797
- if not has_fflags:
798
- cmd.extend(["-fflags", "+genpts+igndts"])
799
-
800
- # Add all user flags to global scope.
801
- # Users who want input-specific flags (before -i) must rely on ffmpeg parsing them correctly
802
- # or we would need a more complex config structure.
803
- # For now, most robustness flags (-re, -rtsp_transport, etc) work as global or are handled below.
804
- cmd.extend(user_flags)
805
-
806
- base_input_args = [
807
- "-rtsp_transport",
808
- "tcp",
809
- "-i",
810
- self.detect_rtsp_url,
811
- "-f",
812
- "rawvideo",
813
- "-pix_fmt",
814
- "gray",
815
- "-vf",
816
- f"fps=10,scale={detect_width}:{detect_height}",
817
- "-an",
818
- "-",
819
- ]
820
-
821
- timeout_us_connect = str(int(max(0.1, self.rtsp_connect_timeout_s) * 1_000_000))
822
- attempts: list[tuple[str, list[str]]] = [
823
- (
824
- "stimeout",
825
- ["-stimeout", timeout_us_connect] + base_input_args,
826
- ),
827
- ("stimeout", ["-stimeout", timeout_us_connect] + base_input_args),
828
- ("no_timeouts", base_input_args),
829
- ]
830
-
831
- process: subprocess.Popen[bytes] | None = None
832
- stderr_file: Any | None = None
833
-
834
- for label, extra_args in attempts:
835
- cmd_attempt = list(cmd) + extra_args
836
- logger.debug("Starting frame pipeline (%s), logging to: %s", label, stderr_log)
837
- safe_cmd = list(cmd_attempt)
838
- try:
839
- idx = safe_cmd.index("-i")
840
- safe_cmd[idx + 1] = self._redact_rtsp_url(str(safe_cmd[idx + 1]))
841
- except Exception as exc:
842
- logger.warning(
843
- "Failed to redact frame pipeline RTSP URL: %s",
844
- exc,
845
- exc_info=True,
846
- )
847
- logger.debug("Frame pipeline ffmpeg (%s): %s", label, self._format_cmd(safe_cmd))
848
-
849
- try:
850
- stderr_file = open(stderr_log, "w")
851
- process = subprocess.Popen(
852
- cmd_attempt,
853
- stdout=subprocess.PIPE,
854
- stderr=stderr_file,
855
- bufsize=detect_width * detect_height,
856
- )
857
- except Exception:
858
- try:
859
- if stderr_file:
860
- stderr_file.close()
861
- except Exception as exc:
862
- logger.warning("Failed to close stderr log: %s", exc, exc_info=True)
863
- logger.exception("Failed to start frame pipeline subprocess (%s)", label)
864
- continue
865
-
866
- time.sleep(1)
867
- if process.poll() is None:
868
- return process, stderr_file, detect_width, detect_height
869
-
870
- try:
871
- stderr_file.close()
872
- except Exception as exc:
873
- logger.warning("Failed to close stderr log: %s", exc, exc_info=True)
874
- stderr_tail = _read_tail(stderr_log)
875
- logger.error(
876
- "Frame pipeline died immediately (%s, exit code: %s)", label, process.returncode
877
- )
878
- if stderr_tail:
879
- logger.error("Frame pipeline stderr tail (%s):\n%s", label, stderr_tail)
880
- process = None
881
- stderr_file = None
882
- continue
883
-
884
- raise RuntimeError("Frame pipeline failed to start")
885
-
886
- def _stop_process(
887
- self, proc: subprocess.Popen[bytes], name: str, terminate_timeout_s: float
888
- ) -> None:
889
- if proc.poll() is not None:
890
- return
891
- try:
892
- proc.terminate()
893
- proc.wait(timeout=terminate_timeout_s)
894
- except subprocess.TimeoutExpired:
895
- logger.warning("%s did not terminate, killing (PID: %s)", name, proc.pid)
896
- proc.kill()
897
- try:
898
- proc.wait(timeout=2)
899
- except Exception:
900
- logger.exception("Failed waiting after kill for %s (PID: %s)", name, proc.pid)
901
-
902
- def _stop_frame_pipeline(self) -> None:
903
- if self._frame_reader_stop is not None:
904
- self._frame_reader_stop.set()
905
-
906
- if self.frame_pipe is not None:
907
- try:
908
- self._stop_process(self.frame_pipe, "Frame pipeline", terminate_timeout_s=2)
909
- except Exception:
910
- logger.exception("Error stopping frame pipeline process")
911
- self.frame_pipe = None
912
-
913
- if self._frame_reader_thread is not None:
914
- try:
915
- self._frame_reader_thread.join(timeout=5)
916
- except Exception:
917
- logger.exception("Error joining frame reader thread")
918
- self._frame_reader_thread = None
919
-
920
- self._frame_reader_stop = None
921
-
922
- if self._frame_pipe_stderr is not None:
923
- try:
924
- self._frame_pipe_stderr.close()
925
- except Exception:
926
- logger.exception("Error closing frame pipeline stderr log")
927
- self._frame_pipe_stderr = None
928
-
929
- self._frame_queue = None
930
- self._frame_width = None
931
- self._frame_height = None
932
- self._frame_size = None
933
-
934
- def _start_frame_pipeline(self) -> None:
935
- self._stop_frame_pipeline()
936
-
937
- self.frame_pipe, self._frame_pipe_stderr, self._frame_width, self._frame_height = (
938
- self.get_frame_pipe()
939
- )
940
- self._frame_size = int(self._frame_width) * int(self._frame_height)
941
- self._frame_queue = Queue(maxsize=self.frame_queue_size)
942
- self._frame_reader_stop = Event()
943
-
944
- frame_pipe = self.frame_pipe
945
- frame_size = self._frame_size
946
- frame_queue = self._frame_queue
947
- stop_event = self._frame_reader_stop
948
-
949
- def reader_loop() -> None:
950
- stdout = frame_pipe.stdout if frame_pipe else None
951
- if stdout is None:
952
- logger.error("Frame pipeline stdout is None")
953
- return
954
-
955
- while not stop_event.is_set():
956
- try:
957
- raw = stdout.read(frame_size)
958
- except Exception:
959
- logger.exception("Error reading from frame pipeline stdout")
960
- return
961
-
962
- if not raw or len(raw) != frame_size:
963
- return
964
-
965
- self._touch_heartbeat()
966
-
967
- try:
968
- frame_queue.put_nowait(raw)
969
- except Full:
970
- try:
971
- _ = frame_queue.get_nowait()
972
- except Empty:
973
- pass
974
- try:
975
- frame_queue.put_nowait(raw)
976
- except Full:
977
- pass
978
-
979
- self._frame_reader_thread = Thread(target=reader_loop, name="frame-reader", daemon=True)
980
- self._frame_reader_thread.start()
981
-
982
- def _wait_for_first_frame(self, timeout_s: float) -> bool:
983
- if not self._frame_queue:
984
- return False
985
- try:
986
- raw = self._frame_queue.get(timeout=float(timeout_s))
987
- except Empty:
988
- return False
989
- try:
990
- self._frame_queue.put_nowait(raw)
991
- except Full:
992
- pass
993
- return True
994
-
995
- def _reconnect_frame_pipeline(self, *, aggressive: bool) -> bool:
996
- initial_backoff_s = 0.2 if aggressive else float(self.reconnect_backoff_s)
997
- backoff_s = initial_backoff_s
998
- backoff_cap_s = 10.0 if aggressive else float(self.reconnect_backoff_s)
999
-
1000
- first_attempt = True
1001
- while not self._stop_event.is_set():
1002
- self.reconnect_count += 1
1003
-
1004
- if self.max_reconnect_attempts == 0:
1005
- logger.warning(
1006
- "Reconnect attempt %s (mode=%s max=inf)...",
1007
- self.reconnect_count,
1008
- "aggressive" if aggressive else "normal",
1009
- )
1010
- else:
1011
- logger.warning(
1012
- "Reconnect attempt %s/%s (mode=%s)...",
1013
- self.reconnect_count,
1014
- self.max_reconnect_attempts,
1015
- "aggressive" if aggressive else "normal",
1016
- )
1017
- if self.reconnect_count >= self.max_reconnect_attempts:
1018
- logger.error("Max reconnect attempts reached. Camera may be offline.")
1019
- return False
1020
-
1021
- if first_attempt and aggressive:
1022
- sleep_s = 0.0
1023
- else:
1024
- jitter = random.uniform(0.0, min(0.25, backoff_s * 0.25))
1025
- sleep_s = backoff_s + jitter
1026
- first_attempt = False
1027
-
1028
- if sleep_s > 0:
1029
- logger.debug("Waiting %.2fs before reconnect...", sleep_s)
1030
- time.sleep(sleep_s)
1031
-
1032
- try:
1033
- self._start_frame_pipeline()
1034
- except Exception as exc:
1035
- logger.warning("Failed to restart frame pipeline: %s", exc, exc_info=True)
1036
- if aggressive:
1037
- backoff_s = min(backoff_s * 1.6, backoff_cap_s)
1038
- continue
1039
-
1040
- startup_timeout_s = min(2.0, max(0.5, float(self.frame_timeout_s)))
1041
- if self._wait_for_first_frame(startup_timeout_s):
1042
- logger.info("Reconnected successfully")
1043
- self.reconnect_count = 0
1044
- return True
1045
-
1046
- logger.warning("Frame pipeline restarted but still no frames; retrying...")
1047
- try:
1048
- self._stop_frame_pipeline()
1049
- except Exception as exc:
1050
- logger.warning(
1051
- "Failed to stop frame pipeline after reconnect: %s",
1052
- exc,
1053
- exc_info=True,
1054
- )
1055
- if aggressive:
1056
- backoff_s = min(backoff_s * 1.6, backoff_cap_s)
1057
-
1058
- return False
1059
-
1060
- def cleanup(self) -> None:
1061
- """Clean up resources."""
1062
- logger.info("Cleaning up...")
1063
- try:
1064
- self.stop_recording()
1065
- except Exception:
1066
- logger.exception("Error stopping recording")
1067
- try:
1068
- self._stop_frame_pipeline()
1069
- except Exception:
1070
- logger.exception("Error stopping frame pipeline")
1071
-
1072
- def _finalize_clip(
1073
- self,
1074
- output_file: Path | None,
1075
- started_wall: datetime | None,
1076
- started_mono: float | None,
1077
- ) -> None:
1078
- if output_file is None:
1079
- return
1080
- try:
1081
- if not output_file.exists() or output_file.stat().st_size == 0:
1082
- return
1083
- except Exception as exc:
1084
- logger.warning(
1085
- "Failed to stat output clip %s: %s",
1086
- output_file,
1087
- exc,
1088
- exc_info=True,
1089
- )
1090
- return
1091
-
1092
- end_ts = datetime.now()
1093
- if started_wall is None:
1094
- if started_mono is not None:
1095
- duration_s = time.monotonic() - started_mono
1096
- start_ts = end_ts - timedelta(seconds=duration_s)
1097
- else:
1098
- start_ts = end_ts
1099
- duration_s = 0.0
1100
- else:
1101
- start_ts = started_wall
1102
- duration_s = (end_ts - start_ts).total_seconds()
1103
-
1104
- clip = Clip(
1105
- clip_id=output_file.stem,
1106
- camera_name=self.camera_name,
1107
- local_path=output_file,
1108
- start_ts=start_ts,
1109
- end_ts=end_ts,
1110
- duration_s=duration_s,
1111
- source_type="rtsp",
1112
- )
1113
-
1114
- self._emit_clip(clip)
1115
-
1116
- @staticmethod
1117
- def _normalize_blur_kernel(blur_kernel: int) -> int:
1118
- kernel = int(blur_kernel)
1119
- if kernel < 0:
1120
- return 0
1121
- if kernel % 2 == 0 and kernel != 0:
1122
- return kernel + 1
1123
- return kernel
1124
-
1125
- def _log_startup_info(self) -> None:
1126
- logger.info("Connecting to camera...")
1127
- logger.info(
1128
- "Motion: pixel_threshold=%s min_changed_pct=%.3f%% blur=%s stop_delay=%.1fs max_recording_s=%.1fs",
1129
- self.pixel_threshold,
1130
- self.min_changed_pct,
1131
- self.blur_kernel,
1132
- self.motion_stop_delay,
1133
- self.max_recording_s,
1134
- )
1135
- logger.info("Max reconnect attempts: %s", self.max_reconnect_attempts)
1136
- if self.max_reconnect_attempts == 0:
1137
- logger.info("Reconnect policy: retry forever")
1138
-
1139
- if self.hwaccel_config.is_available:
1140
- device_info = (
1141
- f" (device: {self.hwaccel_config.hwaccel_device})"
1142
- if self.hwaccel_config.hwaccel_device
1143
- else ""
1144
- )
1145
- logger.info("Hardware acceleration: %s%s", self.hwaccel_config.hwaccel, device_info)
1146
- else:
1147
- logger.info("Hardware acceleration: disabled (using software decoding)")
1148
-
1149
- logger.info("Detecting camera resolution...")
1150
- native_width, native_height = self.probe_stream_resolution()
1151
- logger.info("Camera resolution: %sx%s", native_width, native_height)
1152
-
1153
- if self.detect_rtsp_url != self.rtsp_url:
1154
- info = self._probe_stream_info(self.detect_rtsp_url)
1155
- if info and info.get("width") and info.get("height"):
1156
- logger.info(
1157
- "Motion RTSP stream (%s) available: %s (%sx%s @ %s)",
1158
- self._detect_rtsp_url_source,
1159
- self._redact_rtsp_url(self.detect_rtsp_url),
1160
- info.get("width"),
1161
- info.get("height"),
1162
- info.get("avg_frame_rate"),
1163
- )
1164
- else:
1165
- logger.warning(
1166
- "Motion RTSP stream (%s) did not probe cleanly; falling back to main RTSP stream",
1167
- self._detect_rtsp_url_source,
1168
- )
1169
- self.detect_rtsp_url = self.rtsp_url
1170
- self._detect_rtsp_url_source = "fallback_to_rtsp_url"
1171
-
1172
- logger.info("Motion detection: 320x240@10fps (downscaled for efficiency)")
1173
-
1174
- def _handle_frame_timeout(self) -> bool:
1175
- pipe_status = self.frame_pipe.poll() if self.frame_pipe else None
1176
- if pipe_status is not None:
1177
- logger.error(
1178
- "Frame pipeline exited (code: %s). Check logs: %s/frame_pipeline.log",
1179
- pipe_status,
1180
- self.output_dir,
1181
- )
1182
- else:
1183
- logger.warning(
1184
- "No frames received for %.1fs (stall). Last frame %.1fs ago.",
1185
- self.frame_timeout_s,
1186
- time.monotonic() - self.last_successful_frame,
1187
- )
1188
-
1189
- aggressive = self.recording_process is None
1190
- return self._reconnect_frame_pipeline(aggressive=aggressive)
1191
-
1192
- def _stall_grace_remaining(self, now: float) -> float:
1193
- if not self.recording_process or not self.last_motion_time:
1194
- return 0.0
1195
- return max(0.0, self.motion_stop_delay - (now - self.last_motion_time))
1196
-
1197
- def _run(self) -> None:
1198
- """Start monitoring camera for motion and recording videos."""
1199
- self._log_startup_info()
1200
-
1201
- try:
1202
- self._start_frame_pipeline()
1203
- logger.info("Connected, monitoring for motion...")
1204
- self.reconnect_count = 0
1205
-
1206
- frame_count = 0
1207
- last_heartbeat = time.monotonic()
1208
-
1209
- while not self._stop_event.is_set():
1210
- if time.monotonic() - last_heartbeat > self.heartbeat_s:
1211
- logger.debug(
1212
- "[HEARTBEAT] Processed %s frames, recording=%s",
1213
- frame_count,
1214
- self.recording_process is not None,
1215
- )
1216
- if self.frame_pipe and self.frame_pipe.poll() is not None:
1217
- logger.error(
1218
- "Frame pipeline died! Exit code: %s", self.frame_pipe.returncode
1219
- )
1220
- break
1221
- last_heartbeat = time.monotonic()
1222
-
1223
- frame_count += 1
1224
-
1225
- if (
1226
- not self._frame_queue
1227
- or not self._frame_width
1228
- or not self._frame_height
1229
- or not self._frame_size
1230
- ):
1231
- break
1232
-
1233
- try:
1234
- raw_frame = self._frame_queue.get(timeout=self.frame_timeout_s)
1235
- except Empty:
1236
- now = time.monotonic()
1237
- if self._stall_grace_until is not None and now < self._stall_grace_until:
1238
- time.sleep(min(0.5, self._stall_grace_until - now))
1239
- continue
1240
-
1241
- if not self._handle_frame_timeout():
1242
- remaining = self._stall_grace_remaining(now)
1243
- if remaining > 0:
1244
- self._stall_grace_until = now + remaining
1245
- logger.warning(
1246
- "Frame pipeline stalled; keeping recording alive for %.1fs",
1247
- remaining,
1248
- )
1249
- time.sleep(min(0.5, remaining))
1250
- continue
1251
- break
1252
- continue
1253
-
1254
- self._stall_grace_until = None
1255
- frame = np.frombuffer(raw_frame, dtype=np.uint8).reshape(
1256
- (self._frame_height, self._frame_width)
1257
- )
1258
- now = time.monotonic()
1259
- motion_detected = self.detect_motion(frame)
1260
-
1261
- if motion_detected:
1262
- logger.debug(
1263
- "[MOTION DETECTED at frame %s] changed_pct=%.3f%%",
1264
- frame_count,
1265
- self._last_changed_pct,
1266
- )
1267
- self.last_motion_time = now
1268
- if not self.recording_process:
1269
- logger.debug("-> Starting new recording...")
1270
- self.start_recording()
1271
- if not self.recording_process:
1272
- logger.error("-> Recording failed to start!")
1273
- else:
1274
- if not self.check_recording_health():
1275
- logger.warning("-> Recording was dead, trying to restart...")
1276
- self.recording_process = None
1277
- self.start_recording()
1278
-
1279
- if self.recording_process and self.last_motion_time:
1280
- keepalive_threshold = self.min_changed_pct * 0.5
1281
- if self._last_changed_pct >= keepalive_threshold:
1282
- self.last_motion_time = now
1283
-
1284
- if self.recording_process and self.last_motion_time:
1285
- if not self.check_recording_health():
1286
- logger.warning("Recording died, stopping...")
1287
- self.stop_recording()
1288
- self.last_motion_time = None
1289
- elif now - self.last_motion_time > self.motion_stop_delay:
1290
- logger.info(
1291
- "No motion for %.1fs > stop_delay=%.1fs (last changed_pct=%.3f%%), stopping",
1292
- now - self.last_motion_time,
1293
- self.motion_stop_delay,
1294
- self._last_changed_pct,
1295
- )
1296
- self.stop_recording()
1297
- self.last_motion_time = None
1298
- else:
1299
- self._rotate_recording_if_needed()
1300
-
1301
- except Exception as e:
1302
- logger.exception("Unexpected error: %s", e)
1303
- finally:
1304
- self.cleanup()