scrypted-detection-trainer 0.1.9 → 0.1.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrypted-detection-trainer",
-  "version": "0.1.9",
+  "version": "0.1.11",
   "description": "Collect and label detections to fine-tune the Scrypted NVR object detection model.",
   "keywords": [
     "scrypted-plugin"

package/src/main.ts

@@ -157,6 +157,13 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
                 readonly: true,
                 value: `<a href="${uiUrl}" target="_blank" style="display:inline-block;padding:8px 16px;background:#1a4d8a;color:#fff;border-radius:6px;text-decoration:none;font-size:13px;">Open Review UI ↗</a>`,
             },
+            {
+                key: 'autoCapture',
+                title: 'Auto-Capture',
+                description: 'Automatically capture detections in the background. Disable to use manual browsing only.',
+                type: 'boolean',
+                value: (this.storage.getItem('autoCapture') ?? 'true'),
+            },
         ];
 
         for (const cam of cameras) {
@@ -214,6 +221,7 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
     // ── Detection Handler ─────────────────────────────────────────────────────
 
     private async onDetection(cameraId: string, cameraName: string, data: ObjectsDetected, rateLimitMs: number) {
+        if ((this.storage.getItem('autoCapture') ?? 'true') === 'false') return;
         if (!data?.detections?.length || !data.inputDimensions) return;
 
         // Rate limit per camera
@@ -283,6 +291,22 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
         const url = new URL(request.url, 'http://localhost');
         const path = url.pathname.replace(request.rootPath, '');
 
+        // Serve browse event image via getVideoClipThumbnail
+        if (path === '/api/browse-img') {
+            const params = new URL(request.url, 'http://localhost').searchParams;
+            const cameraId = params.get('cameraId')?.replace(/[^a-zA-Z0-9_\-]/g, '');
+            const thumbnailId = params.get('thumbnailId')?.replace(/[^a-zA-Z0-9_\-:.]/g, '');
+            if (!cameraId || !thumbnailId) return response.send('Missing params', { code: 400 });
+            try {
+                const cam = systemManager.getDeviceById(cameraId) as any;
+                const mo = await cam.getVideoClipThumbnail(thumbnailId);
+                const jpeg = await mediaManager.convertMediaObjectToBuffer(mo, 'image/jpeg');
+                return response.send(jpeg, { headers: { 'Content-Type': 'image/jpeg', 'Cache-Control': 'max-age=3600' } });
+            } catch (e) {
+                return response.send('Image unavailable', { code: 404 });
+            }
+        }
+
         // Serve image
         if (path.startsWith('/img/')) {
             const id = path.slice(5).replace(/[^a-zA-Z0-9_\-]/g, ''); // sanitize
@@ -308,6 +332,91 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
             return response.send(JSON.stringify({ ok: true }), { headers: { 'Content-Type': 'application/json' } });
         }
 
+        // API: list cameras for browse
+        if (path === '/api/cameras') {
+            const cameras = Object.keys(systemManager.getSystemState())
+                .map(id => systemManager.getDeviceById(id))
+                .filter(d => d &&
+                    (d.type === ScryptedDeviceType.Camera || d.type === ScryptedDeviceType.Doorbell) &&
+                    d.interfaces?.includes(ScryptedInterface.ObjectDetector)
+                )
+                .map(d => ({ id: d.id, name: d.name }));
+            return response.send(JSON.stringify(cameras), { headers: { 'Content-Type': 'application/json' } });
+        }
+
+        // API: browse recent events for a camera
+        if (path === '/api/browse') {
+            const params = new URL(request.url, 'http://localhost').searchParams;
+            const cameraId = params.get('cameraId');
+            const hours = parseInt(params.get('hours') || '24');
+            if (!cameraId) return response.send('Missing cameraId', { code: 400 });
+
+            try {
+                const cam = systemManager.getDeviceById(cameraId) as any;
+                if (!cam) return response.send('Camera not found', { code: 404 });
+
+                const endTime = Date.now();
+                const startTime = endTime - hours * 3600 * 1000;
+                const clips = await cam.getVideoClips({ startTime, endTime });
+
+                const events = (clips || [])
+                    .filter((c: any) => c.detectionClasses?.length && c.thumbnailId)
+                    .slice(0, 100)
+                    .map((c: any) => ({
+                        clipId: c.id,
+                        thumbnailId: c.thumbnailId,
+                        timestamp: c.startTime,
+                        detectionClasses: c.detectionClasses || [],
+                        // bounding box not available at clip level — use full frame
+                        boundingBox: null,
+                        inputDimensions: null,
+                        cameraId,
+                        cameraName: cam.name,
+                    }));
+
+                return response.send(JSON.stringify(events), { headers: { 'Content-Type': 'application/json' } });
+            } catch (e: any) {
+                return response.send(JSON.stringify({ error: e.message }), { headers: { 'Content-Type': 'application/json' }, code: 500 });
+            }
+        }
+
+        // API: add a browsed event directly to dataset as labeled
+        if (path === '/api/add-event' && request.body) {
+            const rawBody = request.body as any;
+            const body = JSON.parse(typeof rawBody === 'string' ? rawBody : Buffer.isBuffer(rawBody) ? rawBody.toString() : String(rawBody));
+            const { cameraId, cameraName, thumbnailId, timestamp, detectedClass, boundingBox, inputDimensions, label } = body;
+
+            if (!label || label === 'discard') return response.send(JSON.stringify({ ok: true }), { headers: { 'Content-Type': 'application/json' } });
+
+            // Get image via thumbnail
+            let jpeg: Buffer | undefined;
+            try {
+                const cam = systemManager.getDeviceById(cameraId) as any;
+                const mo = await cam.getVideoClipThumbnail(thumbnailId);
+                jpeg = await mediaManager.convertMediaObjectToBuffer(mo, 'image/jpeg');
+            } catch (e) {
+                this.console.warn(`Could not get thumbnail for browse event:`, e);
+            }
+
+            if (!jpeg) return response.send(JSON.stringify({ error: 'Could not retrieve image' }), { headers: { 'Content-Type': 'application/json' }, code: 500 });
+
+            const id = `browse-${timestamp}-${Math.random().toString(36).slice(2, 6)}`;
+            // For clips we don't have per-detection bounding boxes — store full frame dimensions
+            const record: CaptureRecord = {
+                id, cameraId, cameraName, timestamp,
+                detectedClass: detectedClass || 'unknown',
+                score: 1,
+                boundingBox: boundingBox || [0, 0, inputDimensions?.[0] || 1920, inputDimensions?.[1] || 1080],
+                inputDimensions: inputDimensions || [1920, 1080],
+                reviewed: true, label,
+            };
+            this.captures.set(id, record);
+            this.saveImage(id, jpeg);
+            this.saveCaptures();
+
+            return response.send(JSON.stringify({ ok: true, id }), { headers: { 'Content-Type': 'application/json' } });
+        }
+
         // API: get pending captures
         if (path === '/api/pending') {
             const pending = [...this.captures.values()]
@@ -508,6 +617,7 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
   <div class="card">
     <div class="tab-bar">
       <div class="tab active" onclick="showTab('review')">Review</div>
+      <div class="tab" onclick="showTab('browse')">Browse Events</div>
       <div class="tab" onclick="showTab('labeled')">Labeled</div>
       <div class="tab" onclick="showTab('stats')">Stats</div>
       <div class="tab" onclick="showTab('export')">Export Dataset</div>
@@ -518,12 +628,30 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
       <div id="detections-list"></div>
     </div>
 
+    <!-- Browse tab -->
+    <div class="tab-panel" id="tab-browse">
+      <div class="tab-content">
+        <div style="display:flex;gap:12px;align-items:center;flex-wrap:wrap;margin-bottom:16px;">
+          <select id="browse-camera" style="padding:8px 12px;background:#222;border:1px solid #444;color:#fff;border-radius:6px;font-size:13px;">
+            <option value="">Select camera…</option>
+          </select>
+          <select id="browse-hours" style="padding:8px 12px;background:#222;border:1px solid #444;color:#fff;border-radius:6px;font-size:13px;">
+            <option value="1">Last 1 hour</option>
+            <option value="6">Last 6 hours</option>
+            <option value="24" selected>Last 24 hours</option>
+            <option value="72">Last 3 days</option>
+          </select>
+          <button class="export-btn" onclick="loadBrowse()" style="padding:8px 16px;">Load Events</button>
+          <span id="browse-status" style="font-size:13px;color:#888;"></span>
+        </div>
+        <div id="browse-list"></div>
+      </div>
+    </div>
+
     <!-- Labeled tab -->
     <div class="tab-panel" id="tab-labeled">
       <div id="labeled-list"></div>
     </div>
-
-    <!-- Stats tab -->
     <div class="tab-panel" id="tab-stats">
       <div class="tab-content">
         <p style="font-size:13px;color:#888;margin-bottom:16px;">Breakdown of captured and labeled detections.</p>
@@ -670,6 +798,22 @@ function openLightbox(r) {
   document.addEventListener('keydown', lbKeyHandler);
 }
 
+function openLightboxImg(img, cameraName, timestamp) {
+  const lb = document.getElementById('lightbox');
+  const lbCanvas = document.getElementById('lightbox-canvas');
+  const maxW = window.innerWidth * 0.9;
+  const maxH = window.innerHeight * 0.8;
+  const scale = Math.min(maxW / img.naturalWidth, maxH / img.naturalHeight, 1);
+  lbCanvas.width = Math.round(img.naturalWidth * scale);
+  lbCanvas.height = Math.round(img.naturalHeight * scale);
+  const ctx = lbCanvas.getContext('2d');
+  ctx.drawImage(img, 0, 0, lbCanvas.width, lbCanvas.height);
+  document.getElementById('lightbox-meta').textContent =
+    cameraName + ' · ' + new Date(timestamp).toLocaleString();
+  lb.classList.add('open');
+  document.addEventListener('keydown', lbKeyHandler);
+}
+
 function closeLightbox() {
   document.getElementById('lightbox').classList.remove('open');
   document.removeEventListener('keydown', lbKeyHandler);
@@ -680,7 +824,7 @@ function lbKeyHandler(e) {
 }
 
 function showTab(name) {
-  const names = ['review', 'labeled', 'stats', 'export'];
+  const names = ['review', 'browse', 'labeled', 'stats', 'export'];
   document.querySelectorAll('.tab').forEach((t, i) => {
     t.classList.toggle('active', names[i] === name);
   });
@@ -689,6 +833,250 @@ function showTab(name) {
   if (name === 'stats') loadStats();
   if (name === 'export') loadExportInfo();
   if (name === 'labeled') loadLabeled(0);
+  if (name === 'browse') initBrowse();
+}
+
+async function initBrowse() {
+  const sel = document.getElementById('browse-camera');
+  if (sel.options.length > 1) return; // already loaded
+  try {
+    const res = await fetch(BASE + '/api/cameras');
+    const cameras = await res.json();
+    for (const cam of cameras) {
+      const opt = document.createElement('option');
+      opt.value = cam.id;
+      opt.textContent = cam.name;
+      sel.appendChild(opt);
+    }
+    if (cameras.length === 1) sel.value = cameras[0].id;
+  } catch(e) {
+    document.getElementById('browse-status').textContent = 'Error loading cameras';
+  }
+}
+
+async function loadBrowse() {
+  const cameraId = document.getElementById('browse-camera').value;
+  const hours = document.getElementById('browse-hours').value;
+  const status = document.getElementById('browse-status');
+  const list = document.getElementById('browse-list');
+
+  if (!cameraId) { status.textContent = 'Select a camera first'; return; }
+
+  status.textContent = 'Loading…';
+  list.innerHTML = '';
+
+  try {
+    const res = await fetch(BASE + '/api/browse?cameraId=' + cameraId + '&hours=' + hours);
+    const events = await res.json();
+
+    if (events.error) { status.textContent = 'Error: ' + events.error; return; }
+    if (!events.length) { status.textContent = 'No detection events found.'; list.innerHTML = '<div class="empty"><div class="icon">🔍</div><div>No ObjectDetector events in this time range.</div></div>'; return; }
+
+    status.textContent = events.length + ' events found';
+
+    list.innerHTML = events.map((ev, i) => {
+      const date = new Date(ev.timestamp).toLocaleString();
+      const dets = ev.detections || [];
+      const primary = dets[0] || {};
+      const score = Math.round((primary.score || 0) * 100);
+      const allClasses = dets.map(d => d.className + ' ' + Math.round((d.score||0)*100) + '%').join(', ');
+      return \`
+      <div class="detection" id="bev-\${i}" style="opacity:1;transition:opacity .3s">
+        <div class="detection-imgs">
+          <div class="img-panel">
+            <div class="img-label">Full frame</div>
+            <canvas id="bcanvas-\${i}" class="det-canvas" width="240" height="160"></canvas>
+          </div>
+          <div class="img-panel" id="bcrop-panel-\${i}">
+            <div class="img-label">Crop</div>
+            <canvas id="bcanvas-crop-\${i}" class="det-canvas" width="160" height="160"></canvas>
+          </div>
+        </div>
+        <div class="detection-info">
+          <div class="detection-meta">
+            <div><strong>\${ev.cameraName}</strong></div>
+            <div>\${date}</div>
+            <div class="det-class-badge">\${allClasses}</div>
+          </div>
+          <div style="font-size:12px;color:#888;">Add to dataset as:</div>
+          <div class="label-buttons" id="blabels-\${i}">
+            <button class="label-btn person" onclick="addEvent(\${i})('person')">👤 Person</button>
+            <button class="label-btn animal" onclick="addEvent(\${i})('animal')">🐾 Animal</button>
+            <button class="label-btn face" onclick="addEvent(\${i})('face')">😀 Face</button>
+            <button class="label-btn vehicle" onclick="addEvent(\${i})('vehicle')">🚗 Vehicle</button>
+            <button class="label-btn" onclick="addEvent(\${i})('plate')">🔢 Plate</button>
+            <button class="label-btn" onclick="addEvent(\${i})('package')">📦 Package</button>
+            <button class="label-btn discard" onclick="addEvent(\${i})('discard')">🗑 Skip</button>
+          </div>
+        </div>
+      </div>\`;
+    }).join('');
+
+    // Load images for each event
+    for (let i = 0; i < events.length; i++) {
+      const ev = events[i];
+      loadBrowseImage(i, ev);
+    }
+
+  } catch(e) {
+    status.textContent = 'Error: ' + e.message;
+  }
+}
+
+// Store browse events for addEvent closure
+let browseEvents = [];
+
+async function loadBrowse() {
+  const cameraId = document.getElementById('browse-camera').value;
+  const hours = document.getElementById('browse-hours').value;
+  const status = document.getElementById('browse-status');
+  const list = document.getElementById('browse-list');
+
+  if (!cameraId) { status.textContent = 'Select a camera first'; return; }
+
+  status.textContent = 'Loading…';
+  list.innerHTML = '';
+  browseEvents = [];
+
+  try {
+    const res = await fetch(BASE + '/api/browse?cameraId=' + cameraId + '&hours=' + hours);
+    const events = await res.json();
+
+    if (events.error) { status.textContent = 'Error: ' + events.error; return; }
+    if (!events.length) {
+      status.textContent = 'No detection events found.';
+      list.innerHTML = '<div class="empty"><div class="icon">🔍</div><div>No ObjectDetector events in this time range.</div></div>';
+      return;
+    }
+
+    browseEvents = events;
+    status.textContent = events.length + ' events';
+
+    list.innerHTML = events.map((ev, i) => {
+      const date = new Date(ev.timestamp).toLocaleString();
+      const dets = ev.detections || [];
+      const allClasses = [...new Set(dets.map(d => d.className))].join(', ');
+      return \`
+      <div class="detection" id="bev-\${i}">
+        <div class="detection-imgs">
+          <div class="img-panel">
+            <div class="img-label">Full frame</div>
+            <canvas id="bcanvas-full-\${i}" class="det-canvas" width="240" height="160"></canvas>
+          </div>
+          <div class="img-panel">
+            <div class="img-label">Crop</div>
+            <canvas id="bcanvas-crop-\${i}" class="det-canvas" width="160" height="160"></canvas>
+          </div>
+        </div>
+        <div class="detection-info">
+          <div class="detection-meta">
+            <div><strong>\${ev.cameraName}</strong></div>
+            <div>\${date}</div>
+            <div class="det-class-badge">\${allClasses}</div>
+          </div>
+          <div style="font-size:12px;color:#888;">Add to dataset as:</div>
+          <div class="label-buttons">
+            <button class="label-btn person" onclick="addBrowseEvent(\${i},'person')">👤 Person</button>
+            <button class="label-btn animal" onclick="addBrowseEvent(\${i},'animal')">🐾 Animal</button>
+            <button class="label-btn face" onclick="addBrowseEvent(\${i},'face')">😀 Face</button>
+            <button class="label-btn vehicle" onclick="addBrowseEvent(\${i},'vehicle')">🚗 Vehicle</button>
+            <button class="label-btn" onclick="addBrowseEvent(\${i},'plate')">🔢 Plate</button>
+            <button class="label-btn" onclick="addBrowseEvent(\${i},'package')">📦 Package</button>
+            <button class="label-btn discard" onclick="addBrowseEvent(\${i},'discard')">🗑 Skip</button>
+          </div>
+        </div>
+      </div>\`;
+    }).join('');
+
+    // Load thumbnails for each event
+    for (let i = 0; i < events.length; i++) {
+      loadBrowseImage(i, events[i]);
+    }
+
+  } catch(e) {
+    status.textContent = 'Error: ' + e.message;
+  }
+}
+
+function loadBrowseImage(i, ev) {
+  fetch(BASE + '/api/browse-img?cameraId=' + ev.cameraId + '&thumbnailId=' + encodeURIComponent(ev.thumbnailId))
+    .then(r => r.ok ? r.blob() : null)
+    .then(blob => {
+      if (!blob) return;
+      const url = URL.createObjectURL(blob);
+      const img = new Image();
+      img.onload = () => {
+        imgCache.set('browse-' + i, img);
+        const fullCanvas = document.getElementById('bcanvas-full-' + i);
+        const cropCanvas = document.getElementById('bcanvas-crop-' + i);
+        const iw = img.naturalWidth, ih = img.naturalHeight;
+        // No bounding box for clip thumbnails — just draw the full image
+        if (fullCanvas) {
+          const ctx = fullCanvas.getContext('2d');
+          const cw = fullCanvas.width, ch = fullCanvas.height;
+          const scale = Math.min(cw / iw, ch / ih);
+          const dw = iw * scale, dh = ih * scale;
+          ctx.fillStyle = '#111'; ctx.fillRect(0, 0, cw, ch);
+          ctx.drawImage(img, (cw-dw)/2, (ch-dh)/2, dw, dh);
+          // Label classes
+          const labels = (ev.detectionClasses || []).join(', ');
+          ctx.fillStyle = 'rgba(0,0,0,0.6)'; ctx.fillRect(0, ch-20, cw, 20);
+          ctx.fillStyle = '#f90'; ctx.font = 'bold 11px sans-serif';
+          ctx.fillText(labels, 4, ch-5);
+          fullCanvas.onclick = () => openLightboxImg(img, ev.cameraName, ev.timestamp);
+        }
+        // Hide crop panel — no bounding box available
+        const cropPanel = document.getElementById('bcanvas-crop-' + i)?.closest('.img-panel') as HTMLElement;
+        if (cropPanel) cropPanel.style.display = 'none';
+        URL.revokeObjectURL(url);
+      };
+      img.src = url;
+    }).catch(() => {});
+}
+
+async function addBrowseEvent(i, label) {
+  const ev = browseEvents[i];
+  if (!ev) return;
+  const el = document.getElementById('bev-' + i);
+  if (el) { el.style.opacity = '0.4'; el.querySelectorAll('button').forEach(b => b.disabled = true); }
+
+  if (label !== 'discard') {
+    try {
+      const res = await fetch(BASE + '/api/add-event', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          cameraId: ev.cameraId,
+          cameraName: ev.cameraName,
+          thumbnailId: ev.thumbnailId,
+          timestamp: ev.timestamp,
+          detectedClass: (ev.detectionClasses || [])[0] || 'unknown',
+          boundingBox: null,
+          inputDimensions: null,
+          label,
+        }),
+      });
+      const data = await res.json();
+      if (data.error) { toast('Error: ' + data.error, '#633'); if (el) el.style.opacity = '1'; el?.querySelectorAll('button').forEach(b => b.disabled = false); return; }
+      toast('Added: ' + label, '#1a6');
+    } catch(e) {
+      toast('Failed: ' + e.message, '#633');
+      if (el) el.style.opacity = '1';
+      el?.querySelectorAll('button').forEach(b => b.disabled = false);
+      return;
+    }
+  } else {
+    toast('Skipped', '#555');
+  }
+
+  // Remove from list after short delay
+  setTimeout(() => { if (el) el.remove(); }, 400);
+
+  // Update stats
+  const statsRes = await fetch(BASE + '/api/stats');
+  const stats = await statsRes.json();
+  document.getElementById('stat-labeled').textContent = stats.labeled;
+  document.getElementById('stat-total').textContent = stats.total;
 }
 
 const LABEL_COLORS = { person:'#4d9', animal:'#d85', face:'#6be', vehicle:'#99d', plate:'#cc9', package:'#fc9', discard:'#a44' };