scrypted-detection-trainer 0.1.10 → 0.1.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrypted-detection-trainer",
-  "version": "0.1.10",
+  "version": "0.1.12",
   "description": "Collect and label detections to fine-tune the Scrypted NVR object detection model.",
   "keywords": [
     "scrypted-plugin"

package/src/main.ts

@@ -291,17 +291,17 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
         const url = new URL(request.url, 'http://localhost');
         const path = url.pathname.replace(request.rootPath, '');
 
-        // Serve browse event image (fetch on demand from camera)
+        // Serve browse event image via getVideoClipThumbnail
         if (path === '/api/browse-img') {
             const params = new URL(request.url, 'http://localhost').searchParams;
             const cameraId = params.get('cameraId')?.replace(/[^a-zA-Z0-9_\-]/g, '');
-            const detectionId = params.get('detectionId')?.replace(/[^a-zA-Z0-9_\-]/g, '');
-            if (!cameraId || !detectionId) return response.send('Missing params', { code: 400 });
+            const thumbnailId = params.get('thumbnailId')?.replace(/[^a-zA-Z0-9_\-:.]/g, '');
+            if (!cameraId || !thumbnailId) return response.send('Missing params', { code: 400 });
             try {
-                const cam = systemManager.getDeviceById(cameraId) as unknown as ObjectDetector;
-                const mo = await cam.getDetectionInput(detectionId);
+                const cam = systemManager.getDeviceById(cameraId) as any;
+                const mo = await cam.getVideoClipThumbnail(thumbnailId);
                 const jpeg = await mediaManager.convertMediaObjectToBuffer(mo, 'image/jpeg');
-                return response.send(jpeg, { headers: { 'Content-Type': 'image/jpeg', 'Cache-Control': 'max-age=60' } });
+                return response.send(jpeg, { headers: { 'Content-Type': 'image/jpeg', 'Cache-Control': 'max-age=3600' } });
             } catch (e) {
                 return response.send('Image unavailable', { code: 404 });
             }
@@ -357,27 +357,24 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
 
                 const endTime = Date.now();
                 const startTime = endTime - hours * 3600 * 1000;
-                const events = await cam.getRecordedEvents({ startTime, endTime });
+                const clips = await cam.getVideoClips({ startTime, endTime });
 
-                // Filter to ObjectDetector events only and limit to 100
-                const detectionEvents = (events || [])
-                    .filter((e: any) => e.details?.eventInterface === 'ObjectDetector' && e.data?.detections?.length)
+                const events = (clips || [])
+                    .filter((c: any) => c.detectionClasses?.length && c.thumbnailId)
                     .slice(0, 100)
-                    .map((e: any) => ({
-                        detectionId: e.data?.detectionId,
-                        timestamp: e.details?.eventTime || e.data?.timestamp,
-                        detections: (e.data?.detections || []).map((d: any) => ({
-                            className: d.className,
-                            score: d.score,
-                            boundingBox: d.boundingBox,
-                        })),
-                        inputDimensions: e.data?.inputDimensions,
+                    .map((c: any) => ({
+                        clipId: c.id,
+                        thumbnailId: c.thumbnailId,
+                        timestamp: c.startTime,
+                        detectionClasses: c.detectionClasses || [],
+                        // bounding box not available at clip level — use full frame
+                        boundingBox: null,
+                        inputDimensions: null,
                         cameraId,
                         cameraName: cam.name,
-                    }))
-                    .filter((e: any) => e.detectionId && e.inputDimensions);
+                    }));
 
-                return response.send(JSON.stringify(detectionEvents), { headers: { 'Content-Type': 'application/json' } });
+                return response.send(JSON.stringify(events), { headers: { 'Content-Type': 'application/json' } });
             } catch (e: any) {
                 return response.send(JSON.stringify({ error: e.message }), { headers: { 'Content-Type': 'application/json' }, code: 500 });
             }
@@ -387,26 +384,31 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
         if (path === '/api/add-event' && request.body) {
             const rawBody = request.body as any;
             const body = JSON.parse(typeof rawBody === 'string' ? rawBody : Buffer.isBuffer(rawBody) ? rawBody.toString() : String(rawBody));
-            const { cameraId, cameraName, detectionId, timestamp, detectedClass, score, boundingBox, inputDimensions, label } = body;
+            const { cameraId, cameraName, thumbnailId, timestamp, detectedClass, boundingBox, inputDimensions, label } = body;
 
             if (!label || label === 'discard') return response.send(JSON.stringify({ ok: true }), { headers: { 'Content-Type': 'application/json' } });
 
-            // Try to get the image
+            // Get image via thumbnail
             let jpeg: Buffer | undefined;
             try {
-                const cam = systemManager.getDeviceById(cameraId) as unknown as ObjectDetector;
-                const mo = await cam.getDetectionInput(detectionId);
+                const cam = systemManager.getDeviceById(cameraId) as any;
+                const mo = await cam.getVideoClipThumbnail(thumbnailId);
                 jpeg = await mediaManager.convertMediaObjectToBuffer(mo, 'image/jpeg');
             } catch (e) {
-                this.console.warn(`Could not get image for browse event ${detectionId}:`, e);
+                this.console.warn(`Could not get thumbnail for browse event:`, e);
             }
 
             if (!jpeg) return response.send(JSON.stringify({ error: 'Could not retrieve image' }), { headers: { 'Content-Type': 'application/json' }, code: 500 });
 
             const id = `browse-${timestamp}-${Math.random().toString(36).slice(2, 6)}`;
+            // For clips we don't have per-detection bounding boxes — store full frame dimensions
             const record: CaptureRecord = {
-                id, cameraId, cameraName, timestamp, detectedClass, score,
-                boundingBox, inputDimensions, detectionId, reviewed: true, label,
+                id, cameraId, cameraName, timestamp,
+                detectedClass: detectedClass || 'unknown',
+                score: 1,
+                boundingBox: boundingBox || [0, 0, inputDimensions?.[0] || 1920, inputDimensions?.[1] || 1080],
+                inputDimensions: inputDimensions || [1920, 1080],
+                reviewed: true, label,
             };
             this.captures.set(id, record);
             this.saveImage(id, jpeg);
@@ -796,6 +798,22 @@ function openLightbox(r) {
   document.addEventListener('keydown', lbKeyHandler);
 }
 
+function openLightboxImg(img, cameraName, timestamp) {
+  const lb = document.getElementById('lightbox');
+  const lbCanvas = document.getElementById('lightbox-canvas');
+  const maxW = window.innerWidth * 0.9;
+  const maxH = window.innerHeight * 0.8;
+  const scale = Math.min(maxW / img.naturalWidth, maxH / img.naturalHeight, 1);
+  lbCanvas.width = Math.round(img.naturalWidth * scale);
+  lbCanvas.height = Math.round(img.naturalHeight * scale);
+  const ctx = lbCanvas.getContext('2d');
+  ctx.drawImage(img, 0, 0, lbCanvas.width, lbCanvas.height);
+  document.getElementById('lightbox-meta').textContent =
+    cameraName + ' · ' + new Date(timestamp).toLocaleString();
+  lb.classList.add('open');
+  document.addEventListener('keydown', lbKeyHandler);
+}
+
 function closeLightbox() {
   document.getElementById('lightbox').classList.remove('open');
   document.removeEventListener('keydown', lbKeyHandler);
@@ -836,75 +854,6 @@ async function initBrowse() {
   }
 }
 
-async function loadBrowse() {
-  const cameraId = document.getElementById('browse-camera').value;
-  const hours = document.getElementById('browse-hours').value;
-  const status = document.getElementById('browse-status');
-  const list = document.getElementById('browse-list');
-
-  if (!cameraId) { status.textContent = 'Select a camera first'; return; }
-
-  status.textContent = 'Loading…';
-  list.innerHTML = '';
-
-  try {
-    const res = await fetch(BASE + '/api/browse?cameraId=' + cameraId + '&hours=' + hours);
-    const events = await res.json();
-
-    if (events.error) { status.textContent = 'Error: ' + events.error; return; }
-    if (!events.length) { status.textContent = 'No detection events found.'; list.innerHTML = '<div class="empty"><div class="icon">🔍</div><div>No ObjectDetector events in this time range.</div></div>'; return; }
-
-    status.textContent = events.length + ' events found';
-
-    list.innerHTML = events.map((ev, i) => {
-      const date = new Date(ev.timestamp).toLocaleString();
-      const dets = ev.detections || [];
-      const primary = dets[0] || {};
-      const score = Math.round((primary.score || 0) * 100);
-      const allClasses = dets.map(d => d.className + ' ' + Math.round((d.score||0)*100) + '%').join(', ');
-      return \`
-      <div class="detection" id="bev-\${i}" style="opacity:1;transition:opacity .3s">
-        <div class="detection-imgs">
-          <div class="img-panel">
-            <div class="img-label">Full frame</div>
-            <canvas id="bcanvas-\${i}" class="det-canvas" width="240" height="160"></canvas>
-          </div>
-          <div class="img-panel" id="bcrop-panel-\${i}">
-            <div class="img-label">Crop</div>
-            <canvas id="bcanvas-crop-\${i}" class="det-canvas" width="160" height="160"></canvas>
-          </div>
-        </div>
-        <div class="detection-info">
-          <div class="detection-meta">
-            <div><strong>\${ev.cameraName}</strong></div>
-            <div>\${date}</div>
-            <div class="det-class-badge">\${allClasses}</div>
-          </div>
-          <div style="font-size:12px;color:#888;">Add to dataset as:</div>
-          <div class="label-buttons" id="blabels-\${i}">
-            <button class="label-btn person" onclick="addEvent(\${i})('person')">👤 Person</button>
-            <button class="label-btn animal" onclick="addEvent(\${i})('animal')">🐾 Animal</button>
-            <button class="label-btn face" onclick="addEvent(\${i})('face')">😀 Face</button>
-            <button class="label-btn vehicle" onclick="addEvent(\${i})('vehicle')">🚗 Vehicle</button>
-            <button class="label-btn" onclick="addEvent(\${i})('plate')">🔢 Plate</button>
-            <button class="label-btn" onclick="addEvent(\${i})('package')">📦 Package</button>
-            <button class="label-btn discard" onclick="addEvent(\${i})('discard')">🗑 Skip</button>
-          </div>
-        </div>
-      </div>\`;
-    }).join('');
-
-    // Load images for each event
-    for (let i = 0; i < events.length; i++) {
-      const ev = events[i];
-      loadBrowseImage(i, ev);
-    }
-
-  } catch(e) {
-    status.textContent = 'Error: ' + e.message;
-  }
-}
-
 // Store browse events for addEvent closure
 let browseEvents = [];
 
@@ -981,12 +930,7 @@ async function loadBrowse() {
 }
 
 function loadBrowseImage(i, ev) {
-  const primary = (ev.detections || [])[0];
-  if (!primary?.boundingBox) return;
-  // Request image via the img endpoint using detectionId as the key
-  // We store a browse-prefixed image server-side only after adding — for preview
-  // use a placeholder fetch to trigger server-side caching
-  fetch(BASE + '/api/browse-img?cameraId=' + ev.cameraId + '&detectionId=' + ev.detectionId)
+  fetch(BASE + '/api/browse-img?cameraId=' + ev.cameraId + '&thumbnailId=' + encodeURIComponent(ev.thumbnailId))
     .then(r => r.ok ? r.blob() : null)
     .then(blob => {
       if (!blob) return;
@@ -994,15 +938,27 @@ function loadBrowseImage(i, ev) {
       const img = new Image();
       img.onload = () => {
         imgCache.set('browse-' + i, img);
-        // Draw on full canvas
         const fullCanvas = document.getElementById('bcanvas-full-' + i);
         const cropCanvas = document.getElementById('bcanvas-crop-' + i);
-        if (fullCanvas) fullCanvas.id = 'canvas-full-browse' + i;
-        if (cropCanvas) cropCanvas.id = 'canvas-crop-browse' + i;
-        const fakeR = { ...ev, id: 'browse' + i, boundingBox: primary.boundingBox, detectedClass: primary.className, score: primary.score };
-        drawDetection(img, fakeR);
-        if (fullCanvas) { fullCanvas.id = 'bcanvas-full-' + i; fullCanvas.onclick = () => openLightbox(fakeR); }
-        if (cropCanvas) { cropCanvas.id = 'bcanvas-crop-' + i; cropCanvas.onclick = () => openLightbox(fakeR); }
+        const iw = img.naturalWidth, ih = img.naturalHeight;
+        // No bounding box for clip thumbnails — just draw the full image
+        if (fullCanvas) {
+          const ctx = fullCanvas.getContext('2d');
+          const cw = fullCanvas.width, ch = fullCanvas.height;
+          const scale = Math.min(cw / iw, ch / ih);
+          const dw = iw * scale, dh = ih * scale;
+          ctx.fillStyle = '#111'; ctx.fillRect(0, 0, cw, ch);
+          ctx.drawImage(img, (cw-dw)/2, (ch-dh)/2, dw, dh);
+          // Label classes
+          const labels = (ev.detectionClasses || []).join(', ');
+          ctx.fillStyle = 'rgba(0,0,0,0.6)'; ctx.fillRect(0, ch-20, cw, 20);
+          ctx.fillStyle = '#f90'; ctx.font = 'bold 11px sans-serif';
+          ctx.fillText(labels, 4, ch-5);
+          fullCanvas.onclick = () => openLightboxImg(img, ev.cameraName, ev.timestamp);
+        }
+        // Hide crop panel — no bounding box available
+        const cropPanel = document.getElementById('bcanvas-crop-' + i)?.closest('.img-panel') as HTMLElement;
+        if (cropPanel) cropPanel.style.display = 'none';
         URL.revokeObjectURL(url);
       };
       img.src = url;
@@ -1016,8 +972,6 @@ async function addBrowseEvent(i, label) {
   if (el) { el.style.opacity = '0.4'; el.querySelectorAll('button').forEach(b => b.disabled = true); }
 
   if (label !== 'discard') {
-    const primary = (ev.detections || [])[0];
-    if (!primary) return;
     try {
       const res = await fetch(BASE + '/api/add-event', {
         method: 'POST',
@@ -1025,12 +979,11 @@ async function addBrowseEvent(i, label) {
         body: JSON.stringify({
           cameraId: ev.cameraId,
           cameraName: ev.cameraName,
-          detectionId: ev.detectionId,
+          thumbnailId: ev.thumbnailId,
           timestamp: ev.timestamp,
-          detectedClass: primary.className,
-          score: primary.score,
-          boundingBox: primary.boundingBox,
-          inputDimensions: ev.inputDimensions,
+          detectedClass: (ev.detectionClasses || [])[0] || 'unknown',
+          boundingBox: null,
+          inputDimensions: null,
           label,
         }),
       });