scrypted-detection-trainer 0.1.14 → 0.1.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrypted-detection-trainer",
-  "version": "0.1.14",
+  "version": "0.1.16",
   "description": "Collect and label detections to fine-tune the Scrypted NVR object detection model.",
   "keywords": [
     "scrypted-plugin"

package/src/main.ts

@@ -291,15 +291,15 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
         const url = new URL(request.url, 'http://localhost');
         const path = url.pathname.replace(request.rootPath, '');
 
-        // Serve browse event image via getVideoClipThumbnail
+        // Serve browse event image via getDetectionInput
         if (path === '/api/browse-img' && request.body) {
             const rawBody = request.body as any;
             const body = JSON.parse(typeof rawBody === 'string' ? rawBody : Buffer.isBuffer(rawBody) ? rawBody.toString() : String(rawBody));
-            const { cameraId, thumbnailId } = body;
-            if (!cameraId || thumbnailId === undefined) return response.send('Missing params', { code: 400 });
+            const { cameraId, detectionId } = body;
+            if (!cameraId || !detectionId) return response.send('Missing params', { code: 400 });
             try {
-                const cam = systemManager.getDeviceById(cameraId) as any;
-                const mo = await cam.getVideoClipThumbnail(thumbnailId);
+                const cam = systemManager.getDeviceById(cameraId) as unknown as ObjectDetector;
+                const mo = await cam.getDetectionInput(detectionId);
                 const jpeg = await mediaManager.convertMediaObjectToBuffer(mo, 'image/jpeg');
                 return response.send(jpeg, { headers: { 'Content-Type': 'image/jpeg', 'Cache-Control': 'max-age=3600' } });
             } catch (e) {
@@ -349,6 +349,7 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
             const params = new URL(request.url, 'http://localhost').searchParams;
             const cameraId = params.get('cameraId');
             const hours = parseInt(params.get('hours') || '24');
+            const limit = parseInt(params.get('limit') || '100');
             if (!cameraId) return response.send('Missing cameraId', { code: 400 });
 
             try {
@@ -357,25 +358,35 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
 
                 const endTime = Date.now();
                 const startTime = endTime - hours * 3600 * 1000;
-                const clips = await cam.getVideoClips({ startTime, endTime });
 
-                const limit = parseInt(params.get('limit') || '100');
-                const events = (clips || [])
-                    .filter((c: any) => c.detectionClasses?.length && c.thumbnailId)
+                // Use getRecordedEvents to get ObjectDetector events with bounding boxes
+                const recorded = await cam.getRecordedEvents({ startTime, endTime });
+
+                const detectionEvents = (recorded || [])
+                    .filter((e: any) =>
+                        e.details?.eventInterface === 'ObjectDetector' &&
+                        e.data?.detections?.length &&
+                        e.data?.detectionId &&
+                        e.data?.inputDimensions
+                    )
                     .slice(0, limit)
-                    .map((c: any) => ({
-                        clipId: c.id,
-                        thumbnailId: c.thumbnailId,
-                        timestamp: c.startTime,
-                        detectionClasses: c.detectionClasses || [],
-                        // bounding box not available at clip level — use full frame
-                        boundingBox: null,
-                        inputDimensions: null,
-                        cameraId,
-                        cameraName: cam.name,
-                    }));
-
-                return response.send(JSON.stringify(events), { headers: { 'Content-Type': 'application/json' } });
+                    .map((e: any) => {
+                        const ts = e.details?.eventTime || e.data?.timestamp;
+                        return {
+                            detectionId: e.data.detectionId,
+                            timestamp: ts,
+                            detections: (e.data.detections || []).map((d: any) => ({
+                                className: d.className,
+                                score: d.score,
+                                boundingBox: d.boundingBox,
+                            })),
+                            inputDimensions: e.data.inputDimensions,
+                            cameraId,
+                            cameraName: cam.name,
+                        };
+                    });
+
+                return response.send(JSON.stringify(detectionEvents), { headers: { 'Content-Type': 'application/json' } });
             } catch (e: any) {
                 return response.send(JSON.stringify({ error: e.message }), { headers: { 'Content-Type': 'application/json' }, code: 500 });
             }
@@ -385,30 +396,29 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
         if (path === '/api/add-event' && request.body) {
             const rawBody = request.body as any;
             const body = JSON.parse(typeof rawBody === 'string' ? rawBody : Buffer.isBuffer(rawBody) ? rawBody.toString() : String(rawBody));
-            const { cameraId, cameraName, thumbnailId, timestamp, detectedClass, boundingBox, inputDimensions, label } = body;
+            const { cameraId, cameraName, detectionId, timestamp, detectedClass, score, boundingBox, inputDimensions, label } = body;
 
             if (!label || label === 'discard') return response.send(JSON.stringify({ ok: true }), { headers: { 'Content-Type': 'application/json' } });
 
-            // Get image via thumbnail
             let jpeg: Buffer | undefined;
             try {
-                const cam = systemManager.getDeviceById(cameraId) as any;
-                const mo = await cam.getVideoClipThumbnail(thumbnailId);
+                const cam = systemManager.getDeviceById(cameraId) as unknown as ObjectDetector;
+                const mo = await cam.getDetectionInput(detectionId);
                 jpeg = await mediaManager.convertMediaObjectToBuffer(mo, 'image/jpeg');
             } catch (e) {
-                this.console.warn(`Could not get thumbnail for browse event:`, e);
+                this.console.warn(`Could not get image for browse event:`, e);
             }
 
             if (!jpeg) return response.send(JSON.stringify({ error: 'Could not retrieve image' }), { headers: { 'Content-Type': 'application/json' }, code: 500 });
 
             const id = `browse-${timestamp}-${Math.random().toString(36).slice(2, 6)}`;
-            // For clips we don't have per-detection bounding boxes — store full frame dimensions
             const record: CaptureRecord = {
                 id, cameraId, cameraName, timestamp,
                 detectedClass: detectedClass || 'unknown',
-                score: 1,
+                score: score || 1,
                 boundingBox: boundingBox || [0, 0, inputDimensions?.[0] || 1920, inputDimensions?.[1] || 1080],
                 inputDimensions: inputDimensions || [1920, 1080],
+                detectionId,
                 reviewed: true, label,
             };
             this.captures.set(id, record);
@@ -941,7 +951,7 @@ function loadBrowseImage(i, ev) {
   fetch(BASE + '/api/browse-img', {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({ cameraId: ev.cameraId, thumbnailId: ev.thumbnailId }),
+    body: JSON.stringify({ cameraId: ev.cameraId, detectionId: ev.detectionId }),
   })
     .then(r => r.ok ? r.blob() : null)
     .then(blob => {
@@ -949,28 +959,18 @@ function loadBrowseImage(i, ev) {
       const url = URL.createObjectURL(blob);
       const img = new Image();
       img.onload = () => {
+        const primary = (ev.detections || [])[0];
+        if (!primary?.boundingBox) return;
         imgCache.set('browse-' + i, img);
+        // Use drawDetection by temporarily remapping canvas IDs
         const fullCanvas = document.getElementById('bcanvas-full-' + i);
         const cropCanvas = document.getElementById('bcanvas-crop-' + i);
-        const iw = img.naturalWidth, ih = img.naturalHeight;
-        // No bounding box for clip thumbnails — just draw the full image
-        if (fullCanvas) {
-          const ctx = fullCanvas.getContext('2d');
-          const cw = fullCanvas.width, ch = fullCanvas.height;
-          const scale = Math.min(cw / iw, ch / ih);
-          const dw = iw * scale, dh = ih * scale;
-          ctx.fillStyle = '#111'; ctx.fillRect(0, 0, cw, ch);
-          ctx.drawImage(img, (cw-dw)/2, (ch-dh)/2, dw, dh);
-          // Label classes
-          const labels = (ev.detectionClasses || []).join(', ');
-          ctx.fillStyle = 'rgba(0,0,0,0.6)'; ctx.fillRect(0, ch-20, cw, 20);
-          ctx.fillStyle = '#f90'; ctx.font = 'bold 11px sans-serif';
-          ctx.fillText(labels, 4, ch-5);
-          fullCanvas.onclick = () => openLightboxImg(img, ev.cameraName, ev.timestamp);
-        }
-        // Hide crop panel — no bounding box available
-        const cropPanel = document.getElementById('bcanvas-crop-' + i)?.closest('.img-panel');
-        if (cropPanel) cropPanel.style.display = 'none';
+        if (fullCanvas) fullCanvas.id = 'canvas-full-' + i;
+        if (cropCanvas) cropCanvas.id = 'canvas-crop-' + i;
+        const fakeR = { id: String(i), boundingBox: primary.boundingBox, detectedClass: primary.className, score: primary.score, inputDimensions: ev.inputDimensions };
+        drawDetection(img, fakeR);
+        if (fullCanvas) { fullCanvas.id = 'bcanvas-full-' + i; fullCanvas.onclick = () => openLightbox(fakeR); }
+        if (cropCanvas) { cropCanvas.id = 'bcanvas-crop-' + i; cropCanvas.onclick = () => openLightbox(fakeR); }
         URL.revokeObjectURL(url);
       };
       img.src = url;
@@ -985,17 +985,19 @@ async function addBrowseEvent(i, label) {
 
   if (label !== 'discard') {
     try {
+      const primary = (ev.detections || [])[0] || {};
       const res = await fetch(BASE + '/api/add-event', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify({
           cameraId: ev.cameraId,
           cameraName: ev.cameraName,
-          thumbnailId: ev.thumbnailId,
+          detectionId: ev.detectionId,
           timestamp: ev.timestamp,
-          detectedClass: (ev.detectionClasses || [])[0] || 'unknown',
-          boundingBox: null,
-          inputDimensions: null,
+          detectedClass: primary.className || 'unknown',
+          score: primary.score || 1,
+          boundingBox: primary.boundingBox || null,
+          inputDimensions: ev.inputDimensions || null,
           label,
         }),
       });