scrypted-detection-trainer 0.1.8 → 0.1.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrypted-detection-trainer",
-  "version": "0.1.8",
+  "version": "0.1.10",
   "description": "Collect and label detections to fine-tune the Scrypted NVR object detection model.",
   "keywords": [
     "scrypted-plugin"

package/src/main.ts

@@ -10,6 +10,8 @@ import sdk, {
     ObjectsDetected,
     ObjectDetector,
 } from '@scrypted/sdk';
+import fs from 'fs';
+import path from 'path';
 
 const { systemManager, deviceManager, mediaManager } = sdk;
 
@@ -55,13 +57,16 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
     private lastCapture = new Map<string, number>();
     // Map<captureId, CaptureRecord>
     private captures = new Map<string, CaptureRecord>();
-    // Map<captureId, jpegBuffer>
-    private images = new Map<string, Buffer>();
     // Active event listeners
     private listeners: (() => void)[] = [];
+    // Directory for storing images on disk
+    private imgDir: string;
 
     constructor(nativeId?: string) {
         super(nativeId);
+        // Use a stable directory inside the plugin's volume
+        this.imgDir = path.join(process.env.SCRYPTED_PLUGIN_VOLUME || '/tmp', 'detection-trainer-images');
+        try { fs.mkdirSync(this.imgDir, { recursive: true }); } catch {}
         this.loadState();
         this.registerListeners();
     }
@@ -79,26 +84,43 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
         } catch (e) {
             this.console.warn('Could not load captures from storage:', e);
         }
-        // images are stored as individual items
+        // Clean up any old base64 image entries from previous versions
         for (const [id] of this.captures) {
-            const raw = this.storage.getItem(`img:${id}`);
-            if (raw) this.images.set(id, Buffer.from(raw, 'base64'));
+            try { this.storage.removeItem(`img:${id}`); } catch {}
         }
     }
 
     private saveCaptures() {
-        this.storage.setItem('captures', JSON.stringify([...this.captures.values()]));
+        try {
+            this.storage.setItem('captures', JSON.stringify([...this.captures.values()]));
+        } catch (e) {
+            this.console.warn('Could not save captures:', e);
+        }
+    }
+
+    private imgPath(id: string): string {
+        return path.join(this.imgDir, `${id}.jpg`);
     }
 
     private saveImage(id: string, buf: Buffer) {
-        this.storage.setItem(`img:${id}`, buf.toString('base64'));
+        try {
+            fs.writeFileSync(this.imgPath(id), buf);
+        } catch (e) {
+            this.console.warn(`Could not save image ${id}:`, e);
+        }
+    }
+
+    private loadImage(id: string): Buffer | undefined {
+        try {
+            const p = this.imgPath(id);
+            if (fs.existsSync(p)) return fs.readFileSync(p);
+        } catch {}
+        return undefined;
     }
 
     private deleteCapture(id: string) {
-        const old = this.storage.getItem(`img:${id}`);
-        if (old) this.storage.removeItem(`img:${id}`);
+        try { fs.unlinkSync(this.imgPath(id)); } catch {}
         this.captures.delete(id);
-        this.images.delete(id);
         this.saveCaptures();
     }
 
@@ -135,6 +157,13 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
                 readonly: true,
                 value: `<a href="${uiUrl}" target="_blank" style="display:inline-block;padding:8px 16px;background:#1a4d8a;color:#fff;border-radius:6px;text-decoration:none;font-size:13px;">Open Review UI ↗</a>`,
             },
+            {
+                key: 'autoCapture',
+                title: 'Auto-Capture',
+                description: 'Automatically capture detections in the background. Disable to use manual browsing only.',
+                type: 'boolean',
+                value: (this.storage.getItem('autoCapture') ?? 'true'),
+            },
         ];
 
         for (const cam of cameras) {
@@ -192,6 +221,7 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
     // ── Detection Handler ─────────────────────────────────────────────────────
 
     private async onDetection(cameraId: string, cameraName: string, data: ObjectsDetected, rateLimitMs: number) {
+        if ((this.storage.getItem('autoCapture') ?? 'true') === 'false') return;
         if (!data?.detections?.length || !data.inputDimensions) return;
 
         // Rate limit per camera
@@ -249,7 +279,6 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
         };
 
         this.captures.set(id, record);
-        this.images.set(id, jpeg);
         this.saveImage(id, jpeg);
         this.saveCaptures();
 
@@ -262,10 +291,26 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
         const url = new URL(request.url, 'http://localhost');
         const path = url.pathname.replace(request.rootPath, '');
 
+        // Serve browse event image (fetch on demand from camera)
+        if (path === '/api/browse-img') {
+            const params = new URL(request.url, 'http://localhost').searchParams;
+            const cameraId = params.get('cameraId')?.replace(/[^a-zA-Z0-9_\-]/g, '');
+            const detectionId = params.get('detectionId')?.replace(/[^a-zA-Z0-9_\-]/g, '');
+            if (!cameraId || !detectionId) return response.send('Missing params', { code: 400 });
+            try {
+                const cam = systemManager.getDeviceById(cameraId) as unknown as ObjectDetector;
+                const mo = await cam.getDetectionInput(detectionId);
+                const jpeg = await mediaManager.convertMediaObjectToBuffer(mo, 'image/jpeg');
+                return response.send(jpeg, { headers: { 'Content-Type': 'image/jpeg', 'Cache-Control': 'max-age=60' } });
+            } catch (e) {
+                return response.send('Image unavailable', { code: 404 });
+            }
+        }
+
         // Serve image
         if (path.startsWith('/img/')) {
-            const id = path.slice(5);
-            const img = this.images.get(id);
+            const id = path.slice(5).replace(/[^a-zA-Z0-9_\-]/g, ''); // sanitize
+            const img = this.loadImage(id);
             if (!img) return response.send('Not found', { code: 404 });
             return response.send(img, { headers: { 'Content-Type': 'image/jpeg', 'Cache-Control': 'max-age=3600' } });
         }
@@ -287,6 +332,89 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
             return response.send(JSON.stringify({ ok: true }), { headers: { 'Content-Type': 'application/json' } });
         }
 
+        // API: list cameras for browse
+        if (path === '/api/cameras') {
+            const cameras = Object.keys(systemManager.getSystemState())
+                .map(id => systemManager.getDeviceById(id))
+                .filter(d => d &&
+                    (d.type === ScryptedDeviceType.Camera || d.type === ScryptedDeviceType.Doorbell) &&
+                    d.interfaces?.includes(ScryptedInterface.ObjectDetector)
+                )
+                .map(d => ({ id: d.id, name: d.name }));
+            return response.send(JSON.stringify(cameras), { headers: { 'Content-Type': 'application/json' } });
+        }
+
+        // API: browse recent events for a camera
+        if (path === '/api/browse') {
+            const params = new URL(request.url, 'http://localhost').searchParams;
+            const cameraId = params.get('cameraId');
+            const hours = parseInt(params.get('hours') || '24');
+            if (!cameraId) return response.send('Missing cameraId', { code: 400 });
+
+            try {
+                const cam = systemManager.getDeviceById(cameraId) as any;
+                if (!cam) return response.send('Camera not found', { code: 404 });
+
+                const endTime = Date.now();
+                const startTime = endTime - hours * 3600 * 1000;
+                const events = await cam.getRecordedEvents({ startTime, endTime });
+
+                // Filter to ObjectDetector events only and limit to 100
+                const detectionEvents = (events || [])
+                    .filter((e: any) => e.details?.eventInterface === 'ObjectDetector' && e.data?.detections?.length)
+                    .slice(0, 100)
+                    .map((e: any) => ({
+                        detectionId: e.data?.detectionId,
+                        timestamp: e.details?.eventTime || e.data?.timestamp,
+                        detections: (e.data?.detections || []).map((d: any) => ({
+                            className: d.className,
+                            score: d.score,
+                            boundingBox: d.boundingBox,
+                        })),
+                        inputDimensions: e.data?.inputDimensions,
+                        cameraId,
+                        cameraName: cam.name,
+                    }))
+                    .filter((e: any) => e.detectionId && e.inputDimensions);
+
+                return response.send(JSON.stringify(detectionEvents), { headers: { 'Content-Type': 'application/json' } });
+            } catch (e: any) {
+                return response.send(JSON.stringify({ error: e.message }), { headers: { 'Content-Type': 'application/json' }, code: 500 });
+            }
+        }
+
+        // API: add a browsed event directly to dataset as labeled
+        if (path === '/api/add-event' && request.body) {
+            const rawBody = request.body as any;
+            const body = JSON.parse(typeof rawBody === 'string' ? rawBody : Buffer.isBuffer(rawBody) ? rawBody.toString() : String(rawBody));
+            const { cameraId, cameraName, detectionId, timestamp, detectedClass, score, boundingBox, inputDimensions, label } = body;
+
+            if (!label || label === 'discard') return response.send(JSON.stringify({ ok: true }), { headers: { 'Content-Type': 'application/json' } });
+
+            // Try to get the image
+            let jpeg: Buffer | undefined;
+            try {
+                const cam = systemManager.getDeviceById(cameraId) as unknown as ObjectDetector;
+                const mo = await cam.getDetectionInput(detectionId);
+                jpeg = await mediaManager.convertMediaObjectToBuffer(mo, 'image/jpeg');
+            } catch (e) {
+                this.console.warn(`Could not get image for browse event ${detectionId}:`, e);
+            }
+
+            if (!jpeg) return response.send(JSON.stringify({ error: 'Could not retrieve image' }), { headers: { 'Content-Type': 'application/json' }, code: 500 });
+
+            const id = `browse-${timestamp}-${Math.random().toString(36).slice(2, 6)}`;
+            const record: CaptureRecord = {
+                id, cameraId, cameraName, timestamp, detectedClass, score,
+                boundingBox, inputDimensions, detectionId, reviewed: true, label,
+            };
+            this.captures.set(id, record);
+            this.saveImage(id, jpeg);
+            this.saveCaptures();
+
+            return response.send(JSON.stringify({ ok: true, id }), { headers: { 'Content-Type': 'application/json' } });
+        }
+
         // API: get pending captures
         if (path === '/api/pending') {
             const pending = [...this.captures.values()]
@@ -339,7 +467,7 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
             const dataset: { filename: string; content: string; encoding: string }[] = [];
 
             for (const record of labeled) {
-                const img = this.images.get(record.id);
+                const img = this.loadImage(record.id);
                 if (!img) continue;
 
                 const fname = `${record.id}`;
@@ -487,6 +615,7 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
   <div class="card">
     <div class="tab-bar">
       <div class="tab active" onclick="showTab('review')">Review</div>
+      <div class="tab" onclick="showTab('browse')">Browse Events</div>
       <div class="tab" onclick="showTab('labeled')">Labeled</div>
       <div class="tab" onclick="showTab('stats')">Stats</div>
       <div class="tab" onclick="showTab('export')">Export Dataset</div>
@@ -497,12 +626,30 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
       <div id="detections-list"></div>
     </div>
 
+    <!-- Browse tab -->
+    <div class="tab-panel" id="tab-browse">
+      <div class="tab-content">
+        <div style="display:flex;gap:12px;align-items:center;flex-wrap:wrap;margin-bottom:16px;">
+          <select id="browse-camera" style="padding:8px 12px;background:#222;border:1px solid #444;color:#fff;border-radius:6px;font-size:13px;">
+            <option value="">Select camera…</option>
+          </select>
+          <select id="browse-hours" style="padding:8px 12px;background:#222;border:1px solid #444;color:#fff;border-radius:6px;font-size:13px;">
+            <option value="1">Last 1 hour</option>
+            <option value="6">Last 6 hours</option>
+            <option value="24" selected>Last 24 hours</option>
+            <option value="72">Last 3 days</option>
+          </select>
+          <button class="export-btn" onclick="loadBrowse()" style="padding:8px 16px;">Load Events</button>
+          <span id="browse-status" style="font-size:13px;color:#888;"></span>
+        </div>
+        <div id="browse-list"></div>
+      </div>
+    </div>
+
     <!-- Labeled tab -->
     <div class="tab-panel" id="tab-labeled">
       <div id="labeled-list"></div>
     </div>
-
-    <!-- Stats tab -->
     <div class="tab-panel" id="tab-stats">
       <div class="tab-content">
         <p style="font-size:13px;color:#888;margin-bottom:16px;">Breakdown of captured and labeled detections.</p>
@@ -659,7 +806,7 @@ function lbKeyHandler(e) {
 }
 
 function showTab(name) {
-  const names = ['review', 'labeled', 'stats', 'export'];
+  const names = ['review', 'browse', 'labeled', 'stats', 'export'];
   document.querySelectorAll('.tab').forEach((t, i) => {
     t.classList.toggle('active', names[i] === name);
   });
@@ -668,6 +815,246 @@ function showTab(name) {
   if (name === 'stats') loadStats();
   if (name === 'export') loadExportInfo();
   if (name === 'labeled') loadLabeled(0);
+  if (name === 'browse') initBrowse();
+}
+
+async function initBrowse() {
+  const sel = document.getElementById('browse-camera');
+  if (sel.options.length > 1) return; // already loaded
+  try {
+    const res = await fetch(BASE + '/api/cameras');
+    const cameras = await res.json();
+    for (const cam of cameras) {
+      const opt = document.createElement('option');
+      opt.value = cam.id;
+      opt.textContent = cam.name;
+      sel.appendChild(opt);
+    }
+    if (cameras.length === 1) sel.value = cameras[0].id;
+  } catch(e) {
+    document.getElementById('browse-status').textContent = 'Error loading cameras';
+  }
+}
+
+async function loadBrowse() {
+  const cameraId = document.getElementById('browse-camera').value;
+  const hours = document.getElementById('browse-hours').value;
+  const status = document.getElementById('browse-status');
+  const list = document.getElementById('browse-list');
+
+  if (!cameraId) { status.textContent = 'Select a camera first'; return; }
+
+  status.textContent = 'Loading…';
+  list.innerHTML = '';
+
+  try {
+    const res = await fetch(BASE + '/api/browse?cameraId=' + cameraId + '&hours=' + hours);
+    const events = await res.json();
+
+    if (events.error) { status.textContent = 'Error: ' + events.error; return; }
+    if (!events.length) { status.textContent = 'No detection events found.'; list.innerHTML = '<div class="empty"><div class="icon">🔍</div><div>No ObjectDetector events in this time range.</div></div>'; return; }
+
+    status.textContent = events.length + ' events found';
+
+    list.innerHTML = events.map((ev, i) => {
+      const date = new Date(ev.timestamp).toLocaleString();
+      const dets = ev.detections || [];
+      const primary = dets[0] || {};
+      const score = Math.round((primary.score || 0) * 100);
+      const allClasses = dets.map(d => d.className + ' ' + Math.round((d.score||0)*100) + '%').join(', ');
+      return \`
+      <div class="detection" id="bev-\${i}" style="opacity:1;transition:opacity .3s">
+        <div class="detection-imgs">
+          <div class="img-panel">
+            <div class="img-label">Full frame</div>
+            <canvas id="bcanvas-\${i}" class="det-canvas" width="240" height="160"></canvas>
+          </div>
+          <div class="img-panel" id="bcrop-panel-\${i}">
+            <div class="img-label">Crop</div>
+            <canvas id="bcanvas-crop-\${i}" class="det-canvas" width="160" height="160"></canvas>
+          </div>
+        </div>
+        <div class="detection-info">
+          <div class="detection-meta">
+            <div><strong>\${ev.cameraName}</strong></div>
+            <div>\${date}</div>
+            <div class="det-class-badge">\${allClasses}</div>
+          </div>
+          <div style="font-size:12px;color:#888;">Add to dataset as:</div>
+          <div class="label-buttons" id="blabels-\${i}">
+            <button class="label-btn person" onclick="addEvent(\${i})('person')">👤 Person</button>
+            <button class="label-btn animal" onclick="addEvent(\${i})('animal')">🐾 Animal</button>
+            <button class="label-btn face" onclick="addEvent(\${i})('face')">😀 Face</button>
+            <button class="label-btn vehicle" onclick="addEvent(\${i})('vehicle')">🚗 Vehicle</button>
+            <button class="label-btn" onclick="addEvent(\${i})('plate')">🔢 Plate</button>
+            <button class="label-btn" onclick="addEvent(\${i})('package')">📦 Package</button>
+            <button class="label-btn discard" onclick="addEvent(\${i})('discard')">🗑 Skip</button>
+          </div>
+        </div>
+      </div>\`;
+    }).join('');
+
+    // Load images for each event
+    for (let i = 0; i < events.length; i++) {
+      const ev = events[i];
+      loadBrowseImage(i, ev);
+    }
+
+  } catch(e) {
+    status.textContent = 'Error: ' + e.message;
+  }
+}
+
+// Store browse events for addEvent closure
+let browseEvents = [];
+
+async function loadBrowse() {
+  const cameraId = document.getElementById('browse-camera').value;
+  const hours = document.getElementById('browse-hours').value;
+  const status = document.getElementById('browse-status');
+  const list = document.getElementById('browse-list');
+
+  if (!cameraId) { status.textContent = 'Select a camera first'; return; }
+
+  status.textContent = 'Loading…';
+  list.innerHTML = '';
+  browseEvents = [];
+
+  try {
+    const res = await fetch(BASE + '/api/browse?cameraId=' + cameraId + '&hours=' + hours);
+    const events = await res.json();
+
+    if (events.error) { status.textContent = 'Error: ' + events.error; return; }
+    if (!events.length) {
+      status.textContent = 'No detection events found.';
+      list.innerHTML = '<div class="empty"><div class="icon">🔍</div><div>No ObjectDetector events in this time range.</div></div>';
+      return;
+    }
+
+    browseEvents = events;
+    status.textContent = events.length + ' events';
+
+    list.innerHTML = events.map((ev, i) => {
+      const date = new Date(ev.timestamp).toLocaleString();
+      const dets = ev.detections || [];
+      const allClasses = [...new Set(dets.map(d => d.className))].join(', ');
+      return \`
+      <div class="detection" id="bev-\${i}">
+        <div class="detection-imgs">
+          <div class="img-panel">
+            <div class="img-label">Full frame</div>
+            <canvas id="bcanvas-full-\${i}" class="det-canvas" width="240" height="160"></canvas>
+          </div>
+          <div class="img-panel">
+            <div class="img-label">Crop</div>
+            <canvas id="bcanvas-crop-\${i}" class="det-canvas" width="160" height="160"></canvas>
+          </div>
+        </div>
+        <div class="detection-info">
+          <div class="detection-meta">
+            <div><strong>\${ev.cameraName}</strong></div>
+            <div>\${date}</div>
+            <div class="det-class-badge">\${allClasses}</div>
+          </div>
+          <div style="font-size:12px;color:#888;">Add to dataset as:</div>
+          <div class="label-buttons">
+            <button class="label-btn person" onclick="addBrowseEvent(\${i},'person')">👤 Person</button>
+            <button class="label-btn animal" onclick="addBrowseEvent(\${i},'animal')">🐾 Animal</button>
+            <button class="label-btn face" onclick="addBrowseEvent(\${i},'face')">😀 Face</button>
+            <button class="label-btn vehicle" onclick="addBrowseEvent(\${i},'vehicle')">🚗 Vehicle</button>
+            <button class="label-btn" onclick="addBrowseEvent(\${i},'plate')">🔢 Plate</button>
+            <button class="label-btn" onclick="addBrowseEvent(\${i},'package')">📦 Package</button>
+            <button class="label-btn discard" onclick="addBrowseEvent(\${i},'discard')">🗑 Skip</button>
+          </div>
+        </div>
+      </div>\`;
+    }).join('');
+
+    // Load thumbnails for each event
+    for (let i = 0; i < events.length; i++) {
+      loadBrowseImage(i, events[i]);
+    }
+
+  } catch(e) {
+    status.textContent = 'Error: ' + e.message;
+  }
+}
+
+function loadBrowseImage(i, ev) {
+  const primary = (ev.detections || [])[0];
+  if (!primary?.boundingBox) return;
+  // Request image via the img endpoint using detectionId as the key
+  // We store a browse-prefixed image server-side only after adding — for preview
+  // use a placeholder fetch to trigger server-side caching
+  fetch(BASE + '/api/browse-img?cameraId=' + ev.cameraId + '&detectionId=' + ev.detectionId)
+    .then(r => r.ok ? r.blob() : null)
+    .then(blob => {
+      if (!blob) return;
+      const url = URL.createObjectURL(blob);
+      const img = new Image();
+      img.onload = () => {
+        imgCache.set('browse-' + i, img);
+        // Draw on full canvas
+        const fullCanvas = document.getElementById('bcanvas-full-' + i);
+        const cropCanvas = document.getElementById('bcanvas-crop-' + i);
+        if (fullCanvas) fullCanvas.id = 'canvas-full-browse' + i;
+        if (cropCanvas) cropCanvas.id = 'canvas-crop-browse' + i;
+        const fakeR = { ...ev, id: 'browse' + i, boundingBox: primary.boundingBox, detectedClass: primary.className, score: primary.score };
+        drawDetection(img, fakeR);
+        if (fullCanvas) { fullCanvas.id = 'bcanvas-full-' + i; fullCanvas.onclick = () => openLightbox(fakeR); }
+        if (cropCanvas) { cropCanvas.id = 'bcanvas-crop-' + i; cropCanvas.onclick = () => openLightbox(fakeR); }
+        URL.revokeObjectURL(url);
+      };
+      img.src = url;
+    }).catch(() => {});
+}
+
+async function addBrowseEvent(i, label) {
+  const ev = browseEvents[i];
+  if (!ev) return;
+  const el = document.getElementById('bev-' + i);
+  if (el) { el.style.opacity = '0.4'; el.querySelectorAll('button').forEach(b => b.disabled = true); }
+
+  if (label !== 'discard') {
+    const primary = (ev.detections || [])[0];
+    if (!primary) return;
+    try {
+      const res = await fetch(BASE + '/api/add-event', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          cameraId: ev.cameraId,
+          cameraName: ev.cameraName,
+          detectionId: ev.detectionId,
+          timestamp: ev.timestamp,
+          detectedClass: primary.className,
+          score: primary.score,
+          boundingBox: primary.boundingBox,
+          inputDimensions: ev.inputDimensions,
+          label,
+        }),
+      });
+      const data = await res.json();
+      if (data.error) { toast('Error: ' + data.error, '#633'); if (el) el.style.opacity = '1'; el?.querySelectorAll('button').forEach(b => b.disabled = false); return; }
+      toast('Added: ' + label, '#1a6');
+    } catch(e) {
+      toast('Failed: ' + e.message, '#633');
+      if (el) el.style.opacity = '1';
+      el?.querySelectorAll('button').forEach(b => b.disabled = false);
+      return;
+    }
+  } else {
+    toast('Skipped', '#555');
+  }
+
+  // Remove from list after short delay
+  setTimeout(() => { if (el) el.remove(); }, 400);
+
+  // Update stats
+  const statsRes = await fetch(BASE + '/api/stats');
+  const stats = await statsRes.json();
+  document.getElementById('stat-labeled').textContent = stats.labeled;
+  document.getElementById('stat-total').textContent = stats.total;
 }
 
 const LABEL_COLORS = { person:'#4d9', animal:'#d85', face:'#6be', vehicle:'#99d', plate:'#cc9', package:'#fc9', discard:'#a44' };