scrypted-detection-trainer 0.1.6 → 0.1.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrypted-detection-trainer",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "Collect and label detections to fine-tune the Scrypted NVR object detection model.",
   "keywords": [
     "scrypted-plugin"

package/src/main.ts

@@ -112,7 +112,12 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
                 d.interfaces?.includes(ScryptedInterface.ObjectDetector)
             );
 
-        const uiUrl = await sdk.endpointManager.getLocalEndpoint(undefined, { public: true }).catch(() => '/endpoint/scrypted-detection-trainer/public/');
+        let uiUrl: string;
+        try {
+            uiUrl = await sdk.endpointManager.getLocalEndpoint(undefined, { public: true });
+        } catch {
+            uiUrl = '/endpoint/scrypted-detection-trainer/public/';
+        }
 
         const settings: Setting[] = [
             {
@@ -291,6 +296,17 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
             return response.send(JSON.stringify(pending), { headers: { 'Content-Type': 'application/json' } });
         }
 
+        // API: get labeled captures
+        if (path === '/api/labeled') {
+            const page = parseInt(new URL(request.url, 'http://localhost').searchParams.get('page') || '0');
+            const pageSize = 50;
+            const labeled = [...this.captures.values()]
+                .filter(c => c.reviewed)
+                .sort((a, b) => b.timestamp - a.timestamp);
+            const slice = labeled.slice(page * pageSize, (page + 1) * pageSize);
+            return response.send(JSON.stringify({ items: slice, total: labeled.length, page, pageSize }), { headers: { 'Content-Type': 'application/json' } });
+        }
+
         // API: stats
         if (path === '/api/stats') {
             const all = [...this.captures.values()];
@@ -392,11 +408,13 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
   .badge.green { background: #0d2d0d; color: #4c4; }
 
   /* Detection card */
-  .detection { display: grid; grid-template-columns: 200px 1fr; gap: 0; border-bottom: 1px solid #222; }
+  .detection { display: grid; grid-template-columns: 420px 1fr; gap: 0; border-bottom: 1px solid #222; }
   .detection:last-child { border-bottom: none; }
-  .detection-img { position: relative; background: #111; display: flex; align-items: center; justify-content: center; min-height: 150px; }
-  .detection-img img { width: 100%; height: 150px; object-fit: cover; display: block; }
-  .detection-class { position: absolute; top: 6px; left: 6px; background: rgba(0,0,0,0.7); color: #fff; font-size: 11px; padding: 2px 6px; border-radius: 4px; }
+  .detection-imgs { display: flex; gap: 8px; padding: 10px; background: #111; align-items: center; }
+  .img-panel { display: flex; flex-direction: column; align-items: center; gap: 4px; }
+  .img-label { font-size: 10px; color: #555; text-transform: uppercase; letter-spacing: .5px; }
+  .det-canvas { border-radius: 6px; display: block; }
+  .det-class-badge { display: inline-block; background: #3d2a00; color: #f90; font-size: 11px; padding: 2px 8px; border-radius: 4px; }
   .detection-info { padding: 14px 16px; display: flex; flex-direction: column; gap: 10px; }
   .detection-meta { font-size: 12px; color: #888; display: flex; flex-wrap: wrap; gap: 10px; }
   .detection-meta strong { color: #ccc; }
@@ -444,6 +462,15 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
   .breakdown-item { background: #222; border-radius: 8px; padding: 12px; }
   .breakdown-item .label { font-size: 12px; color: #888; margin-bottom: 4px; }
   .breakdown-item .value { font-size: 20px; font-weight: 600; color: #fff; }
+
+  /* Lightbox */
+  .lightbox { display: none; position: fixed; inset: 0; background: rgba(0,0,0,0.92); z-index: 1000; align-items: center; justify-content: center; flex-direction: column; gap: 12px; }
+  .lightbox.open { display: flex; }
+  .lightbox canvas { max-width: 92vw; max-height: 82vh; border-radius: 8px; cursor: zoom-out; }
+  .lightbox-meta { color: #aaa; font-size: 13px; text-align: center; }
+  .lightbox-close { position: absolute; top: 16px; right: 20px; font-size: 28px; color: #888; cursor: pointer; line-height: 1; }
+  .lightbox-close:hover { color: #fff; }
+  .det-canvas { cursor: zoom-in; }
 </style>
 </head>
 <body>
@@ -460,6 +487,7 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
   <div class="card">
     <div class="tab-bar">
       <div class="tab active" onclick="showTab('review')">Review</div>
+      <div class="tab" onclick="showTab('labeled')">Labeled</div>
       <div class="tab" onclick="showTab('stats')">Stats</div>
       <div class="tab" onclick="showTab('export')">Export Dataset</div>
     </div>
@@ -469,6 +497,11 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
       <div id="detections-list"></div>
     </div>
 
+    <!-- Labeled tab -->
+    <div class="tab-panel" id="tab-labeled">
+      <div id="labeled-list"></div>
+    </div>
+
     <!-- Stats tab -->
     <div class="tab-panel" id="tab-stats">
       <div class="tab-content">
@@ -499,6 +532,12 @@ class DetectionTrainer extends ScryptedDeviceBase implements Settings, HttpReque
   </div>
 </div>
 
+<div class="lightbox" id="lightbox" onclick="closeLightbox()">
+  <div class="lightbox-close" onclick="closeLightbox()">✕</div>
+  <canvas id="lightbox-canvas"></canvas>
+  <div class="lightbox-meta" id="lightbox-meta"></div>
+</div>
+
 <div class="toast" id="toast"></div>
 
 <script>
@@ -510,15 +549,244 @@ function imgError(img) {
   img.parentElement.innerHTML = '<div style="padding:20px;color:#555;font-size:12px;text-align:center">No image</div>';
 }
 
+function drawDetection(img, r) {
+  const [bx, by, bw, bh] = r.boundingBox;
+  const [iw, ih] = r.inputDimensions;
+
+  // --- Full frame canvas with box overlay ---
+  const fullCanvas = document.getElementById('canvas-full-' + r.id);
+  if (fullCanvas) {
+    const cw = fullCanvas.width, ch = fullCanvas.height;
+    const scale = Math.min(cw / iw, ch / ih);
+    const dw = iw * scale, dh = ih * scale;
+    const ox = (cw - dw) / 2, oy = (ch - dh) / 2;
+    const ctx = fullCanvas.getContext('2d');
+    ctx.fillStyle = '#111';
+    ctx.fillRect(0, 0, cw, ch);
+    ctx.drawImage(img, ox, oy, dw, dh);
+    // Draw bounding box
+    const rx = ox + bx * scale, ry = oy + by * scale;
+    const rw = bw * scale, rh = bh * scale;
+    ctx.strokeStyle = '#f90';
+    ctx.lineWidth = 2;
+    ctx.strokeRect(rx, ry, rw, rh);
+    // Label badge
+    ctx.fillStyle = 'rgba(255,153,0,0.85)';
+    ctx.fillRect(rx, ry - 18, rw, 18);
+    ctx.fillStyle = '#000';
+    ctx.font = 'bold 11px sans-serif';
+    ctx.fillText(r.detectedClass + ' ' + Math.round(r.score * 100) + '%', rx + 3, ry - 4);
+  }
+
+  // --- Crop canvas ---
+  const cropCanvas = document.getElementById('canvas-crop-' + r.id);
+  if (cropCanvas) {
+    const cc = cropCanvas.width, ch2 = cropCanvas.height;
+    const ctx2 = cropCanvas.getContext('2d');
+    ctx2.fillStyle = '#111';
+    ctx2.fillRect(0, 0, cc, ch2);
+    // Add padding around the crop
+    const pad = Math.min(bw, bh) * 0.15;
+    const sx = Math.max(0, bx - pad), sy = Math.max(0, by - pad);
+    const sw = Math.min(iw - sx, bw + pad * 2), sh = Math.min(ih - sy, bh + pad * 2);
+    const scale2 = Math.min(cc / sw, ch2 / sh);
+    const dw2 = sw * scale2, dh2 = sh * scale2;
+    const ox2 = (cc - dw2) / 2, oy2 = (ch2 - dh2) / 2;
+    ctx2.drawImage(img, sx, sy, sw, sh, ox2, oy2, dw2, dh2);
+    // Thin box outline on crop
+    ctx2.strokeStyle = '#f90';
+    ctx2.lineWidth = 1.5;
+    const rx2 = ox2 + pad * scale2, ry2 = oy2 + pad * scale2;
+    ctx2.strokeRect(rx2, ry2, bw * scale2, bh * scale2);
+  }
+}
+
+// Cache loaded images so lightbox reuses them
+const imgCache = new Map();
+
+function openLightbox(r) {
+  const img = imgCache.get(r.id);
+  if (!img) return;
+
+  const lb = document.getElementById('lightbox');
+  const lbCanvas = document.getElementById('lightbox-canvas');
+
+  // Size canvas to image, capped at viewport
+  const maxW = window.innerWidth * 0.9;
+  const maxH = window.innerHeight * 0.8;
+  const [iw, ih] = r.inputDimensions;
+  const scale = Math.min(maxW / iw, maxH / ih, 1);
+  lbCanvas.width = Math.round(iw * scale);
+  lbCanvas.height = Math.round(ih * scale);
+
+  const ctx = lbCanvas.getContext('2d');
+  ctx.drawImage(img, 0, 0, lbCanvas.width, lbCanvas.height);
+
+  // Draw all bounding boxes for this detection (primary + others in same event if available)
+  const boxes = r.allDetections || [{ boundingBox: r.boundingBox, detectedClass: r.detectedClass, score: r.score }];
+  const colors = ['#f90', '#4af', '#f44', '#4f4', '#f4f', '#ff4'];
+  boxes.forEach((d, i) => {
+    const [bx, by, bw, bh] = d.boundingBox;
+    const color = colors[i % colors.length];
+    const rx = bx * scale, ry = by * scale, rw = bw * scale, rh = bh * scale;
+    ctx.strokeStyle = color;
+    ctx.lineWidth = 2;
+    ctx.strokeRect(rx, ry, rw, rh);
+    const label = d.detectedClass + ' ' + Math.round((d.score || 0) * 100) + '%';
+    const textW = ctx.measureText(label).width + 8;
+    ctx.fillStyle = color;
+    ctx.globalAlpha = 0.85;
+    ctx.fillRect(rx, Math.max(0, ry - 20), textW, 20);
+    ctx.globalAlpha = 1;
+    ctx.fillStyle = '#000';
+    ctx.font = 'bold 12px sans-serif';
+    ctx.fillText(label, rx + 4, Math.max(14, ry - 4));
+  });
+
+  document.getElementById('lightbox-meta').textContent =
+    r.cameraName + ' · ' + new Date(r.timestamp).toLocaleString() + ' · ' + iw + '×' + ih;
+  lb.classList.add('open');
+  document.addEventListener('keydown', lbKeyHandler);
+}
+
+function closeLightbox() {
+  document.getElementById('lightbox').classList.remove('open');
+  document.removeEventListener('keydown', lbKeyHandler);
+}
+
+function lbKeyHandler(e) {
+  if (e.key === 'Escape') closeLightbox();
+}
+
 function showTab(name) {
+  const names = ['review', 'labeled', 'stats', 'export'];
   document.querySelectorAll('.tab').forEach((t, i) => {
-    const names = ['review', 'stats', 'export'];
     t.classList.toggle('active', names[i] === name);
   });
   document.querySelectorAll('.tab-panel').forEach(p => p.classList.remove('active'));
   document.getElementById('tab-' + name).classList.add('active');
   if (name === 'stats') loadStats();
   if (name === 'export') loadExportInfo();
+  if (name === 'labeled') loadLabeled(0);
+}
+
+const LABEL_COLORS = { person:'#4d9', animal:'#d85', face:'#6be', vehicle:'#99d', plate:'#cc9', package:'#fc9', discard:'#a44' };
+
+async function loadLabeled(page) {
+  const list = document.getElementById('labeled-list');
+  list.innerHTML = '<div class="empty"><div style="color:#888">Loading…</div></div>';
+  try {
+    const res = await fetch(BASE + '/api/labeled?page=' + page);
+    const data = await res.json();
+    const { items, total, pageSize } = data;
+    const totalPages = Math.ceil(total / pageSize);
+
+    if (!items.length) {
+      list.innerHTML = '<div class="empty"><div class="icon">🏷️</div><div>No labeled detections yet.</div></div>';
+      return;
+    }
+
+    const pagerHtml = totalPages > 1 ? \`
+      <div style="display:flex;align-items:center;gap:12px;padding:14px 16px;border-top:1px solid #222;font-size:13px;color:#888;">
+        \${page > 0 ? \`<button class="label-btn" onclick="loadLabeled(\${page-1})">← Prev</button>\` : ''}
+        <span>Page \${page+1} of \${totalPages} · \${total} total</span>
+        \${page < totalPages-1 ? \`<button class="label-btn" onclick="loadLabeled(\${page+1})">Next →</button>\` : ''}
+      </div>\` : '';
+
+    list.innerHTML = items.map(r => {
+      const date = new Date(r.timestamp).toLocaleString();
+      const score = Math.round(r.score * 100);
+      const labelColor = LABEL_COLORS[r.label] || '#aaa';
+      return \`
+      <div class="detection" id="ldet-\${r.id}">
+        <div class="detection-imgs">
+          <div class="img-panel">
+            <div class="img-label">Full frame</div>
+            <canvas id="lcanvas-full-\${r.id}" class="det-canvas" width="240" height="160"></canvas>
+          </div>
+          <div class="img-panel">
+            <div class="img-label">Crop</div>
+            <canvas id="lcanvas-crop-\${r.id}" class="det-canvas" width="160" height="160"></canvas>
+          </div>
+        </div>
+        <div class="detection-info">
+          <div class="detection-meta">
+            <div><strong>\${r.cameraName}</strong></div>
+            <div>\${date}</div>
+            <div class="det-class-badge">\${r.detectedClass} \${score}%</div>
+            <div style="display:inline-block;background:#1a1a1a;border:1px solid \${labelColor};color:\${labelColor};font-size:11px;padding:2px 8px;border-radius:4px;">✓ \${r.label}</div>
+          </div>
+          <div style="font-size:12px;color:#888;">Change label:</div>
+          <div class="label-buttons">
+            <button class="label-btn person" onclick="relabel('\${r.id}', 'person')">👤 Person</button>
+            <button class="label-btn animal" onclick="relabel('\${r.id}', 'animal')">🐾 Animal</button>
+            <button class="label-btn face" onclick="relabel('\${r.id}', 'face')">😀 Face</button>
+            <button class="label-btn vehicle" onclick="relabel('\${r.id}', 'vehicle')">🚗 Vehicle</button>
+            <button class="label-btn" onclick="relabel('\${r.id}', 'plate')">🔢 Plate</button>
+            <button class="label-btn" onclick="relabel('\${r.id}', 'package')">📦 Package</button>
+            <button class="label-btn discard" onclick="relabel('\${r.id}', 'discard')">🗑 Discard</button>
+          </div>
+        </div>
+      </div>\`;
+    }).join('') + pagerHtml;
+
+    // Draw bounding boxes
+    for (const r of items) {
+      const img = new Image();
+      img.onload = () => {
+        imgCache.set(r.id, img);
+        // Reuse drawDetection with the labeled canvases
+        const origFull = document.getElementById('canvas-full-' + r.id);
+        const origCrop = document.getElementById('canvas-crop-' + r.id);
+        // Temporarily point to labeled canvases
+        const fakeFull = document.getElementById('lcanvas-full-' + r.id);
+        const fakeCrop = document.getElementById('lcanvas-crop-' + r.id);
+        if (fakeFull) fakeFull.id = 'canvas-full-' + r.id;
+        if (fakeCrop) fakeCrop.id = 'canvas-crop-' + r.id;
+        drawDetection(img, r);
+        if (fakeFull) fakeFull.id = 'lcanvas-full-' + r.id;
+        if (fakeCrop) fakeCrop.id = 'lcanvas-crop-' + r.id;
+        if (fakeFull) fakeFull.onclick = () => openLightbox(r);
+        if (fakeCrop) fakeCrop.onclick = () => openLightbox(r);
+      };
+      img.src = BASE + '/img/' + r.id;
+    }
+  } catch(e) {
+    list.innerHTML = '<div class="empty"><div style="color:#a44">Error: ' + e.message + '</div></div>';
+  }
+}
+
+async function relabel(id, labelVal) {
+  await fetch(BASE + '/api/label', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ id, label: labelVal }),
+  });
+  const el = document.getElementById('ldet-' + id);
+  if (labelVal === 'discard') {
+    if (el) el.remove();
+    toast('Discarded', '#633');
+  } else {
+    // Update the label badge in place
+    const badge = el && el.querySelector('[style*="✓"]');
+    const labelColor = LABEL_COLORS[labelVal] || '#aaa';
+    if (el) {
+      const badges = el.querySelectorAll('.detection-meta > div');
+      badges.forEach(b => { if (b.textContent.startsWith('✓')) b.remove(); });
+      const meta = el.querySelector('.detection-meta');
+      const newBadge = document.createElement('div');
+      newBadge.style.cssText = \`display:inline-block;background:#1a1a1a;border:1px solid \${labelColor};color:\${labelColor};font-size:11px;padding:2px 8px;border-radius:4px;\`;
+      newBadge.textContent = '✓ ' + labelVal;
+      meta.appendChild(newBadge);
+    }
+    toast('Re-labeled: ' + labelVal, '#1a6');
+  }
+  // Refresh stats
+  const statsRes = await fetch(BASE + '/api/stats');
+  const stats = await statsRes.json();
+  document.getElementById('stat-pending').textContent = stats.pending;
+  document.getElementById('stat-labeled').textContent = stats.labeled;
+  document.getElementById('stat-total').textContent = stats.total;
 }
 
 function toast(msg, color='#1a3') {
@@ -549,17 +817,25 @@ async function loadPending() {
   list.innerHTML = pending.map(r => {
     const date = new Date(r.timestamp).toLocaleString();
     const score = Math.round(r.score * 100);
+    const bb = r.boundingBox;
+    const dim = r.inputDimensions;
     return \`
     <div class="detection" id="det-\${r.id}">
-      <div class="detection-img">
-        <img src="\${BASE}/img/\${r.id}" alt="\${r.detectedClass}" loading="lazy" onerror="imgError(this)">
-        <div class="detection-class">\${r.detectedClass} \${score}%</div>
+      <div class="detection-imgs">
+        <div class="img-panel">
+          <div class="img-label">Full frame</div>
+          <canvas id="canvas-full-\${r.id}" class="det-canvas" width="240" height="160"></canvas>
+        </div>
+        <div class="img-panel">
+          <div class="img-label">Crop</div>
+          <canvas id="canvas-crop-\${r.id}" class="det-canvas" width="160" height="160"></canvas>
+        </div>
       </div>
       <div class="detection-info">
         <div class="detection-meta">
           <div><strong>\${r.cameraName}</strong></div>
           <div>\${date}</div>
-          <div>Box: \${r.boundingBox.map(v => Math.round(v)).join(', ')}</div>
+          <div class="det-class-badge">\${r.detectedClass} \${score}%</div>
         </div>
         <div style="font-size:12px;color:#888;">What is this actually?</div>
         <div class="label-buttons">
@@ -574,7 +850,25 @@ async function loadPending() {
       </div>
     </div>\`;
   }).join('');
-  } catch(e) {
+
+  // Draw bounding boxes and crops onto canvases after DOM is ready
+  for (const r of pending) {
+    const img = new Image();
+    img.onload = () => {
+      imgCache.set(r.id, img);
+      drawDetection(img, r);
+      // Wire up click to open lightbox
+      const fullCanvas = document.getElementById('canvas-full-' + r.id);
+      const cropCanvas = document.getElementById('canvas-crop-' + r.id);
+      if (fullCanvas) fullCanvas.onclick = () => openLightbox(r);
+      if (cropCanvas) cropCanvas.onclick = () => openLightbox(r);
+    };
+    img.onerror = () => {
+      const c = document.getElementById('canvas-full-' + r.id);
+      if (c) c.parentElement.innerHTML = '<div style="padding:10px;color:#555;font-size:11px">No image</div>';
+    };
+    img.src = BASE + '/img/' + r.id;
+  }  } catch(e) {
     console.error('loadPending error', e);
     const list = document.getElementById('detections-list');
     if (list) list.innerHTML = '<div class="empty"><div style="color:#a44">Error loading captures: ' + e.message + '</div></div>';