tperm-visor 1.0.1

package/backend/utils/smoothing.py

@@ -0,0 +1,40 @@
+import numpy as np
+
+
+class EMA:
+    """Exponential Moving Average — same pattern as Gesture-Media-control baseline (alpha=0.25)."""
+
+    def __init__(self, alpha: float = 0.25):
+        self.alpha = alpha
+        self.value = None
+
+    def update(self, new_value):
+        if self.value is None:
+            self.value = np.array(new_value, dtype=float) if hasattr(new_value, '__len__') else float(new_value)
+        else:
+            self.value = self.alpha * np.array(new_value, dtype=float) + (1 - self.alpha) * self.value
+        return self.value
+
+    def reset(self):
+        self.value = None
+
+
+class QuatEMA:
+    """EMA for quaternions — lerps component-wise then renormalises."""
+
+    def __init__(self, alpha: float = 0.15):
+        self.alpha = alpha
+        self.value = np.array([0.0, 0.0, 0.0, 1.0])  # identity [x,y,z,w]
+
+    def update(self, q: np.ndarray) -> np.ndarray:
+        # Ensure same hemisphere (avoid double-cover flip)
+        if np.dot(self.value, q) < 0:
+            q = -q
+        self.value = self.alpha * q + (1 - self.alpha) * self.value
+        norm = np.linalg.norm(self.value)
+        if norm > 1e-6:
+            self.value /= norm
+        return self.value.copy()
+
+    def reset(self):
+        self.value = np.array([0.0, 0.0, 0.0, 1.0])

package/backend/utils/transforms.py

@@ -0,0 +1,148 @@
+import numpy as np
+from scipy.spatial.transform import Rotation
+
+
+def compute_palm_normal(landmarks) -> np.ndarray:
+    """
+    Cross-product of wrist→index_MCP and wrist→pinky_MCP vectors.
+    Gives the 3-D outward-facing normal of the palm.
+    landmarks: list of 21 (x,y,z) normalized tuples/objects with .x .y .z
+    """
+    def lm(idx):
+        p = landmarks[idx]
+        return np.array([p.x, p.y, p.z])
+
+    v1 = lm(5) - lm(0)   # wrist → index MCP
+    v2 = lm(17) - lm(0)  # wrist → pinky MCP
+    n = np.cross(v1, v2)
+    norm = np.linalg.norm(n)
+    return n / norm if norm > 1e-6 else np.array([0.0, 0.0, 1.0])
+
+
+def compute_finger_direction(landmarks) -> np.ndarray:
+    """
+    Direction from wrist to middle finger MCP.
+    Combined with palm_normal this gives the full hand orientation frame.
+    """
+    def lm(idx):
+        p = landmarks[idx]
+        return np.array([p.x, p.y, p.z])
+
+    v = lm(9) - lm(0)   # wrist → middle MCP
+    norm = np.linalg.norm(v)
+    return v / norm if norm > 1e-6 else np.array([0.0, 1.0, 0.0])
+
+
+def delta_rotation(prev_normal: np.ndarray, curr_normal: np.ndarray,
+                   prev_finger: np.ndarray = None, curr_finger: np.ndarray = None) -> np.ndarray:
+    """
+    Rotation (as quaternion [x,y,z,w]) that takes prev orientation to curr.
+    If finger directions are provided, uses both vectors for full 3D tracking.
+    Otherwise falls back to single-vector alignment (yaw only).
+    """
+    try:
+        if prev_finger is not None and curr_finger is not None:
+            # Full 3D: align two vectors (palm normal + finger direction)
+            R, _ = Rotation.align_vectors(
+                [curr_normal, curr_finger],
+                [prev_normal, prev_finger],
+                weights=[1.0, 0.7]  # normal is primary, finger is secondary
+            )
+        else:
+            R, _ = Rotation.align_vectors([curr_normal], [prev_normal])
+        return R.as_quat()  # [x, y, z, w]
+    except Exception:
+        return np.array([0.0, 0.0, 0.0, 1.0])
+
+
+def hand_orientation_quat(palm_normal: np.ndarray,
+                          finger_direction: np.ndarray) -> np.ndarray:
+    """Absolute orientation of a hand as a quaternion [x, y, z, w].
+
+    Builds an orthonormal frame from the two hand vectors and returns the
+    rotation that carries the reference frame onto it. Absolute, not a
+    frame-to-frame delta: the cube can then be driven straight from where the
+    hand actually points, instead of integrating deltas whose small errors
+    accumulate until the cube no longer corresponds to the hand at all.
+    """
+    forward = np.asarray(palm_normal, dtype=float)
+    n = np.linalg.norm(forward)
+    if n < 1e-6:
+        return np.array([0.0, 0.0, 0.0, 1.0])
+    forward = forward / n
+
+    # Gram-Schmidt the finger direction against the palm normal so the two axes
+    # are exactly orthogonal even though the measured vectors never quite are.
+    up = np.asarray(finger_direction, dtype=float)
+    up = up - np.dot(up, forward) * forward
+    n = np.linalg.norm(up)
+    if n < 1e-6:                      # degenerate: fingers along the normal
+        return np.array([0.0, 0.0, 0.0, 1.0])
+    up = up / n
+
+    right = np.cross(up, forward)
+    n = np.linalg.norm(right)
+    if n < 1e-6:
+        return np.array([0.0, 0.0, 0.0, 1.0])
+    right = right / n
+
+    return Rotation.from_matrix(np.column_stack((right, up, forward))).as_quat()
+
+
+def quat_conjugate(q: np.ndarray) -> np.ndarray:
+    """Inverse of a unit quaternion [x, y, z, w]."""
+    return np.array([-q[0], -q[1], -q[2], q[3]])
+
+
+def quat_multiply(q1: np.ndarray, q2: np.ndarray) -> np.ndarray:
+    """Hamilton product of two quaternions [x,y,z,w]."""
+    x1, y1, z1, w1 = q1
+    x2, y2, z2, w2 = q2
+    return np.array([
+        w1*x2 + x1*w2 + y1*z2 - z1*y2,
+        w1*y2 - x1*z2 + y1*w2 + z1*x2,
+        w1*z2 + x1*y2 - y1*x2 + z1*w2,
+        w1*w2 - x1*x2 - y1*y2 - z1*z2,
+    ])
+
+
+def snap_to_nearest_90(angle_degrees: float) -> float:
+    """Round to nearest multiple of 90°."""
+    return round(angle_degrees / 90.0) * 90.0
+
+
+def cube_axes_on_screen(q: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+    """
+    Given cube rotation quaternion, return the cube's local X and Y
+    axes projected to 2D screen space (as unit vectors in pixel-direction).
+    
+    Uses the SAME quaternion convention as the renderer: [x, y, z, w].
+
+    Returns (screen_x, screen_y) — each is a 2D numpy array.
+    """
+    x, y, z, w = q
+    
+    # Rotation matrix columns (local axes in world space)
+    # Same formula as renderer.py's quat_to_matrix
+    local_x = np.array([
+        1 - 2*y*y - 2*z*z,
+        2*x*y + 2*z*w,
+        2*x*z - 2*y*w
+    ])
+    local_y = np.array([
+        2*x*y - 2*z*w,
+        1 - 2*x*x - 2*z*z,
+        2*y*z + 2*x*w
+    ])
+    
+    # Project to screen: take x,y components, flip y for pixel coords
+    sx = np.array([local_x[0], -local_x[1]])
+    sy = np.array([local_y[0], -local_y[1]])
+    
+    # Normalize
+    nx = np.linalg.norm(sx)
+    ny = np.linalg.norm(sy)
+    if nx > 1e-6: sx /= nx
+    if ny > 1e-6: sy /= ny
+    
+    return sx, sy

package/bin/t-perm.js

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const { execSync, spawn } = require('child_process');
+const http = require('http');
+const path = require('path');
+const fs = require('fs');
+
+// Under `npx` these files live in the npm cache next to this script; run from a
+// clone, the same relative path applies.
+const ROOT    = path.resolve(__dirname, '..');
+const BACKEND = path.join(ROOT, 'backend');
+const PORT    = Number(process.env.T_PERM_PORT) || 5000;
+const URL     = `http://localhost:${PORT}`;
+
+let serverExited = false;
+
+// ── Helpers ─────────────────────────────────────────────────────────────────
+
+function run(cmd, opts = {}) {
+  return execSync(cmd, { stdio: 'inherit', ...opts });
+}
+
+function tryRun(cmd) {
+  try { execSync(cmd, { stdio: 'ignore' }); return true; } catch { return false; }
+}
+
+function capture(cmd) {
+  try { return execSync(cmd, { stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim(); }
+  catch { return null; }
+}
+
+function openBrowser(url) {
+  const cmd =
+    process.platform === 'win32'  ? `start "" "${url}"` :
+    process.platform === 'darwin' ? `open "${url}"` :
+                                    `xdg-open "${url}"`;
+  try { execSync(cmd); } catch { /* not fatal - the URL is printed anyway */ }
+}
+
+function abort(msg) {
+  console.error('\n' + msg + '\n');
+  process.exit(1);
+}
+
+// Poll /health until the engine reports ready. The old code slept a flat 3s,
+// which opened the browser onto a dead port on a slow machine and wasted time
+// on a fast one.
+function waitForServer(timeoutMs = 90000) {
+  const deadline = Date.now() + timeoutMs;
+  return new Promise((resolve, reject) => {
+    (function poll() {
+      if (serverExited) return reject(new Error('server exited during startup'));
+      if (Date.now() > deadline) return reject(new Error('timed out waiting for the server'));
+      const req = http.get(`${URL}/health`, (res) => {
+        res.resume();
+        if (res.statusCode === 200) resolve();
+        else setTimeout(poll, 300);
+      });
+      req.on('error', () => setTimeout(poll, 300));
+      req.setTimeout(1000, () => { req.destroy(); });
+    })();
+  });
+}
+
+// ── Step 1: Detect Python ────────────────────────────────────────────────────
+
+const py =
+  tryRun('python --version')  ? 'python'  :
+  tryRun('python3 --version') ? 'python3' :
+  null;
+
+if (!py) {
+  abort(
+    'Python not found.\n' +
+    '  Install Python 3.9+ from https://python.org and make sure it is on your PATH.'
+  );
+}
+
+// ── Step 2: Confirm the MediaPipe model shipped ─────────────────────────────
+
+const TASK = path.join(BACKEND, 'hand_landmarker.task');
+if (!fs.existsSync(TASK)) {
+  abort(
+    `MediaPipe model missing: ${TASK}\n` +
+    '  If you installed via npx:  npm cache clean --force && npx t-perm\n' +
+    '  If you cloned the repo:    re-clone; the model is committed at\n' +
+    '                             backend/hand_landmarker.task'
+  );
+}
+
+// ── Step 3: Install Python dependencies, only if missing ────────────────────
+
+// Re-running pip every launch costs seconds and writes to the user's system
+// Python for no reason. Import the heavy deps first; install only if that fails.
+const DEPS_OK = !process.argv.includes('--deps') &&
+  tryRun(`${py} -c "import cv2, mediapipe, flask, flask_cors, OpenGL, pyglet, scipy"`);
+
+if (!DEPS_OK) {
+  console.log('\nInstalling Python dependencies (first run only, ~1-2 min)...\n');
+  try {
+    run(`${py} -m pip install -r "${path.join(BACKEND, 'requirements.txt')}"`);
+  } catch {
+    abort('pip install failed. Check the error above and retry.');
+  }
+}
+
+// ── Step 4: Start the backend ────────────────────────────────────────────────
+
+console.log('\nStarting T-PERM...\n');
+
+const server = spawn(py, ['server.py'], {
+  cwd: BACKEND,
+  stdio: 'inherit',
+  env: { ...process.env, PYTHONUNBUFFERED: '1', T_PERM_PORT: String(PORT) },
+});
+
+server.on('error', (err) => abort('Failed to start the Python server: ' + err.message));
+server.on('exit', (code) => {
+  serverExited = true;
+  process.exit(code ?? 0);
+});
+
+// ── Step 5: Open the browser once the server actually answers ───────────────
+
+waitForServer()
+  .then(() => {
+    console.log(`\n  T-PERM is running at ${URL}`);
+    console.log('  Press the "Stop server" button in the page, or Ctrl+C here, to quit.\n');
+    openBrowser(URL);
+  })
+  .catch((err) => {
+    if (!serverExited) console.error('\n' + err.message + `\n  Try opening ${URL} manually.\n`);
+  });
+
+// ── Graceful shutdown ────────────────────────────────────────────────────────
+
+let shuttingDown = false;
+function shutdown() {
+  if (shuttingDown) return;
+  shuttingDown = true;
+  server.kill('SIGINT');
+  // If the engine is wedged mid-frame, do not hang the terminal forever.
+  setTimeout(() => { server.kill('SIGKILL'); process.exit(0); }, 5000).unref();
+}
+process.on('SIGINT',  shutdown);
+process.on('SIGTERM', shutdown);

package/frontend/css/style.css

@@ -0,0 +1,330 @@
+:root {
+  --graphite: #14171a;
+  --panel: #1b1f23;
+  --panel-2: #20252a;
+  --hairline: #2c3238;
+  --ink: #edebe4;
+  --ink-dim: #9098a1;
+  --amber: #ffa23c;
+  --amber-dim: #a8752f;
+  --cyan: #63e6d4;
+  --red: #ff5a5a;
+  --green: #3ce87f;
+  --font-display: 'Space Grotesk', sans-serif;
+  --font-mono: 'JetBrains Mono', monospace;
+  --font-body: 'Inter', sans-serif;
+}
+
+* { box-sizing: border-box; }
+
+html, body {
+  margin: 0;
+  padding: 0;
+  background: var(--graphite);
+  color: var(--ink);
+  font-family: var(--font-body);
+  min-height: 100%;
+}
+
+@media (prefers-reduced-motion: reduce) {
+  *, *::before, *::after { animation-duration: 0.001ms !important; transition-duration: 0.001ms !important; }
+}
+
+:focus-visible {
+  outline: 2px solid var(--amber);
+  outline-offset: 2px;
+}
+
+.dot {
+  display: inline-block;
+  width: 7px; height: 7px;
+  border-radius: 50%;
+  background: var(--amber);
+  box-shadow: 0 0 8px var(--amber);
+  margin-right: 8px;
+  vertical-align: middle;
+}
+
+/* ── Boot screen ─────────────────────────────────────────────────────────── */
+
+#boot-screen {
+  position: relative;
+  min-height: 100vh;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  padding: 48px 20px;
+  overflow: hidden;
+  background:
+    radial-gradient(ellipse 900px 500px at 50% -10%, rgba(255,162,60,0.08), transparent),
+    var(--graphite);
+}
+
+.scanlines {
+  position: absolute; inset: 0;
+  pointer-events: none;
+  background: repeating-linear-gradient(
+    to bottom, rgba(255,255,255,0.018) 0px, rgba(255,255,255,0.018) 1px,
+    transparent 1px, transparent 3px
+  );
+  mix-blend-mode: overlay;
+}
+
+.boot-frame {
+  position: relative;
+  width: 100%;
+  max-width: 620px;
+}
+
+.boot-eyebrow {
+  font-family: var(--font-mono);
+  font-size: 11.5px;
+  letter-spacing: 0.12em;
+  color: var(--ink-dim);
+  margin-bottom: 20px;
+}
+
+.boot-title {
+  font-family: var(--font-display);
+  font-weight: 700;
+  font-size: clamp(42px, 9vw, 76px);
+  line-height: 0.98;
+  letter-spacing: -0.01em;
+  margin: 0 0 20px;
+  color: var(--ink);
+}
+
+.boot-sub {
+  font-size: 15.5px;
+  line-height: 1.6;
+  color: var(--ink-dim);
+  max-width: 46ch;
+  margin: 0 0 32px;
+}
+
+.boot-panel {
+  border: 1px solid var(--hairline);
+  border-radius: 10px;
+  background: var(--panel);
+  overflow: hidden;
+  margin-bottom: 28px;
+}
+
+.panel-row {
+  display: grid;
+  grid-template-columns: 1.15fr 1fr;
+  gap: 12px;
+  padding: 11px 16px;
+  font-family: var(--font-mono);
+  font-size: 12.5px;
+  border-top: 1px solid var(--hairline);
+}
+.panel-row:first-child { border-top: none; }
+.panel-row span:first-child { color: var(--ink); }
+.panel-row span:last-child { color: var(--cyan); text-align: right; }
+.panel-head {
+  background: var(--panel-2);
+  color: var(--ink-dim) !important;
+  letter-spacing: 0.08em;
+}
+.panel-head span { color: var(--ink-dim) !important; font-size: 10.5px; }
+
+.btn-engage {
+  font-family: var(--font-mono);
+  font-size: 14px;
+  font-weight: 600;
+  letter-spacing: 0.04em;
+  color: #171310;
+  background: var(--amber);
+  border: none;
+  border-radius: 8px;
+  padding: 15px 26px;
+  cursor: pointer;
+  transition: transform 0.15s ease, box-shadow 0.15s ease;
+  box-shadow: 0 0 0 rgba(255,162,60,0);
+}
+.btn-engage:hover { transform: translateY(-1px); box-shadow: 0 6px 24px rgba(255,162,60,0.25); }
+.btn-engage:active { transform: translateY(0); }
+.btn-engage:disabled { opacity: 0.6; cursor: wait; transform: none; box-shadow: none; }
+
+.boot-error {
+  color: var(--red);
+  font-family: var(--font-mono);
+  font-size: 12.5px;
+  margin: 14px 0 0;
+  min-height: 1em;
+}
+
+.boot-footnote {
+  font-size: 12px;
+  color: var(--ink-dim);
+  margin: 18px 0 0;
+}
+
+/* ── Live app stage ──────────────────────────────────────────────────────── */
+
+#stage {
+  min-height: 100vh;
+  display: flex;
+  flex-direction: column;
+}
+
+.topbar {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  padding: 16px 20px;
+  border-bottom: 1px solid var(--hairline);
+  font-family: var(--font-mono);
+  font-size: 12.5px;
+}
+.brand { letter-spacing: 0.05em; color: var(--ink-dim); }
+.topbar-actions { display: flex; gap: 10px; }
+
+.btn-ghost {
+  font-family: var(--font-mono);
+  font-size: 11.5px;
+  letter-spacing: 0.04em;
+  color: var(--ink);
+  background: transparent;
+  border: 1px solid var(--hairline);
+  border-radius: 6px;
+  padding: 8px 14px;
+  cursor: pointer;
+  transition: border-color 0.15s ease, color 0.15s ease;
+}
+.btn-ghost:hover { border-color: var(--amber-dim); color: var(--amber); }
+
+.viewport-wrap {
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  padding: 24px;
+  gap: 16px;
+}
+
+.viewport-frame {
+  position: relative;
+  width: min(100%, 980px);
+  aspect-ratio: 16 / 9;
+  background: #000;
+  border-radius: 4px;
+  overflow: hidden;
+  box-shadow: 0 0 0 1px var(--hairline), 0 30px 80px rgba(0,0,0,0.55);
+}
+
+/* The MJPEG stream from the Python backend fills the frame. */
+#backend-stream {
+  position: absolute;
+  inset: 0;
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+}
+
+.bracket {
+  position: absolute;
+  width: 26px; height: 26px;
+  border: 2px solid var(--amber);
+  opacity: 0.85;
+  pointer-events: none;
+}
+.bracket.tl { top: 10px; left: 10px; border-right: none; border-bottom: none; }
+.bracket.tr { top: 10px; right: 10px; border-left: none; border-bottom: none; }
+.bracket.bl { bottom: 10px; left: 10px; border-right: none; border-top: none; }
+.bracket.br { bottom: 10px; right: 10px; border-left: none; border-top: none; }
+
+.legend-panel {
+  position: absolute;
+  top: 14px; right: 14px;
+  width: 300px;
+  background: rgba(20,23,26,0.82);
+  backdrop-filter: blur(10px);
+  border: 1px solid var(--hairline);
+  border-radius: 8px;
+  overflow: hidden;
+  transform: translateY(-8px);
+  opacity: 0;
+  pointer-events: none;
+  transition: transform 0.18s ease, opacity 0.18s ease;
+}
+.legend-panel.open { transform: translateY(0); opacity: 1; pointer-events: auto; }
+.legend-panel .panel-row { padding: 9px 13px; font-size: 11.5px; }
+
+.telemetry {
+  display: flex;
+  gap: 28px;
+  font-family: var(--font-mono);
+  padding: 4px 6px;
+}
+.tel-item { display: flex; flex-direction: column; align-items: center; gap: 2px; }
+.tel-label { font-size: 10px; letter-spacing: 0.1em; color: var(--ink-dim); }
+.tel-value { font-size: 16px; color: var(--cyan); font-weight: 600; }
+
+.footbar {
+  text-align: center;
+  font-size: 12px;
+  color: var(--ink-dim);
+  padding: 16px 20px 24px;
+}
+.footbar strong { color: var(--ink); }
+
+@media (max-width: 640px) {
+  .boot-title { font-size: 44px; }
+  .panel-row { font-size: 11.5px; }
+  .legend-panel { width: 230px; }
+  .telemetry { gap: 18px; }
+}
+
+/* ── Stop server ─────────────────────────────────────────────────────────── */
+
+.btn-quit {
+  font-family: var(--font-mono);
+  font-size: 11.5px;
+  letter-spacing: 0.04em;
+  color: var(--red);
+  background: transparent;
+  border: 1px solid var(--hairline);
+  border-radius: 6px;
+  padding: 8px 14px;
+  cursor: pointer;
+  transition: border-color 0.15s ease, background 0.15s ease;
+}
+.btn-quit:hover { border-color: var(--red); background: rgba(255,90,90,0.08); }
+
+#quit-screen {
+  position: fixed;
+  inset: 0;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  background: rgba(20,23,26,0.94);
+  backdrop-filter: blur(6px);
+  z-index: 10;
+}
+.quit-card {
+  text-align: center;
+  max-width: 420px;
+  padding: 32px;
+}
+.quit-card h2 {
+  font-family: var(--font-display);
+  font-size: 26px;
+  margin: 0 0 12px;
+  color: var(--ink);
+}
+.quit-card p {
+  font-size: 14px;
+  line-height: 1.6;
+  color: var(--ink-dim);
+  margin: 0;
+}
+.quit-card code {
+  font-family: var(--font-mono);
+  color: var(--amber);
+  background: var(--panel);
+  padding: 2px 6px;
+  border-radius: 4px;
+}