shiftacle-desktop-agent 0.1.0

package/bin/cli.mjs

@@ -0,0 +1,204 @@
+#!/usr/bin/env node
+
+/**
+ * Shiftacle Desktop Agent
+ *
+ * Auto-fills your EMR from your phone.
+ * Generate a note in Shift Notes, tap "Send to EMR",
+ * and it appears in your EMR on your computer — automatically.
+ *
+ * Usage: npx shiftacle-desktop-agent [options]
+ *
+ * Options:
+ *   --port <number>       Local server port (default: 3847)
+ *   --emr <type>          EMR type: hellonote, webpt, nethealth (default: hellonote)
+ *   --headless            Run browser in headless mode
+ *   --vision-url <url>    Vision AI endpoint URL
+ *   --login               Force re-login to Shiftacle
+ *   --help                Show this help
+ */
+
+import { parseArgs } from 'node:util';
+import { execSync } from 'child_process';
+import { existsSync } from 'fs';
+import { getConfig, ensureDirs } from '../src/config.mjs';
+import { authenticate } from '../src/auth.mjs';
+import { startServer } from '../src/server.mjs';
+import { startPoller } from '../src/poller.mjs';
+
+// --- Parse CLI flags ---
+
+const { values: flags } = parseArgs({
+  options: {
+    port: { type: 'string', short: 'p' },
+    emr: { type: 'string', short: 'e' },
+    headless: { type: 'boolean', default: false },
+    'vision-url': { type: 'string' },
+    login: { type: 'boolean', default: false },
+    help: { type: 'boolean', short: 'h', default: false },
+  },
+  strict: false,
+});
+
+if (flags.help) {
+  console.log(`
+  Shiftacle Desktop Agent v0.1.0
+
+  Auto-fills your EMR with clinical notes from Shift Notes.
+
+  Usage: npx shiftacle-desktop-agent [options]
+
+  Options:
+    -p, --port <number>       Local server port (default: 3847)
+    -e, --emr <type>          EMR type: hellonote, webpt, nethealth (default: hellonote)
+        --headless            Run browser without visible window
+        --vision-url <url>    Custom vision AI endpoint
+        --login               Force re-login to Shiftacle account
+    -h, --help                Show this help
+
+  Environment Variables:
+    VISION_URL                Vision AI endpoint
+    VISION_MODEL              Vision model name
+    SHIFTACLE_API_URL         Shiftacle API base URL
+    EMR_AGENT_PORT            Server port
+
+  Examples:
+    npx shiftacle-desktop-agent
+    npx shiftacle-desktop-agent --emr webpt
+    npx shiftacle-desktop-agent --port 4000
+`);
+  process.exit(0);
+}
+
+// --- Startup ---
+
+const config = getConfig(flags);
+ensureDirs(config);
+
+console.log(`
+  ┌─────────────────────────────────────────┐
+  │       Shiftacle Desktop Agent v0.1.0    │
+  │       Auto-fill your EMR from mobile    │
+  └─────────────────────────────────────────┘
+`);
+
+// Step 1: Check Playwright browsers
+process.stdout.write('  Checking browser installation... ');
+try {
+  const { chromium } = await import('playwright');
+  const executablePath = chromium.executablePath();
+  if (!existsSync(executablePath)) {
+    console.log('installing Chromium (first run only)...');
+    execSync('npx playwright install chromium', { stdio: 'pipe' });
+    console.log('  Chromium installed.');
+  } else {
+    console.log('OK');
+  }
+} catch {
+  console.log('installing Chromium...');
+  try {
+    execSync('npx playwright install chromium', { stdio: 'inherit' });
+  } catch (err) {
+    console.error('\n  Failed to install Chromium. Run manually:');
+    console.error('    npx playwright install chromium\n');
+    process.exit(1);
+  }
+}
+
+// Step 2: Authenticate with Shiftacle
+process.stdout.write('  Authenticating... ');
+let auth;
+try {
+  auth = await authenticate(config, flags.login);
+  console.log('OK');
+} catch (err) {
+  console.error(`FAILED\n\n  ${err.message}\n`);
+  process.exit(1);
+}
+
+// Step 3: Check if port is available
+try {
+  const net = await import('net');
+  await new Promise((resolve, reject) => {
+    const tester = net.createServer();
+    tester.once('error', (err) => {
+      if (err.code === 'EADDRINUSE') {
+        reject(new Error(`Port ${config.port} is already in use. Is another agent running?\n  Use --port <number> to change.`));
+      } else {
+        reject(err);
+      }
+    });
+    tester.listen(config.port, () => {
+      tester.close(resolve);
+    });
+  });
+} catch (err) {
+  console.error(`\n  ${err.message}\n`);
+  process.exit(1);
+}
+
+// Step 4: Start local server
+const agent = startServer(config);
+console.log(`  Server listening on port ${config.port}`);
+
+// Step 5: Open EMR browser
+process.stdout.write(`  Opening ${config.emr} in browser... `);
+try {
+  const res = await fetch(`http://localhost:${config.port}/session/start`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ emrType: config.emr }),
+  });
+  const data = await res.json();
+  if (data.success) {
+    console.log('OK');
+    console.log(`  Log in to your EMR if prompted.`);
+  } else {
+    console.log(`WARN: ${data.error}`);
+  }
+} catch (err) {
+  console.log(`WARN: ${err.message}`);
+}
+
+// Step 6: Start polling for notes
+const poller = startPoller(config, auth, {
+  onNote: (entry) => {
+    const patient = entry.patientLabel || entry.patientName || 'Unknown';
+    console.log(`\n  >> New note received: ${patient} (${entry.noteType || 'daily'})`);
+  },
+  onError: ({ type, message }) => {
+    if (type === 'auth') {
+      console.log(`\n  >> Auth token expired — restart agent or run with --login`);
+    }
+    // Suppress poll errors (normal when API is unreachable during beta)
+  },
+});
+
+// --- Status banner ---
+console.log(`
+  ──────────────────────────────────────────
+  Agent is running.
+
+  EMR:           ${config.emr}
+  Local server:  http://localhost:${config.port}
+  Polling:       every ${config.pollIntervalMs / 1000}s
+
+  Waiting for notes from Shift Notes...
+  Press Ctrl+C to stop.
+  ──────────────────────────────────────────
+`);
+
+// --- Graceful shutdown ---
+function shutdown() {
+  console.log('\n  Shutting down...');
+  poller.stop();
+  agent.close();
+  console.log('  Agent stopped.\n');
+  process.exit(0);
+}
+
+process.on('SIGINT', shutdown);
+process.on('SIGTERM', shutdown);
+
+// Keep process alive
+process.stdin.resume();

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "shiftacle-desktop-agent",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Desktop agent that auto-fills your EMR with clinical notes from Shiftacle",
+  "bin": {
+    "shiftacle-desktop-agent": "bin/cli.mjs"
+  },
+  "scripts": {
+    "start": "node bin/cli.mjs"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "keywords": [
+    "emr",
+    "ehr",
+    "healthcare",
+    "clinical-notes",
+    "hellonote",
+    "webpt",
+    "browser-automation",
+    "shiftacle"
+  ],
+  "dependencies": {
+    "express": "^5.1.0",
+    "playwright": "^1.57.0"
+  },
+  "files": [
+    "bin",
+    "src"
+  ],
+  "license": "UNLICENSED",
+  "homepage": "https://shiftacle.com/desktop-agent",
+  "author": "Shiftacle <support@shiftacle.com>"
+}

package/src/auth.mjs

@@ -0,0 +1,125 @@
+import { readFileSync, writeFileSync, existsSync } from 'fs';
+import { join } from 'path';
+import { createInterface } from 'node:readline';
+
+const AUTH_FILE = 'auth.json';
+
+function decodeJwtPayload(token) {
+  try {
+    const payload = token.split('.')[1];
+    return JSON.parse(Buffer.from(payload, 'base64url').toString());
+  } catch {
+    return null;
+  }
+}
+
+function isTokenValid(token) {
+  const payload = decodeJwtPayload(token);
+  if (!payload?.exp) return false;
+  // Valid if > 5 minutes from expiry
+  return payload.exp * 1000 > Date.now() + 5 * 60 * 1000;
+}
+
+async function promptCredentials() {
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  const ask = (q) => new Promise((resolve) => rl.question(q, resolve));
+
+  console.log('\n  Log in with your Shiftacle account:\n');
+  const email = await ask('  Email: ');
+
+  // Hide password input
+  const password = await new Promise((resolve) => {
+    process.stdout.write('  Password: ');
+    const stdin = process.stdin;
+    const wasRaw = stdin.isRaw;
+    if (stdin.setRawMode) stdin.setRawMode(true);
+    let pw = '';
+    const onData = (ch) => {
+      const c = ch.toString();
+      if (c === '\n' || c === '\r') {
+        if (stdin.setRawMode) stdin.setRawMode(wasRaw ?? false);
+        stdin.removeListener('data', onData);
+        process.stdout.write('\n');
+        resolve(pw);
+      } else if (c === '\u007f' || c === '\b') {
+        if (pw.length > 0) {
+          pw = pw.slice(0, -1);
+          process.stdout.write('\b \b');
+        }
+      } else if (c === '\u0003') {
+        // Ctrl+C
+        process.exit(1);
+      } else {
+        pw += c;
+        process.stdout.write('*');
+      }
+    };
+    stdin.on('data', onData);
+    stdin.resume();
+  });
+
+  rl.close();
+  return { email, password };
+}
+
+async function refreshToken(apiBase, refreshTok) {
+  try {
+    const res = await fetch(`${apiBase}/auth/refresh`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ refreshToken: refreshTok }),
+    });
+    if (!res.ok) return null;
+    return await res.json();
+  } catch {
+    return null;
+  }
+}
+
+export async function authenticate(config, forceLogin = false) {
+  const authPath = join(config.configDir, AUTH_FILE);
+
+  if (!forceLogin && existsSync(authPath)) {
+    try {
+      const stored = JSON.parse(readFileSync(authPath, 'utf8'));
+      if (stored.token && isTokenValid(stored.token)) {
+        return stored;
+      }
+      // Try refresh
+      if (stored.refreshToken) {
+        const refreshed = await refreshToken(config.apiBase, stored.refreshToken);
+        if (refreshed?.token) {
+          const data = { ...stored, ...refreshed };
+          writeFileSync(authPath, JSON.stringify(data), { mode: 0o600 });
+          return data;
+        }
+      }
+    } catch {
+      // Corrupted auth file — fall through to login
+    }
+  }
+
+  // Interactive login
+  const { email, password } = await promptCredentials();
+
+  const res = await fetch(`${config.apiBase}/auth/login`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ email, password }),
+  });
+
+  if (!res.ok) {
+    const body = await res.text().catch(() => '');
+    throw new Error(`Login failed (${res.status}): ${body || 'Invalid credentials'}`);
+  }
+
+  const data = await res.json();
+  if (!data.token) throw new Error('Login response missing token');
+
+  writeFileSync(authPath, JSON.stringify(data), { mode: 0o600 });
+  return data;
+}
+
+export function getAuthHeaders(auth) {
+  return { Authorization: `Bearer ${auth.token}` };
+}

package/src/config.mjs

@@ -0,0 +1,41 @@
+import { homedir } from 'os';
+import { join } from 'path';
+import { existsSync, mkdirSync } from 'fs';
+
+export const DEFAULTS = {
+  visionUrl: 'https://vision.shiftacle.com/v1',
+  visionModel: 'Qwen/Qwen3-VL-8B-Instruct',
+  apiBase: 'https://api.shiftacle.com/api/v1',
+  sessionDir: join(homedir(), '.shiftacle-emr-sessions'),
+  configDir: join(homedir(), '.shiftacle'),
+  port: 3847,
+  emr: 'hellonote',
+  pollIntervalMs: 5000,
+  emrUrls: {
+    hellonote: 'https://app.hellonote.com',
+    webpt: 'https://app.webpt.com',
+    nethealth: 'https://app.nethealth.com',
+  },
+};
+
+export function getConfig(cliFlags = {}) {
+  const config = {
+    visionUrl: cliFlags['vision-url'] || process.env.VISION_URL || DEFAULTS.visionUrl,
+    visionModel: process.env.VISION_MODEL || DEFAULTS.visionModel,
+    apiBase: process.env.SHIFTACLE_API_URL || DEFAULTS.apiBase,
+    port: parseInt(cliFlags.port || process.env.EMR_AGENT_PORT || DEFAULTS.port, 10),
+    emr: cliFlags.emr || DEFAULTS.emr,
+    headless: cliFlags.headless || false,
+    sessionDir: DEFAULTS.sessionDir,
+    configDir: DEFAULTS.configDir,
+    pollIntervalMs: parseInt(process.env.POLL_INTERVAL || DEFAULTS.pollIntervalMs, 10),
+    emrUrls: DEFAULTS.emrUrls,
+  };
+  return config;
+}
+
+export function ensureDirs(config) {
+  for (const dir of [config.sessionDir, config.configDir]) {
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 });
+  }
+}

package/src/poller.mjs

@@ -0,0 +1,109 @@
+/**
+ * Shiftacle API Poller
+ *
+ * Polls the Shiftacle API for pending EMR exports.
+ * When a note arrives (from mobile "Send to EMR"), forwards it
+ * to the local vision-guided agent for auto-fill.
+ */
+
+export function startPoller(config, auth, { onNote, onError } = {}) {
+  let running = true;
+  let interval = config.pollIntervalMs;
+  let consecutiveErrors = 0;
+  let timer = null;
+
+  async function poll() {
+    if (!running) return;
+
+    try {
+      const res = await fetch(`${config.apiBase}/emr-pipeline/queue?status=approved&limit=5`, {
+        headers: {
+          Authorization: `Bearer ${auth.token}`,
+          'Content-Type': 'application/json',
+        },
+      });
+
+      if (res.status === 401) {
+        // Token expired — caller should re-auth
+        onError?.({ type: 'auth', message: 'Token expired' });
+        interval = 30000; // slow down, wait for re-auth
+        scheduleNext();
+        return;
+      }
+
+      if (!res.ok) throw new Error(`API returned ${res.status}`);
+
+      const body = await res.json();
+      const pending = body.data || body.exports || [];
+
+      for (const entry of pending) {
+        try {
+          onNote?.(entry);
+
+          // Forward to local agent
+          const fillRes = await fetch(`http://localhost:${config.port}/auto-fill`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+              data: {
+                soap: entry.structuredNote?.soap || entry.soap || {},
+                billing: entry.structuredNote?.billing || entry.billing || {},
+                patientName: entry.patientLabel || entry.patientName,
+              },
+              patientName: entry.patientLabel || entry.patientName,
+              noteType: entry.noteType || 'daily',
+              visitDate: entry.visitDate || new Date().toLocaleDateString(),
+              emrType: entry.exportTarget || config.emr,
+            }),
+          });
+
+          const result = await fillRes.json();
+
+          // Report result back to Shiftacle API
+          await fetch(`${config.apiBase}/emr-pipeline/queue/${entry._id || entry.id}/export`, {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+              Authorization: `Bearer ${auth.token}`,
+            },
+            body: JSON.stringify({
+              target: entry.exportTarget || config.emr,
+              status: result.success ? 'exported' : 'failed',
+              error: result.error,
+              steps: result.steps,
+              duration: result.duration,
+            }),
+          });
+        } catch (err) {
+          onError?.({ type: 'fill', entry, message: err.message });
+        }
+      }
+
+      consecutiveErrors = 0;
+      interval = config.pollIntervalMs;
+    } catch (err) {
+      consecutiveErrors++;
+      // Exponential backoff: 5s, 10s, 20s, 40s, max 60s
+      interval = Math.min(config.pollIntervalMs * 2 ** consecutiveErrors, 60000);
+      onError?.({ type: 'poll', message: err.message, retryIn: interval });
+    }
+
+    scheduleNext();
+  }
+
+  function scheduleNext() {
+    if (running) {
+      timer = setTimeout(poll, interval);
+    }
+  }
+
+  // Start first poll
+  poll();
+
+  return {
+    stop() {
+      running = false;
+      if (timer) clearTimeout(timer);
+    },
+  };
+}

package/src/server.mjs

@@ -0,0 +1,260 @@
+/**
+ * Shiftacle Desktop Agent — Local API Server
+ *
+ * Runs on the clinician's computer. Receives notes from the
+ * Shiftacle mobile app and auto-fills them into the EMR using
+ * vision-guided browser automation.
+ */
+
+import express from 'express';
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
+import { join } from 'path';
+import VisionNavigator from './vision-navigator.mjs';
+
+export function startServer(config) {
+  const app = express();
+  app.use(express.json({ limit: '10mb' }));
+
+  // CORS for local use
+  app.use((req, res, next) => {
+    res.header('Access-Control-Allow-Origin', '*');
+    res.header('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
+    res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization');
+    if (req.method === 'OPTIONS') return res.sendStatus(200);
+    next();
+  });
+
+  if (!existsSync(config.sessionDir)) {
+    mkdirSync(config.sessionDir, { recursive: true });
+  }
+
+  // Active browser sessions
+  const navigators = new Map();
+
+  // Health check
+  app.get('/health', (req, res) => {
+    res.json({
+      status: 'ok',
+      engine: 'shiftacle-desktop-agent',
+      version: '0.1.0',
+      activeSessions: navigators.size,
+      visionModel: config.visionModel,
+      emr: config.emr,
+    });
+  });
+
+  // Start a new EMR session (opens browser)
+  app.post('/session/start', async (req, res) => {
+    const { emrType = config.emr, loginUrl } = req.body;
+    const sessionId = `emr_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+
+    try {
+      const nav = new VisionNavigator({
+        visionUrl: config.visionUrl,
+        visionModel: config.visionModel,
+        onStep: (step) =>
+          console.log(
+            `  [${sessionId}] Step ${step.step}: ${step.action} — ${(step.observation || '').substring(0, 80)}`,
+          ),
+      });
+
+      const sessionFile = join(config.sessionDir, `${emrType}.json`);
+      const storageState = existsSync(sessionFile) ? sessionFile : undefined;
+
+      await nav.launch({ headless: config.headless, storageState });
+
+      const url = loginUrl || config.emrUrls[emrType] || config.emrUrls.hellonote;
+      await nav.page.goto(url, { waitUntil: 'domcontentloaded' });
+
+      navigators.set(sessionId, { nav, emrType, createdAt: Date.now() });
+
+      // Save session after user has time to log in
+      setTimeout(async () => {
+        try {
+          await nav.saveSession(sessionFile);
+        } catch {}
+      }, 15000);
+
+      res.json({
+        success: true,
+        sessionId,
+        emrType,
+        message: 'Browser opened — log in to your EMR if needed',
+      });
+    } catch (err) {
+      res.status(500).json({ success: false, error: err.message });
+    }
+  });
+
+  // List active sessions
+  app.get('/sessions', (req, res) => {
+    const sessions = [];
+    for (const [id, sess] of navigators) {
+      sessions.push({
+        sessionId: id,
+        emrType: sess.emrType,
+        age: Math.round((Date.now() - sess.createdAt) / 60000) + ' min',
+      });
+    }
+    res.json({ success: true, sessions });
+  });
+
+  // Fill a note into the current EMR session
+  app.post('/fill', async (req, res) => {
+    const { sessionId, data, patientName, noteType, visitDate } = req.body;
+
+    let nav;
+    if (sessionId && navigators.has(sessionId)) {
+      nav = navigators.get(sessionId).nav;
+    } else {
+      const entries = [...navigators.entries()];
+      if (entries.length > 0) {
+        nav = entries[entries.length - 1][1].nav;
+      } else {
+        return res.status(400).json({
+          success: false,
+          error: 'No active EMR session. Start one first: POST /session/start',
+        });
+      }
+    }
+
+    try {
+      console.log('  [EMR-Fill] Starting vision-guided note fill...');
+      const result = await nav.fillNote({
+        soap: data?.soap || {},
+        billing: data?.billing || {},
+        patientName: patientName || data?.patientName,
+        noteType: noteType || data?.noteType || 'daily',
+        visitDate: visitDate || data?.visitDate || new Date().toLocaleDateString(),
+      });
+
+      console.log(
+        `  [EMR-Fill] ${result.success ? 'SUCCESS' : 'FAILED'} in ${result.steps} steps (${Math.round(result.duration / 1000)}s)`,
+      );
+
+      if (result.success) {
+        try {
+          const emrType = navigators.values().next().value?.emrType || config.emr;
+          await nav.saveSession(join(config.sessionDir, `${emrType}.json`));
+        } catch {}
+      }
+
+      res.json({
+        success: result.success,
+        steps: result.steps,
+        duration: result.duration,
+        error: result.error,
+        history: result.history?.slice(-5),
+      });
+    } catch (err) {
+      res.status(500).json({ success: false, error: err.message });
+    }
+  });
+
+  // Auto-fill: open browser if needed, navigate, fill
+  app.post('/auto-fill', async (req, res) => {
+    const { data, patientName, noteType, visitDate, emrType = config.emr } = req.body;
+
+    try {
+      const nav = new VisionNavigator({
+        visionUrl: config.visionUrl,
+        visionModel: config.visionModel,
+        maxSteps: 40,
+        onStep: (step) =>
+          console.log(
+            `  [auto-fill] Step ${step.step}/${step.total}: ${step.action} — ${(step.observation || '').substring(0, 60)}`,
+          ),
+      });
+
+      const sessionFile = join(config.sessionDir, `${emrType}.json`);
+      await nav.launch({
+        headless: config.headless,
+        storageState: existsSync(sessionFile) ? sessionFile : undefined,
+      });
+
+      const url = config.emrUrls[emrType] || config.emrUrls.hellonote;
+      await nav.page.goto(url, { waitUntil: 'domcontentloaded' });
+      await nav.page.waitForTimeout(2000);
+
+      const result = await nav.fillNote({
+        soap: data?.soap || {},
+        billing: data?.billing || {},
+        patientName,
+        noteType: noteType || 'daily',
+        visitDate: visitDate || new Date().toLocaleDateString(),
+      });
+
+      if (result.success) {
+        try {
+          await nav.saveSession(sessionFile);
+        } catch {}
+      }
+
+      const newId = `autofill_${Date.now()}`;
+      navigators.set(newId, { nav, emrType, createdAt: Date.now() });
+
+      res.json({
+        success: result.success,
+        sessionId: newId,
+        steps: result.steps,
+        duration: result.duration,
+        error: result.error,
+        history: result.history?.slice(-5),
+      });
+    } catch (err) {
+      res.status(500).json({ success: false, error: err.message });
+    }
+  });
+
+  // Screenshot of a session
+  app.get('/screenshot/:sessionId', async (req, res) => {
+    const sess = navigators.get(req.params.sessionId);
+    if (!sess) return res.status(404).json({ error: 'Session not found' });
+
+    try {
+      const buffer = await sess.nav.page.screenshot({ type: 'jpeg', quality: 80 });
+      res.set('Content-Type', 'image/jpeg');
+      res.send(buffer);
+    } catch (err) {
+      res.status(500).json({ error: err.message });
+    }
+  });
+
+  // Close a session
+  app.delete('/session/:sessionId', async (req, res) => {
+    const sess = navigators.get(req.params.sessionId);
+    if (!sess) return res.status(404).json({ error: 'Session not found' });
+
+    await sess.nav.close();
+    navigators.delete(req.params.sessionId);
+    res.json({ success: true });
+  });
+
+  // Cleanup expired sessions every 30 minutes
+  const cleanupTimer = setInterval(() => {
+    const maxAge = 24 * 60 * 60 * 1000;
+    for (const [id, sess] of navigators) {
+      if (Date.now() - sess.createdAt > maxAge) {
+        sess.nav.close().catch(() => {});
+        navigators.delete(id);
+        console.log(`  [Cleanup] Closed expired session ${id}`);
+      }
+    }
+  }, 30 * 60 * 1000);
+
+  const server = app.listen(config.port);
+
+  return {
+    app,
+    server,
+    navigators,
+    close() {
+      clearInterval(cleanupTimer);
+      for (const [, sess] of navigators) {
+        sess.nav.close().catch(() => {});
+      }
+      navigators.clear();
+      server.close();
+    },
+  };
+}

package/src/vision-navigator.mjs

@@ -0,0 +1,303 @@
+/**
+ * VisionNavigator — AI-guided EMR browser automation
+ *
+ * Instead of fragile CSS selectors, this agent:
+ * 1. Takes a screenshot of the current page
+ * 2. Sends it to a vision model (Qwen3-VL)
+ * 3. The model identifies elements and returns coordinates/actions
+ * 4. Playwright executes the action
+ * 5. Repeat until the task is done
+ *
+ * Works on ANY web-based EMR — HelloNote, WebPT, Epic, Cerner —
+ * because it "sees" the screen like a human.
+ */
+
+import { chromium } from 'playwright';
+
+const MAX_STEPS = 30;
+const STEP_DELAY = 1000;
+
+export default class VisionNavigator {
+  constructor(options = {}) {
+    this.browser = null;
+    this.page = null;
+    this.visionUrl = options.visionUrl || 'https://vision.shiftacle.com/v1';
+    this.visionModel = options.visionModel || 'Qwen/Qwen3-VL-8B-Instruct';
+    this.maxSteps = options.maxSteps || MAX_STEPS;
+    this.stepDelay = options.stepDelay || STEP_DELAY;
+    this.history = [];
+    this.onStep = options.onStep || (() => {});
+  }
+
+  async launch(options = {}) {
+    this.browser = await chromium.launch({
+      headless: options.headless !== undefined ? options.headless : false,
+      args: ['--window-size=1280,900'],
+    });
+    const context = await this.browser.newContext({
+      viewport: { width: 1280, height: 900 },
+      ...(options.storageState ? { storageState: options.storageState } : {}),
+    });
+    this.page = await context.newPage();
+    return this.page;
+  }
+
+  async screenshot() {
+    const buffer = await this.page.screenshot({ type: 'jpeg', quality: 80 });
+    return buffer.toString('base64');
+  }
+
+  async think(task, context = '') {
+    const imageBase64 = await this.screenshot();
+    const currentUrl = this.page.url();
+
+    const systemPrompt = `You are an AI browser agent navigating a healthcare EMR (Electronic Medical Records) system. You can see a screenshot of the current page.
+
+Your job is to complete the given task by identifying UI elements and telling the system what to click, type, or do next.
+
+RULES:
+- Describe what you see on the page first (briefly)
+- Identify the EXACT element to interact with
+- Give precise coordinates (x, y) for clicks — estimate from the screenshot
+- For text input: specify the field coordinates AND the text to type
+- Only do ONE action per step
+- If you see a loading spinner or modal, wait
+- If you're on the wrong page, navigate back or to the correct URL
+- If login is required, say so — don't guess credentials
+- If you're stuck after 3 attempts on the same step, report failure
+
+AVAILABLE ACTIONS:
+- click(x, y) — click at coordinates
+- type(x, y, "text") — click field at coordinates and type text
+- scroll(direction, amount) — scroll "up" or "down" by pixels
+- wait(ms) — wait for page to load
+- navigate(url) — go to a URL
+- select(x, y, "option") — click dropdown at coordinates, then click option
+- done() — task is complete
+- fail(reason) — cannot complete the task
+
+Current URL: ${currentUrl}
+${context ? `Context: ${context}` : ''}
+
+Return JSON:
+{
+  "observation": "what I see on the page",
+  "reasoning": "why I'm taking this action",
+  "action": "click|type|scroll|wait|navigate|select|done|fail",
+  "params": { "x": 640, "y": 450, "text": "optional", "url": "optional", "direction": "optional", "amount": "optional", "reason": "optional" }
+}`;
+
+    const response = await fetch(`${this.visionUrl}/chat/completions`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        model: this.visionModel,
+        messages: [
+          { role: 'system', content: systemPrompt },
+          {
+            role: 'user',
+            content: [
+              { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${imageBase64}` } },
+              { type: 'text', text: `TASK: ${task}\n\nWhat do you see and what should I do next?` },
+            ],
+          },
+        ],
+        max_tokens: 500,
+        temperature: 0.1,
+        response_format: { type: 'json_object' },
+      }),
+    });
+
+    if (!response.ok) throw new Error(`Vision model error: ${response.status}`);
+
+    const result = await response.json();
+    const content = result.choices?.[0]?.message?.content;
+
+    try {
+      return JSON.parse(content);
+    } catch {
+      const match = content?.match(/\{[\s\S]*\}/);
+      if (match) return JSON.parse(match[0]);
+      throw new Error('Vision model returned non-JSON');
+    }
+  }
+
+  async executeAction(action) {
+    const { action: type, params } = action;
+
+    switch (type) {
+      case 'click':
+        await this.page.mouse.click(params.x, params.y);
+        break;
+      case 'type':
+        await this.page.mouse.click(params.x, params.y);
+        await this.page.waitForTimeout(200);
+        await this.page.keyboard.press('Control+a');
+        await this.page.keyboard.type(params.text, { delay: 30 });
+        break;
+      case 'scroll':
+        await this.page.mouse.wheel(
+          0,
+          params.direction === 'up' ? -(params.amount || 300) : (params.amount || 300),
+        );
+        break;
+      case 'wait':
+        await this.page.waitForTimeout(params.ms || 2000);
+        break;
+      case 'navigate':
+        await this.page.goto(params.url, { waitUntil: 'domcontentloaded' });
+        break;
+      case 'select':
+        await this.page.mouse.click(params.x, params.y);
+        await this.page.waitForTimeout(500);
+        if (params.text) await this.page.keyboard.type(params.text, { delay: 50 });
+        await this.page.waitForTimeout(300);
+        await this.page.keyboard.press('Enter');
+        break;
+      case 'done':
+      case 'fail':
+        break;
+      default:
+        throw new Error(`Unknown action: ${type}`);
+    }
+
+    await this.page.waitForTimeout(this.stepDelay);
+  }
+
+  async runTask(task, context = '') {
+    const startTime = Date.now();
+    let stuckCount = 0;
+    let lastObservation = '';
+
+    for (let step = 0; step < this.maxSteps; step++) {
+      try {
+        const decision = await this.think(task, context);
+
+        this.history.push({
+          step: step + 1,
+          observation: decision.observation,
+          reasoning: decision.reasoning,
+          action: decision.action,
+          params: decision.params,
+          timestamp: Date.now(),
+        });
+
+        this.onStep({
+          step: step + 1,
+          total: this.maxSteps,
+          action: decision.action,
+          observation: decision.observation,
+        });
+
+        if (decision.observation === lastObservation) {
+          stuckCount++;
+          if (stuckCount >= 3) {
+            return {
+              success: false,
+              steps: step + 1,
+              history: this.history,
+              error: 'Agent stuck — same observation 3 times',
+              duration: Date.now() - startTime,
+            };
+          }
+        } else {
+          stuckCount = 0;
+        }
+        lastObservation = decision.observation;
+
+        if (decision.action === 'done') {
+          return { success: true, steps: step + 1, history: this.history, duration: Date.now() - startTime };
+        }
+        if (decision.action === 'fail') {
+          return {
+            success: false,
+            steps: step + 1,
+            history: this.history,
+            error: decision.params?.reason || 'Agent reported failure',
+            duration: Date.now() - startTime,
+          };
+        }
+
+        await this.executeAction(decision);
+      } catch (err) {
+        this.history.push({ step: step + 1, error: err.message, timestamp: Date.now() });
+        if (step > this.maxSteps - 3) {
+          return {
+            success: false,
+            steps: step + 1,
+            history: this.history,
+            error: err.message,
+            duration: Date.now() - startTime,
+          };
+        }
+      }
+    }
+
+    return {
+      success: false,
+      steps: this.maxSteps,
+      history: this.history,
+      error: 'Max steps reached',
+      duration: Date.now() - startTime,
+    };
+  }
+
+  async fillNote(noteData, emrConfig = {}) {
+    const emrType = emrConfig.type || 'hellonote';
+    const { soap, billing, patientName, noteType, visitDate } = noteData;
+
+    const noteContext = `
+EMR Type: ${emrType}
+Patient: ${patientName || 'Unknown'}
+Visit Date: ${visitDate || 'Today'}
+Note Type: ${noteType || 'Daily Note'}
+
+SOAP NOTE TO ENTER:
+SUBJECTIVE: ${soap?.subjective || '[none]'}
+OBJECTIVE: ${soap?.objective || '[none]'}
+ASSESSMENT: ${soap?.assessment || '[none]'}
+PLAN: ${soap?.plan || '[none]'}
+
+${billing?.cpt_codes ? `BILLING CODES: ${Object.entries(billing.cpt_codes).map(([code, info]) => `${code}: ${info.minutes || 15} min, ${info.units || 1} unit`).join(', ')}` : ''}
+`;
+
+    let task;
+    if (patientName) {
+      task = `In this ${emrType} EMR system:
+1. Find patient "${patientName}" using the search/patient list
+2. Open or create a ${noteType || 'daily'} note for ${visitDate || 'today'}
+3. Fill in the SUBJECTIVE, OBJECTIVE, ASSESSMENT, and PLAN sections with the provided text
+4. If there's a billing section, enter the CPT codes and minutes
+5. Save as draft (do NOT finalize/sign)
+6. Report done when complete`;
+    } else {
+      task = `In this ${emrType} EMR system, I'm already on a note form. Fill in:
+1. The SUBJECTIVE section with the provided text
+2. The OBJECTIVE section
+3. The ASSESSMENT section
+4. The PLAN section
+5. If there's a billing section, enter the CPT codes
+6. Save as draft
+7. Report done when complete`;
+    }
+
+    return this.runTask(task, noteContext);
+  }
+
+  async saveSession(path) {
+    if (this.page) {
+      const { writeFileSync } = await import('fs');
+      const state = await this.page.context().storageState();
+      writeFileSync(path, JSON.stringify(state));
+      return path;
+    }
+  }
+
+  async close() {
+    if (this.browser) {
+      await this.browser.close();
+      this.browser = null;
+      this.page = null;
+    }
+  }
+}