shmakk 1.2.3 → 1.2.5

package/src/mobile.js

@@ -0,0 +1,307 @@
+// Mobile app automation via ADB (Android Debug Bridge).
+// Manages a persistent ADB connection to an Android device/emulator
+// across the session. The agent uses this through the `mobile` tool
+// in tools.js.
+//
+// Requires: adb on PATH, device connected or emulator running.
+
+const { execFileSync, execFile } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const SCREENSHOT_DIR = '/tmp/shmakk-screenshots';
+
+let _deviceId = null; // cached device serial, set on first ensure
+
+function isAvailable() {
+  try {
+    execFileSync('adb', ['version'], { timeout: 5000, stdio: 'ignore' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function adb(args, opts = {}) {
+  const fullArgs = _deviceId ? ['-s', _deviceId, ...args] : args;
+  const { stdout } = execFileSync('adb', fullArgs, {
+    timeout: opts.timeout || 15000,
+    maxBuffer: opts.maxBuffer || 2 * 1024 * 1024,
+    stdio: ['ignore', 'pipe', 'pipe'],
+    encoding: 'buffer',
+  });
+  return (stdout || '').toString();
+}
+
+function adbAsync(args, opts = {}) {
+  return new Promise((resolve, reject) => {
+    const fullArgs = _deviceId ? ['-s', _deviceId, ...args] : args;
+    execFile('adb', fullArgs, {
+      timeout: opts.timeout || 15000,
+      maxBuffer: opts.maxBuffer || 2 * 1024 * 1024,
+    }, (err, stdout) => {
+      if (err) reject(err);
+      else resolve((stdout || '').toString());
+    });
+  });
+}
+
+function ensureDevice() {
+  if (_deviceId) return _deviceId;
+
+  try {
+    const devices = adb(['devices']);
+    const lines = devices.split('\n').slice(1).filter(l => l.trim() && !l.startsWith('*'));
+    const online = lines.filter(l => /\tdevice/.test(l)).map(l => l.split('\t')[0].trim());
+
+    if (online.length === 0) {
+      throw new Error(
+        'No Android device/emulator found. Connect a device via USB or start an emulator.\n' +
+        'Run: adb devices  to check available devices.'
+      );
+    }
+
+    _deviceId = online[0];
+    return _deviceId;
+  } catch (e) {
+    if (e.message.includes('No Android device')) throw e;
+    throw new Error(`ADB error: ${e.message}`);
+  }
+}
+
+// ── Commands ──────────────────────────────────────────────────────────────
+
+async function screenshot(args) {
+  try {
+    ensureDevice();
+    const ts = Date.now();
+    if (!fs.existsSync(SCREENSHOT_DIR)) fs.mkdirSync(SCREENSHOT_DIR, { recursive: true });
+
+    const remotePath = '/sdcard/shmakk_screen.png';
+    const localPath = path.join(SCREENSHOT_DIR, `mobile-${ts}.png`);
+
+    // Capture screenshot on device
+    adb(['shell', 'screencap', '-p', remotePath]);
+
+    // Pull to local
+    adb(['pull', remotePath, localPath]);
+
+    // Clean up remote file
+    try { adb(['shell', 'rm', remotePath]); } catch {}
+
+    // Return base64 for vision LLMs
+    const buf = fs.readFileSync(localPath);
+    const b64 = buf.toString('base64');
+
+    return {
+      ok: true,
+      path: localPath,
+      mimeType: 'image/png',
+      images: [{
+        mimeType: 'image/png',
+        data: b64,
+        dataLength: b64.length,
+        truncated: false,
+      }],
+    };
+  } catch (e) {
+    return { error: `screenshot failed: ${e.message}` };
+  }
+}
+
+async function click(args) {
+  const x = Number(args.x);
+  const y = Number(args.y);
+  if (isNaN(x) || isNaN(y)) return { error: 'x and y coordinates required' };
+
+  try {
+    ensureDevice();
+    adb(['shell', 'input', 'tap', String(x), String(y)]);
+    return { ok: true, action: 'click', x, y };
+  } catch (e) {
+    return { error: `click failed: ${e.message}` };
+  }
+}
+
+async function type(args) {
+  const text = String(args.text || '');
+  if (!text) return { error: 'text required' };
+
+  try {
+    ensureDevice();
+    // Escape special shell chars for input text
+    const escaped = text.replace(/(['"\\])/g, '\\$1');
+    adb(['shell', 'input', 'text', escaped]);
+    return { ok: true, action: 'type', length: text.length };
+  } catch (e) {
+    return { error: `type failed: ${e.message}` };
+  }
+}
+
+async function swipe(args) {
+  const x1 = Number(args.x1);
+  const y1 = Number(args.y1);
+  const x2 = Number(args.x2);
+  const y2 = Number(args.y2);
+  const duration = Number(args.duration) || 300;
+
+  if ([x1, y1, x2, y2].some(isNaN)) {
+    return { error: 'x1, y1, x2, y2 coordinates required' };
+  }
+
+  try {
+    ensureDevice();
+    adb(['shell', 'input', 'swipe', String(x1), String(y1), String(x2), String(y2), String(duration)]);
+    return { ok: true, action: 'swipe', from: [x1, y1], to: [x2, y2], duration };
+  } catch (e) {
+    return { error: `swipe failed: ${e.message}` };
+  }
+}
+
+async function key(args) {
+  const code = String(args.code || '').toUpperCase();
+  if (!code) return { error: 'key code required (e.g. BACK, HOME, ENTER, DELETE)' };
+
+  try {
+    ensureDevice();
+    adb(['shell', 'input', 'keyevent', `KEYCODE_${code}`]);
+    return { ok: true, action: 'key', code };
+  } catch (e) {
+    return { error: `key failed: ${e.message}` };
+  }
+}
+
+async function readPage(args) {
+  try {
+    ensureDevice();
+    const remotePath = '/sdcard/shmakk_ui.xml';
+    const localPath = path.join(os.tmpdir(), `shmakk-ui-${Date.now()}.xml`);
+
+    adb(['shell', 'uiautomator', 'dump', remotePath]);
+    adb(['pull', remotePath, localPath]);
+    try { adb(['shell', 'rm', remotePath]); } catch {}
+
+    const xml = fs.readFileSync(localPath, 'utf8');
+    try { fs.unlinkSync(localPath); } catch {}
+
+    // Parse basic structure from accessibility tree
+    const elements = [];
+    const tagRe = /<node[^>]*>/g;
+    let match;
+    while ((match = tagRe.exec(xml)) !== null) {
+      const tag = match[0];
+      const textMatch = tag.match(/text="([^"]*)"/);
+      const classMatch = tag.match(/class="([^"]*)"/);
+      const boundsMatch = tag.match(/bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"/);
+      const clickable = tag.includes('clickable="true"');
+      const checkable = tag.includes('checkable="true"');
+      const scrollable = tag.includes('scrollable="true"');
+      const focused = tag.includes('focused="true"');
+      const enabled = tag.includes('enabled="true"');
+
+      elements.push({
+        text: textMatch ? textMatch[1] : '',
+        className: classMatch ? classMatch[1].split('.').pop() : '',
+        bounds: boundsMatch ? {
+          x: parseInt(boundsMatch[1]),
+          y: parseInt(boundsMatch[2]),
+          w: parseInt(boundsMatch[3]) - parseInt(boundsMatch[1]),
+          h: parseInt(boundsMatch[4]) - parseInt(boundsMatch[2]),
+        } : null,
+        clickable,
+        checkable,
+        scrollable,
+        focused,
+        enabled,
+      });
+    }
+
+    return {
+      ok: true,
+      elements: elements.slice(0, 200), // Cap for size
+      totalElements: elements.length,
+      truncated: elements.length > 200,
+    };
+  } catch (e) {
+    return { error: `read_page failed: ${e.message}` };
+  }
+}
+
+async function listApps(args) {
+  try {
+    ensureDevice();
+    const filter = String(args.filter || '').trim();
+    const output = adb(['shell', 'pm', 'list', 'packages', ...(filter ? [filter] : [])]);
+    const packages = output.split('\n')
+      .filter(l => l.startsWith('package:'))
+      .map(l => l.replace('package:', '').trim());
+
+    return { ok: true, packages, count: packages.length };
+  } catch (e) {
+    return { error: `list_apps failed: ${e.message}` };
+  }
+}
+
+async function launch(args) {
+  const pkg = String(args.package || '').trim();
+  if (!pkg) return { error: 'package name required' };
+
+  try {
+    ensureDevice();
+    // Try launching the main activity first
+    const output = adb(['shell', 'monkey', '-p', pkg, '-c', 'android.intent.category.LAUNCHER', '1'], { timeout: 10000 });
+    return { ok: true, action: 'launch', package: pkg };
+  } catch (e) {
+    return { error: `launch failed: ${e.message}` };
+  }
+}
+
+async function close(args) {
+  const pkg = String(args.package || '').trim();
+  if (!pkg) return { error: 'package name required' };
+
+  try {
+    ensureDevice();
+    adb(['shell', 'am', 'force-stop', pkg]);
+    return { ok: true, action: 'close', package: pkg };
+  } catch (e) {
+    return { error: `close failed: ${e.message}` };
+  }
+}
+
+async function wait(args) {
+  const ms = Number(args.ms) || 1000;
+  await new Promise(r => setTimeout(r, Math.min(ms, 30000)));
+  return { ok: true, action: 'wait', ms };
+}
+
+// ── Dispatch ──────────────────────────────────────────────────────────────
+
+const COMMANDS = { screenshot, click, type, swipe, key, read_page: readPage, list_apps: listApps, launch, close, wait };
+
+function classifyMobileCommand(args) {
+  const cmd = String(args.command || '');
+  // All mobile commands are potentially destructive (interact with a real device)
+  if (cmd === 'screenshot' || cmd === 'read_page' || cmd === 'list_apps' || cmd === 'wait') return 'safe';
+  if (cmd === 'click' || cmd === 'type' || cmd === 'swipe' || cmd === 'key') return 'uncertain';
+  if (cmd === 'launch' || cmd === 'close') return 'unsafe';
+  return 'uncertain';
+}
+
+async function dispatchMobile(args, signal) {
+  if (!isAvailable()) {
+    return { error: 'adb not found. Install Android SDK platform tools.' };
+  }
+  const cmd = String(args.command || '');
+  const fn = COMMANDS[cmd];
+  if (!fn) return { error: `unknown mobile command: ${cmd}. Available: ${Object.keys(COMMANDS).join(', ')}` };
+  try {
+    const result = await fn(args);
+    return result;
+  } catch (e) {
+    return { error: `mobile ${cmd} failed: ${e.message}` };
+  }
+}
+
+module.exports = { dispatchMobile, classifyMobileCommand, isAvailable };

package/src/notify.js

@@ -9,7 +9,6 @@ const NOTIFY_BIN = 'notify-send';
 
 function available() {
   try {
-    // Prefer direct path check; fall back to `command -v` if not at known paths
     if (existsSync('/usr/bin/notify-send')) return true;
     if (existsSync('/usr/local/bin/notify-send')) return true;
     execFileSync('command', ['-v', NOTIFY_BIN], { stdio: 'ignore' });
@@ -29,9 +28,58 @@ function notify(summary, body, urgency) {
   if (urgency === 'critical') args.push('--urgency=critical');
   if (urgency === 'low') args.push('--urgency=low');
   execFile(NOTIFY_BIN, args, (err) => {
-    // Silently ignore — notification daemon may not be running.
     void err;
   });
 }
 
-module.exports = { notify, available };
+// Interactive notification with Yes/No action buttons.
+// Returns { promise, proc } where:
+//   promise resolves to 'yes', 'no', or null (dismissed/error)
+//   proc is the child process (call .kill() to cancel)
+function notifyAsk(summary, body) {
+  if (!available()) return { promise: Promise.resolve(null), proc: null };
+
+  const args = [
+    summary || 'shmakk',
+    body || '',
+    '--app-name=shmakk',
+    '--category=im.received',
+    '--urgency=critical',
+    '--expire-time=30000',
+    '--action=yes=Yes',
+    '--action=no=No',
+    '--wait',
+  ];
+
+  let settled = false;
+  let proc;
+
+  const promise = new Promise((resolve) => {
+    proc = execFile(NOTIFY_BIN, args, { timeout: 60000, encoding: 'utf8' }, (err, stdout) => {
+      if (settled) return;
+      settled = true;
+      if (err) {
+        resolve(null);
+        return;
+      }
+      const action = (stdout || '').trim().toLowerCase();
+      if (action === 'yes' || action === 'no') {
+        resolve(action);
+      } else {
+        resolve(null);
+      }
+    });
+  });
+
+  return {
+    promise,
+    get proc() { return proc; },
+    cancel() {
+      if (settled) return;
+      settled = true;
+      try { proc.kill(); } catch {}
+    },
+  };
+}
+
+module.exports = { notify, notifyAsk, available };

package/src/orchestrator.js

@@ -1,11 +1,45 @@
 // shmakk orchestrator entry point.
 // Manages signal-driven lifecycle (restart, exit, profile changes) and
 // delegates each session to ./session.js.
+//
+// Browser CDP connector — loaded here so tools.js can discover it
+// through the orchestrator without requiring a direct dependency on
+// ./core/browserConnector from every call site.
 
 const { runOneSession } = require('./session');
 const { normalizeProfile } = require('./profiles');
 const { profileSignalPath } = require('./control');
 
+// Lazy-load the browser CDP connector (Playwright optional dep).
+let _browserConnector = null;
+function _getBrowserConnector() {
+  if (_browserConnector) return _browserConnector;
+  try {
+    _browserConnector = require('./core/browserConnector');
+  } catch (e) {
+    _browserConnector = null;
+  }
+  return _browserConnector;
+}
+
+// Execute a browser action against a user-connected Chrome instance via CDP.
+// This is the primary entry point for the agent's 'browser' tool when
+// the user has run `shmakk connect-browser` to attach to their own Chrome.
+async function executeBrowserAction(args) {
+  const bc = _getBrowserConnector();
+  if (!bc) {
+    return { error: 'Browser connector unavailable. playwright must be installed as an optional dependency.' };
+  }
+  return await bc.executeBrowserAction(args);
+}
+
+// Classify a browser action for safety gating.
+function classifyBrowserAction(args) {
+  const bc = _getBrowserConnector();
+  if (!bc) return 'uncertain';
+  return bc.classifyBrowserAction(args);
+}
+
 function isAbortError(e) {
   return e && (e.name === 'AbortError' || /aborted/i.test(String(e.message || '')));
 }
@@ -58,4 +92,4 @@ async function start(opts) {
   return lastExit;
 }
 
-module.exports = { start };
+module.exports = { start, executeBrowserAction, classifyBrowserAction };

package/src/pty.js

@@ -13,7 +13,7 @@ function getSize() {
 
 const VOICE_HOTKEY = 0x0f; // Ctrl+O — triggers voice recording
 
-function startSession({ debug = false, voiceEnabled = false, shellOverride = null } = {}) {
+function startSession({ debug = false, voiceEnabled = false, stsEnabled = false, shellOverride = null, extraEnv = {}, cleanup = null } = {}) {
   const shell = detectShell(shellOverride);
   const cfg = configureForShell(shell.name);
   const { cols, rows } = getSize();
@@ -28,10 +28,11 @@ function startSession({ debug = false, voiceEnabled = false, shellOverride = nul
     cwd: process.cwd(),
     // SHMAKK_PID lets `shmakk --status/--exit/--restart` find us from
     // inside the inner shell.
-    env: { ...process.env, SHMAKK: '1', SHMAKK_PID: String(process.pid), ...cfg.env },
+    env: { ...process.env, SHMAKK: '1', SHMAKK_PID: String(process.pid), ...cfg.env, ...extraEnv },
   });
 
   const ev = new EventEmitter();
+  let voiceHotkeyEnabled = !!(voiceEnabled || stsEnabled);
   // Stack of stdin handlers: top of stack receives data. Null at bottom
   // means "relay to child PTY".
   const stdinStack = [];
@@ -47,11 +48,8 @@ function startSession({ debug = false, voiceEnabled = false, shellOverride = nul
   stdin.resume();
 
   const onStdin = (data) => {
-    const h = topHandler();
-    if (h) return h(data);
-
     // Voice hotkey detection — only when voice is enabled
-    if (voiceEnabled) {
+    if (voiceHotkeyEnabled) {
       const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
       if (buf.length === 1 && buf[0] === VOICE_HOTKEY) {
         ev.emit('voice');
@@ -59,6 +57,9 @@ function startSession({ debug = false, voiceEnabled = false, shellOverride = nul
       }
     }
 
+    const h = topHandler();
+    if (h) return h(data);
+
     const cleaned = filterStdin(Buffer.isBuffer(data) ? data : Buffer.from(data));
     if (cleaned.length) child.write(cleaned);
   };
@@ -82,6 +83,7 @@ function startSession({ debug = false, voiceEnabled = false, shellOverride = nul
       if (stdin.isTTY) { try { stdin.setRawMode(wasRaw); } catch {} }
       stdin.pause();
       cfg.cleanup();
+      if (cleanup) cleanup();
       resolve({ exitCode: exitCode ?? 0, signal });
     });
   });
@@ -90,6 +92,9 @@ function startSession({ debug = false, voiceEnabled = false, shellOverride = nul
     ev,
     stdoutWrite: (s) => stdout.write(s),
     childWrite: (s) => child.write(s),
+    setVoiceEnabled(v) {
+      voiceHotkeyEnabled = !!v;
+    },
     // Push a handler; returns a release fn that pops it. Stacked so nested
     // captures (e.g. ask() inside an AI tap) don't wipe the outer handler.
     captureStdin(handler) {

package/src/review.js

@@ -1,29 +1,61 @@
 // Y/n prompt with cooperative cancellation. The returned `ask` accepts an
-// optional `{ onCancel }` callback that fires when the user hits Ctrl-C.
+// optional `{ onCancel, onWhy, notifyBody }` object.  When `notifyBody` is
+// provided it is used as the desktop notification text instead of the
+// terminal-only `question` string.
 
 function makePrompter(pty, write, opts = {}) {
-  return function ask(question, defaultYes, { onCancel, onWhy } = {}) {
+  return function ask(question, defaultYes, { onCancel, onWhy, notifyBody } = {}) {
     return new Promise((resolve) => {
-      if (opts.notify) {
-        try {
-          const { notify } = require('./notify');
-          const body = typeof question === 'string' ? question.replace(/\x1b\[[0-9;]*m/g, '') : String(question || '');
-          notify('shmakk needs your attention', body.slice(0, 120));
-        } catch {}
-      }
       const tag = defaultYes ? '[Y/n/?]' : '[y/N/?]';
       write(`${question} ${tag} `);
       let buf = '';
+
       function finishYesNo(ans) {
         write('\n');
         release();
         return resolve(ans);
       }
+
+      // Notification with Yes/No buttons races against terminal input
+      let notifResult = null;
+      if (opts.notify) {
+        try {
+          const { notifyAsk } = require('./notify');
+          const raw = notifyBody || question || '';
+          const clean = typeof raw === 'string'
+            ? raw.replace(/\x1b\[[0-9;]*m/g, '').trim()
+            : String(raw || '').trim();
+
+          // Summary: short label. Body: the full command/description.
+          const summary = clean.length > 60 ? clean.slice(0, 57) + '…' : (clean || 'shmakk');
+          notifResult = notifyAsk(summary, clean.slice(0, 200));
+
+          notifResult.promise.then((action) => {
+            if (action === 'yes') {
+              release();
+              write('\n\x1b[2m[approved via notification]\x1b[0m\n');
+              resolve(true);
+            } else if (action === 'no') {
+              release();
+              write('\n\x1b[2m[declined via notification]\x1b[0m\n');
+              resolve(false);
+            }
+            // null = dismissed: keep waiting for terminal input
+          });
+        } catch {}
+      }
+
       const release = pty.captureStdin((data) => {
         for (const ch of data.toString('utf8')) {
           const code = ch.charCodeAt(0);
-          if (!buf && (ch === 'y' || ch === 'Y')) return finishYesNo(true);
-          if (!buf && (ch === 'n' || ch === 'N')) return finishYesNo(false);
+          if (!buf && (ch === 'y' || ch === 'Y')) {
+            if (notifResult) notifResult.cancel();
+            return finishYesNo(true);
+          }
+          if (!buf && (ch === 'n' || ch === 'N')) {
+            if (notifResult) notifResult.cancel();
+            return finishYesNo(false);
+          }
           if (!buf && ch === '?') {
             write('\n');
             if (onWhy) onWhy();
@@ -43,6 +75,7 @@ function makePrompter(pty, write, opts = {}) {
               write(`${question} ${tag} `);
               return;
             }
+            if (notifResult) notifResult.cancel();
             release();
             return resolve(ans === 'y' || ans === 'yes');
           }
@@ -50,6 +83,7 @@ function makePrompter(pty, write, opts = {}) {
             if (buf.length) { buf = buf.slice(0, -1); write('\b \b'); }
           } else if (code === 0x03) { // Ctrl-C
             write('^C\n');
+            if (notifResult) notifResult.cancel();
             release();
             if (onCancel) onCancel();
             return resolve(false);