uwonbot 1.3.0 → 1.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "uwonbot",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Uwonbot AI Assistant CLI — Your AI controls your computer",
   "main": "src/index.js",
   "bin": {
@@ -15,7 +15,13 @@
     "start": "node bin/uwonbot.js",
     "dev": "node bin/uwonbot.js"
   },
-  "keywords": ["ai", "assistant", "automation", "cli", "computer-use"],
+  "keywords": [
+    "ai",
+    "assistant",
+    "automation",
+    "cli",
+    "computer-use"
+  ],
   "author": "Uwonbot",
   "license": "MIT",
   "repository": {

package/src/agent.js

@@ -5,7 +5,7 @@ import { promisify } from 'util';
 import chalk from 'chalk';
 import { getConfig } from './config.js';
 import ClapListener from './clapListener.js';
-import { fetchAssistants, setIdToken } from './firebase-client.js';
+import { fetchAssistants, setIdToken, setRefreshToken } from './firebase-client.js';
 
 const execAsync = promisify(exec);
 const WEB_APP_URL = 'https://chartapp-653e1.web.app';
@@ -193,16 +193,20 @@ async function mouseClick(button = 'left', double = false) {
             const { x, y } = _lastKnownPos;
             if (button === 'right') {
                 await cgMouseEvent('$.kCGEventRightMouseDown', x, y, '$.kCGMouseButtonRight');
+                await new Promise(r => setTimeout(r, 20));
                 await cgMouseEvent('$.kCGEventRightMouseUp', x, y, '$.kCGMouseButtonRight');
             } else {
                 await cgMouseEvent('$.kCGEventLeftMouseDown', x, y, 0);
+                await new Promise(r => setTimeout(r, 20));
                 await cgMouseEvent('$.kCGEventLeftMouseUp', x, y, 0);
                 if (double) {
-                    await new Promise(r => setTimeout(r, 50));
+                    await new Promise(r => setTimeout(r, 60));
                     await cgMouseEvent('$.kCGEventLeftMouseDown', x, y, 0);
+                    await new Promise(r => setTimeout(r, 20));
                     await cgMouseEvent('$.kCGEventLeftMouseUp', x, y, 0);
                 }
             }
+            console.log(chalk.green(`  [click] CG ${button}${double ? ' (double)' : ''} at (${x},${y})`));
         } catch {}
     }
 }
@@ -505,6 +509,8 @@ async function openTerminalWithChat(assistantName) {
 }
 
 let _logThrottle = 0;
+let _clapListener = null;
+
 async function handleCommand(msg) {
     try {
         const cmd = JSON.parse(msg);
@@ -517,6 +523,10 @@ async function handleCommand(msg) {
                 await moveMouse(cmd.x, cmd.y);
                 return { ok: true };
             case 'mouse_click':
+                if (cmd.x !== undefined && cmd.y !== undefined) {
+                    await moveMouse(cmd.x, cmd.y);
+                    await new Promise(r => setTimeout(r, 30));
+                }
                 await mouseClick(cmd.button || 'left', cmd.double || false);
                 return { ok: true };
             case 'mouse_down':
@@ -579,6 +589,71 @@ async function handleCommand(msg) {
                     return { ok: true };
                 } catch (e) { return { ok: false, error: e.message }; }
             }
+            case 'open_chrome_window': {
+                const { url, w, h, x, y } = cmd;
+                if (!url) return { ok: false, error: 'No URL' };
+                const width = w || 380, height = h || 280;
+                const left = x ?? 16, top = y ?? 200;
+                try {
+                    if (platform === 'darwin') {
+                        const script = `
+                            tell application "Google Chrome"
+                                activate
+                                set newWin to make new window
+                                set URL of active tab of newWin to "${url.replace(/"/g, '\\"')}"
+                                set bounds of newWin to {${left}, ${top}, ${left + width}, ${top + height}}
+                            end tell`;
+                        await execAsync(`osascript -e '${script.replace(/\n/g, ' ')}'`);
+                    } else {
+                        await execAsync(`open -na "Google Chrome" --args --new-window --window-size=${width},${height} --window-position=${left},${top} "${url}"`);
+                    }
+                    return { ok: true };
+                } catch (e) { return { ok: false, error: e.message }; }
+            }
+            case 'resize_chrome_by_url': {
+                const { urlPattern } = cmd;
+                const rw = cmd.w || 300, rh = cmd.h || 300;
+                const rx = cmd.x ?? 0, ry = cmd.y ?? 0;
+                if (!urlPattern) return { ok: false, error: 'No urlPattern' };
+                try {
+                    if (platform === 'darwin') {
+                        const escaped = urlPattern.replace(/"/g, '\\"');
+                        const lines = [
+                            'tell application "Google Chrome"',
+                            `set targetUrl to "${escaped}"`,
+                            'set found to "not_found"',
+                            'repeat with cw in windows',
+                            'repeat with t in tabs of cw',
+                            `if URL of t contains targetUrl then`,
+                            `set bounds of cw to {${rx}, ${ry}, ${rx + rw}, ${ry + rh}}`,
+                            'set found to "done"',
+                            'end if',
+                            'end repeat',
+                            'end repeat',
+                            'end tell',
+                            'found',
+                        ];
+                        const script = lines.join('\n');
+                        const { stdout } = await execAsync(`osascript -e '${script.replace(/'/g, "'\\''")}'`);
+                        const result = stdout.trim();
+                        console.log(chalk.gray(`    resize "${urlPattern}" → ${result} (${rw}x${rh} at ${rx},${ry})`));
+                        return { ok: result === 'done' };
+                    }
+                    return { ok: false, error: 'Not supported on this platform yet' };
+                } catch (e) {
+                    console.log(chalk.red(`    resize error: ${e.message}`));
+                    return { ok: false, error: e.message };
+                }
+            }
+            case 'mute_clap': {
+                const ms = cmd.duration || 10000;
+                if (_clapListener) _clapListener.mute(ms);
+                return { ok: true };
+            }
+            case 'unmute_clap': {
+                if (_clapListener) _clapListener.unmute();
+                return { ok: true };
+            }
             case 'ping':
                 return { ok: true, pong: true };
             default:
@@ -590,18 +665,57 @@ async function handleCommand(msg) {
 }
 
 async function openWebAssistant(assistantId) {
-    const chatUrl = `${WEB_APP_URL}/assistant/live?id=${assistantId}&autostart=1`;
-    const camUrl = `${WEB_APP_URL}/camera-control`;
+    const config = getConfig();
+    const rt = encodeURIComponent(config.get('refreshToken') || '');
+    const rtParam = rt ? `&rt=${rt}` : '';
+    const orbUrl = `${WEB_APP_URL}/assistant/orb?id=${assistantId}${rtParam}`;
+    const chatUrl = `${WEB_APP_URL}/assistant/live?id=${assistantId}&autostart=1${rtParam}`;
+    const camUrl = `${WEB_APP_URL}/camera-control?auto=1${rtParam}`;
+    const screen = await getScreenSize();
+
+    const orbW = 440, orbH = 440;
+    const chatW = 320, chatH = 420;
+    const camW = 400, camH = 340;
+
+    const orbX = 20, orbY = 60;
+    const chatX = screen.width - chatW - 20, chatY = 40;
+    const camX = Math.round((screen.width - camW) / 2), camY = 40;
+
     try {
         if (platform === 'darwin') {
-            await execAsync(`open -na "Google Chrome" --args --new-window --window-size=260,380 --window-position=1100,150 "${chatUrl}" 2>/dev/null || open "${chatUrl}"`);
-            await new Promise(r => setTimeout(r, 800));
-            await execAsync(`open -na "Google Chrome" --args --new-window --window-size=320,260 --window-position=760,200 "${camUrl}" 2>/dev/null || open "${camUrl}"`);
+            console.log(chalk.cyan(`  Opening 3 windows (screen: ${screen.width}x${screen.height})`));
+            console.log(chalk.gray(`    Orb: ${orbW}x${orbH} at (${orbX},${orbY})`));
+            console.log(chalk.gray(`    Chat: ${chatW}x${chatH} at (${chatX},${chatY})`));
+            console.log(chalk.gray(`    Cam: ${camW}x${camH} at (${camX},${camY})`));
+
+            const script = [
+                'tell application "Google Chrome"',
+                'activate',
+                'set orbWin to make new window',
+                `set URL of active tab of orbWin to "${orbUrl.replace(/"/g, '\\"')}"`,
+                `set bounds of orbWin to {${orbX}, ${orbY}, ${orbX + orbW}, ${orbY + orbH}}`,
+                'delay 0.2',
+                'set chatWin to make new window',
+                `set URL of active tab of chatWin to "${chatUrl.replace(/"/g, '\\"')}"`,
+                `set bounds of chatWin to {${chatX}, ${chatY}, ${chatX + chatW}, ${chatY + chatH}}`,
+                'delay 0.2',
+                'set camWin to make new window',
+                `set URL of active tab of camWin to "${camUrl.replace(/"/g, '\\"')}"`,
+                `set bounds of camWin to {${camX}, ${camY}, ${camX + camW}, ${camY + camH}}`,
+                'end tell',
+            ].join('\n');
+
+            await execAsync(`osascript -e '${script.replace(/'/g, "'\\''")}'`, { timeout: 15000 });
+            console.log(chalk.green('  ✓ All 3 windows opened'));
         } else if (platform === 'win32') {
-            await execAsync(`start chrome --new-window --window-size=260,380 --window-position=1100,150 "${chatUrl}" 2>nul || start "" "${chatUrl}"`);
+            await execAsync(`start chrome --new-window --window-size=${orbW},${orbH} --window-position=${orbX},${orbY} "${orbUrl}" 2>nul || start "" "${orbUrl}"`);
+            await new Promise(r => setTimeout(r, 800));
+            await execAsync(`start chrome --new-window --window-size=${chatW},${chatH} --window-position=${chatX},${chatY} "${chatUrl}" 2>nul || start "" "${chatUrl}"`);
             await new Promise(r => setTimeout(r, 800));
-            await execAsync(`start chrome --new-window --window-size=320,260 --window-position=760,200 "${camUrl}" 2>nul || start "" "${camUrl}"`);
+            await execAsync(`start chrome --new-window --window-size=${camW},${camH} --window-position=${camX},${camY} "${camUrl}" 2>nul || start "" "${camUrl}"`);
         } else {
+            await execAsync(`google-chrome --new-window "${orbUrl}" 2>/dev/null || xdg-open "${orbUrl}"`);
+            await new Promise(r => setTimeout(r, 800));
             await execAsync(`google-chrome --new-window "${chatUrl}" 2>/dev/null || xdg-open "${chatUrl}"`);
             await new Promise(r => setTimeout(r, 800));
             await execAsync(`google-chrome --new-window "${camUrl}" 2>/dev/null || xdg-open "${camUrl}"`);
@@ -692,6 +806,8 @@ export async function startAgent(port = 9876, options = {}) {
     }
 
     if (idToken) setIdToken(idToken);
+    const refreshToken = config.get('refreshToken');
+    if (refreshToken) setRefreshToken(refreshToken);
 
     console.log(chalk.gray(`  User: ${email}`));
     console.log(chalk.gray(`  Port: ${port}`));
@@ -708,27 +824,57 @@ export async function startAgent(port = 9876, options = {}) {
     console.log('');
 
     let _assistantOpen = false;
+    let _assistantOpenTimer = null;
     let _clapCooldown = false;
 
+    async function isAssistantWindowOpen() {
+        if (platform !== 'darwin') return false;
+        try {
+            const { stdout } = await execAsync(`osascript -e '
+tell application "Google Chrome"
+repeat with cw in windows
+repeat with t in tabs of cw
+if URL of t contains "assistant/live" or URL of t contains "assistant/orb" then
+return "yes"
+end if
+end repeat
+end repeat
+return "no"
+end tell'`, { timeout: 3000 });
+            return stdout.trim() === 'yes';
+        } catch { return false; }
+    }
+
     if (!options.noMic) {
         await ensureSox();
-        const clapListener = new ClapListener(async () => {
-            if (_assistantOpen) {
+        _clapListener = new ClapListener(async () => {
+            if (_assistantOpen || await isAssistantWindowOpen()) {
                 console.log(chalk.gray('  👏 박수 감지 — 이미 비서가 활성 상태입니다.'));
+                _assistantOpen = true;
                 return;
             }
             if (_clapCooldown) return;
             _clapCooldown = true;
-            setTimeout(() => { _clapCooldown = false; }, 5000);
+            setTimeout(() => { _clapCooldown = false; }, 15000);
 
             console.log(chalk.bold.cyan('  👏 박수 감지! 비서 활성화 중...'));
             _assistantOpen = true;
+            clearTimeout(_assistantOpenTimer);
+            _assistantOpenTimer = setTimeout(() => { _assistantOpen = false; }, 30000);
 
             if (userAssistants.length === 0) {
-                console.log(chalk.gray('  → 기본 Uwonbot 웹 실행'));
+                console.log(chalk.gray('  → 기본 Uwonbot 웹 실행 (작은 창)'));
+                const screen = await getScreenSize();
+                const W = 320, H = 220;
                 try {
                     if (platform === 'darwin') {
-                        await execAsync(`open "${WEB_APP_URL}/assistant"`);
+                        const script = `tell application "Google Chrome" to activate
+                            tell application "Google Chrome"
+                                set newWin to make new window
+                                set URL of active tab of newWin to "${WEB_APP_URL}/assistant"
+                                set bounds of newWin to {20, 40, ${20 + W}, ${40 + H}}
+                            end tell`;
+                        await execAsync(`osascript -e '${script.replace(/\n/g, ' ')}'`);
                     } else if (platform === 'win32') {
                         await execAsync(`start "" "${WEB_APP_URL}/assistant"`);
                     } else {
@@ -742,7 +888,7 @@ export async function startAgent(port = 9876, options = {}) {
             console.log(chalk.green(`  → ${target.name} 웹 실행`));
             await openWebAssistant(target.id);
         });
-        await clapListener.start();
+        await _clapListener.start();
         console.log('');
     }
 
@@ -764,6 +910,7 @@ export async function startAgent(port = 9876, options = {}) {
         ws.on('close', () => {
             console.log(chalk.yellow('  ○ Client disconnected'));
             _assistantOpen = false;
+            clearTimeout(_assistantOpenTimer);
         });
 
         ws.send(JSON.stringify({ type: 'welcome', agent: 'uwonbot', version: '1.1.2', uid }));

package/src/brain.js

@@ -31,7 +31,7 @@ IMPORTANT RULES:
 }
 
 async function callGemini(apiKey, model, messages, systemPrompt) {
-    const modelName = model || 'gemini-2.0-flash';
+    const modelName = model || 'gemini-2.5-flash-lite';
     const tools = getToolsForProvider('gemini');
 
     const contents = messages.map(m => ({

package/src/clapListener.js

@@ -1,6 +1,6 @@
 import chalk from 'chalk';
 
-const DEFAULT_THRESHOLD = 0.4;
+const DEFAULT_THRESHOLD = 0.35;
 const CLAP_MIN_INTERVAL = 100;
 const CLAP_MAX_INTERVAL = 800;
 const REQUIRED_CLAPS = 2;
@@ -17,6 +17,19 @@ export default class ClapListener {
         this.resetTimer = null;
         this.mic = null;
         this.running = false;
+        this._muted = false;
+        this._muteTimer = null;
+    }
+
+    mute(durationMs = 10000) {
+        this._muted = true;
+        clearTimeout(this._muteTimer);
+        this._muteTimer = setTimeout(() => { this._muted = false; }, durationMs);
+    }
+
+    unmute() {
+        this._muted = false;
+        clearTimeout(this._muteTimer);
     }
 
     async start() {
@@ -80,6 +93,8 @@ export default class ClapListener {
     }
 
     _processBuffer(buf) {
+        if (this._muted) return;
+
         let peak = 0;
         for (let i = 0; i < buf.length - 1; i += 2) {
             const sample = buf.readInt16LE(i) / 32768;

package/src/firebase-client.js

@@ -47,10 +47,19 @@ function parseFirestoreDoc(doc) {
 export async function fetchAssistants(uid) {
     if (!cachedIdToken) throw new Error('Not logged in');
     const url = `${FIRESTORE_URL}/users/${uid}/assistants?orderBy=createdAt%20desc`;
-    const res = await fetch(url, {
+    let res = await fetch(url, {
         headers: { 'Authorization': `Bearer ${cachedIdToken}` },
     });
-    const data = await res.json();
+    let data = await res.json();
+    if (data.error && data.error.code === 401 || data.error?.status === 'UNAUTHENTICATED') {
+        const refreshed = await refreshIdToken();
+        if (refreshed) {
+            res = await fetch(url, {
+                headers: { 'Authorization': `Bearer ${cachedIdToken}` },
+            });
+            data = await res.json();
+        }
+    }
     if (data.error) throw new Error(data.error.message);
     return (data.documents || []).map(doc => {
         const parts = doc.name.split('/');
@@ -84,6 +93,27 @@ export async function sendPasswordReset(email) {
 
 export function setIdToken(token) { cachedIdToken = token; }
 
+let cachedRefreshToken = null;
+export function setRefreshToken(token) { cachedRefreshToken = token; }
+
+async function refreshIdToken() {
+    if (!cachedRefreshToken) return false;
+    try {
+        const res = await fetch(`https://securetoken.googleapis.com/v1/token?key=${API_KEY}`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+            body: `grant_type=refresh_token&refresh_token=${encodeURIComponent(cachedRefreshToken)}`,
+        });
+        const data = await res.json();
+        if (data.id_token) {
+            cachedIdToken = data.id_token;
+            if (data.refresh_token) cachedRefreshToken = data.refresh_token;
+            return true;
+        }
+    } catch {}
+    return false;
+}
+
 export async function createCLISession(uid, token) {
     if (!cachedIdToken) throw new Error('Not logged in');
     const url = `${FIRESTORE_URL}/users/${uid}/cliSessions/${token}`;

package/src/voiceInput.js

@@ -38,7 +38,7 @@ function getAmplitude(buf) {
 
 async function transcribeWithGemini(wavBuffer, apiKey) {
     const base64Audio = wavBuffer.toString('base64');
-    const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${encodeURIComponent(apiKey)}`;
+    const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=${encodeURIComponent(apiKey)}`;
 
     const body = {
         contents: [{