shellx-ai 1.1.1 → 1.1.3

package/dist/ui-analyzer.js

@@ -0,0 +1,333 @@
+/**
+ * UI Analyzer - Extract online count and comments from UI hierarchy
+ * Direct implementation using ShellX API
+ */
+/**
+ * Get all UI elements from current screen
+ */
+export async function getAllUIElements(findElement) {
+    // Find all elements by using empty selector with multiple=true
+    const result = await findElement({ text: '' }, // Empty text to match all
+    {
+        multiple: true,
+        maxResults: 1000,
+        visibleOnly: false,
+        clickableOnly: false,
+    });
+    if (!result?.success || !result?.elements) {
+        return [];
+    }
+    return result.elements;
+}
+/**
+ * Extract online count from top-right corner
+ */
+export function extractOnlineCount(elements, screenWidth, screenHeight) {
+    // Define top-right corner area
+    const topRightThresholdX = screenWidth * 0.7; // Right 30%
+    const topRightThresholdY = screenHeight * 0.25; // Top 25%
+    // Number patterns (with optional separators)
+    const numberPatterns = [
+        /^\d+$/, // Pure numbers: "1234"
+        /^\d{1,3}(,\d{3})+$/, // Comma separated: "1,234"
+        /^\d+\.\d+[kKmM]?$/, // With decimals/suffixes: "1.2k", "1.5m"
+        /^\d+[\d,\.]*[kKmM]?$/, // Mixed: "1,234.5k"
+    ];
+    const candidates = [];
+    for (const el of elements) {
+        if (!el.text || !el.visible) {
+            continue;
+        }
+        const trimmedText = el.text.trim();
+        const isNumber = numberPatterns.some(pattern => pattern.test(trimmedText));
+        if (!isNumber) {
+            continue;
+        }
+        const centerX = (el.bounds.left + el.bounds.right) / 2;
+        const centerY = (el.bounds.top + el.bounds.bottom) / 2;
+        // Check if in top-right corner
+        if (centerX > topRightThresholdX && centerY < topRightThresholdY) {
+            // Calculate priority (closer to top-right corner = higher priority)
+            const distanceToCorner = Math.sqrt(Math.pow(screenWidth - centerX, 2) + Math.pow(centerY, 2));
+            // Also prefer smaller text (labels vs large numbers)
+            const textSize = el.bounds.bottom - el.bounds.top;
+            candidates.push({
+                element: el,
+                centerX,
+                centerY,
+                priority: -distanceToCorner - textSize * 0.5,
+            });
+        }
+    }
+    if (candidates.length === 0) {
+        return null;
+    }
+    // Sort by priority and get best match
+    candidates.sort((a, b) => b.priority - a.priority);
+    const best = candidates[0]?.element;
+    if (!best) {
+        return null;
+    }
+    // Parse the count
+    let count = 0;
+    const text = best.text.trim();
+    // Handle different formats
+    if (text.includes('k') || text.includes('K')) {
+        count = Math.floor(parseFloat(text.replace(/[kK,]/g, '')) * 1000);
+    }
+    else if (text.includes('m') || text.includes('M')) {
+        count = Math.floor(parseFloat(text.replace(/[mM,]/g, '')) * 1000000);
+    }
+    else {
+        count = parseInt(text.replace(/,/g, ''), 10);
+    }
+    return {
+        count: isNaN(count) ? 0 : count,
+        text,
+        bounds: {
+            left: best.bounds.left,
+            top: best.bounds.top,
+            right: best.bounds.right,
+            bottom: best.bounds.bottom,
+        },
+        confidence: candidates.length === 1 ? 'high' : 'medium',
+    };
+}
+/**
+ * Extract comment information from elements
+ * Looks for text elements that are likely comments based on position and content
+ */
+export function extractComments(elements, maxComments = 20) {
+    const comments = [];
+    const seen = new Set();
+    // Get screen bounds to identify comment sections
+    // Comments are typically in the middle/lower part of the screen
+    let minY = Infinity;
+    let maxY = -Infinity;
+    for (const el of elements) {
+        if (el.text && el.text.trim().length > 0) {
+            minY = Math.min(minY, el.bounds.top);
+            maxY = Math.max(maxY, el.bounds.bottom);
+        }
+    }
+    const screenHeight = maxY - minY;
+    const commentAreaTop = minY + screenHeight * 0.3; // Comments start below top 30%
+    const commentAreaBottom = maxY - screenHeight * 0.1; // Comments end before bottom 10%
+    // Filter elements that look like comments
+    for (const el of elements) {
+        if (!el.text || el.text.trim().length === 0) {
+            continue;
+        }
+        const trimmedText = el.text.trim();
+        const centerY = (el.bounds.top + el.bounds.bottom) / 2;
+        // Skip if not in comment area
+        if (centerY < commentAreaTop || centerY > commentAreaBottom) {
+            continue;
+        }
+        // Skip very short texts (likely buttons/labels)
+        if (trimmedText.length < 3) {
+            continue;
+        }
+        // Skip elements that look like UI controls
+        if (el.clickable) {
+            continue;
+        }
+        // Skip common button/label patterns
+        const skipPatterns = [
+            /^(发送|取消|确定|关闭|返回|更多|展开|收起|关注|私信|分享)$/i,
+            /^(Send|Cancel|OK|Close|Back|More|Expand|Collapse|Follow|Message|Share)$/i,
+            /^\d+$/, // Pure numbers
+            /^\d{1,2}:\d{2}$/, // Time patterns like "14:15"
+            /^店铺销量/, // Shop sales info
+            /^保修期/, // Warranty info
+            /^(维度|内容|型号)$/, // Product specs
+            /^\d+年$/, // "1年", "2年" etc
+        ];
+        if (skipPatterns.some(pattern => pattern.test(trimmedText))) {
+            continue;
+        }
+        // Skip single character or very short labels
+        if (trimmedText.length <= 3 && !containsChinese(trimmedText)) {
+            continue;
+        }
+        // Skip store/shop names (usually short and contain "店" or "旗舰店")
+        if (trimmedText.includes('旗舰店') || trimmedText.includes('专卖店') || trimmedText.includes('店')) {
+            // Only skip if it's short (likely a shop name, not a comment mentioning a shop)
+            if (trimmedText.length < 10) {
+                continue;
+            }
+        }
+        // Deduplicate by text content
+        const key = trimmedText.toLowerCase();
+        if (seen.has(key)) {
+            continue;
+        }
+        seen.add(key);
+        comments.push({
+            text: trimmedText,
+            bounds: {
+                left: el.bounds.left,
+                top: el.bounds.top,
+                right: el.bounds.right,
+                bottom: el.bounds.bottom,
+            },
+            className: el.className,
+        });
+        if (comments.length >= maxComments) {
+            break;
+        }
+    }
+    // Sort by vertical position (top to bottom)
+    comments.sort((a, b) => a.bounds.top - b.bounds.top);
+    return comments;
+}
+/**
+ * Check if text contains Chinese characters
+ */
+function containsChinese(text) {
+    return /[一-龥]/.test(text);
+}
+/**
+ * Analyze screen and provide summary
+ */
+export function analyzeScreen(elements, screenWidth, screenHeight) {
+    const elementsByClass = {};
+    let elementsWithText = 0;
+    let clickableElements = 0;
+    for (const el of elements) {
+        // Count by class
+        elementsByClass[el.className] = (elementsByClass[el.className] || 0) + 1;
+        // Count elements with text
+        if (el.text && el.text.trim().length > 0) {
+            elementsWithText++;
+        }
+        // Count clickable elements
+        if (el.clickable) {
+            clickableElements++;
+        }
+    }
+    return {
+        screenWidth,
+        screenHeight,
+        totalElements: elements.length,
+        elementsWithText,
+        clickableElements,
+        elementsByClass,
+    };
+}
+/**
+ * Format screen analysis as readable text
+ */
+export function formatScreenAnalysis(analysis) {
+    const lines = [
+        '=== 屏幕分析 ===',
+        `屏幕尺寸: ${analysis.screenWidth}x${analysis.screenHeight}`,
+        `总元素数: ${analysis.totalElements}`,
+        `包含文本的元素: ${analysis.elementsWithText}`,
+        `可点击元素: ${analysis.clickableElements}`,
+        '',
+        '=== 元素类别统计 ===',
+    ];
+    // Sort by count and show top 15
+    const sortedClasses = Object.entries(analysis.elementsByClass)
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, 15);
+    for (const [className, count] of sortedClasses) {
+        lines.push(`  ${className}: ${count}`);
+    }
+    return lines.join('\n');
+}
+/**
+ * Format online count result as readable text
+ */
+export function formatOnlineCount(info) {
+    if (!info) {
+        return '=== 在线人数 ===\n未找到在线人数';
+    }
+    return [
+        '=== 在线人数 ===',
+        `数量: ${info.count}`,
+        `原文: "${info.text}"`,
+        `位置: [${info.bounds.left}, ${info.bounds.top} → ${info.bounds.right}, ${info.bounds.bottom}]`,
+        `置信度: ${info.confidence === 'high' ? '高' : info.confidence === 'medium' ? '中' : '低'}`,
+    ].join('\n');
+}
+/**
+ * Format comments as readable text
+ */
+export function formatComments(comments) {
+    if (comments.length === 0) {
+        return '=== 评论信息 ===\n未找到评论';
+    }
+    const lines = [
+        '=== 评论信息 ===',
+        `找到 ${comments.length} 条评论`,
+        '',
+    ];
+    for (let i = 0; i < comments.length; i++) {
+        const comment = comments[i];
+        if (!comment)
+            continue;
+        lines.push(`${i + 1}. ${comment.text}`);
+        lines.push(`   位置: [${comment.bounds.left}, ${comment.bounds.top} → ${comment.bounds.right}, ${comment.bounds.bottom}]`);
+        lines.push(`   类型: ${comment.className}`);
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+/**
+ * Extract all meaningful text content from elements
+ * Returns all text elements that might be useful (comments, descriptions, etc.)
+ */
+export function extractAllTextContent(elements, maxLength = 50) {
+    const results = [];
+    for (const el of elements) {
+        if (!el.text || !el.visible) {
+            continue;
+        }
+        const trimmedText = el.text.trim();
+        // Skip very short or empty text
+        if (trimmedText.length < 2) {
+            continue;
+        }
+        // Skip single characters that are likely UI elements
+        if (trimmedText.length === 1 && !containsChinese(trimmedText)) {
+            continue;
+        }
+        results.push({
+            text: trimmedText.length > maxLength ? trimmedText.substring(0, maxLength) + '...' : trimmedText,
+            className: el.className,
+            bounds: {
+                left: el.bounds.left,
+                top: el.bounds.top,
+                right: el.bounds.right,
+                bottom: el.bounds.bottom,
+            },
+        });
+    }
+    // Sort by vertical position
+    results.sort((a, b) => a.bounds.top - b.bounds.top);
+    return results;
+}
+/**
+ * Format all text content as readable text
+ */
+export function formatAllTextContent(contents) {
+    if (contents.length === 0) {
+        return '=== 所有文本内容 ===\n无文本内容';
+    }
+    const lines = [
+        '=== 所有文本内容 ===',
+        `共 ${contents.length} 条`,
+        '',
+    ];
+    for (let i = 0; i < contents.length; i++) {
+        const item = contents[i];
+        if (!item)
+            continue;
+        lines.push(`${i + 1}. ${item.text}`);
+        lines.push(`   [${item.bounds.left}, ${item.bounds.top} → ${item.bounds.right}, ${item.bounds.bottom}] ${item.className}`);
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=ui-analyzer.js.map

package/dist/ui-analyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ui-analyzer.js","sourceRoot":"","sources":["../src/ui-analyzer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA0BH;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,WAAmE;IAEnE,+DAA+D;IAC/D,MAAM,MAAM,GAAG,MAAM,WAAW,CAC9B,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,0BAA0B;IACxC;QACE,QAAQ,EAAE,IAAI;QACd,UAAU,EAAE,IAAI;QAChB,WAAW,EAAE,KAAK;QAClB,aAAa,EAAE,KAAK;KACrB,CACF,CAAC;IAEF,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC1C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,MAAM,CAAC,QAAQ,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,QAAqB,EACrB,WAAmB,EACnB,YAAoB;IAEpB,+BAA+B;IAC/B,MAAM,kBAAkB,GAAG,WAAW,GAAG,GAAG,CAAC,CAAE,YAAY;IAC3D,MAAM,kBAAkB,GAAG,YAAY,GAAG,IAAI,CAAC,CAAC,UAAU;IAE1D,6CAA6C;IAC7C,MAAM,cAAc,GAAG;QACrB,OAAO,EAAqB,uBAAuB;QACnD,oBAAoB,EAAO,2BAA2B;QACtD,mBAAmB,EAAQ,yCAAyC;QACpE,sBAAsB,EAAK,oBAAoB;KAChD,CAAC;IAEF,MAAM,UAAU,GAKX,EAAE,CAAC;IAER,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QAE3E,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAEvD,+BAA+B;QAC/B,IAAI,OAAO,GAAG,kBAAkB,IAAI,OAAO,GAAG,kBAAkB,EAAE,CAAC;YACjE,oEAAoE;YACpE,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAChC,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,OAAO,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAC1D,CAAC;YAEF,qDAAqD;YACrD,MAAM,QAAQ,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAElD,UAAU,CAAC,IAAI,CAAC;gBACd,OAAO,EAAE,EAAE;gBACX,OAAO;gBACP,OAAO;gBACP,QAAQ,EAAE,CAAC,gBAAgB,GAAG,QAAQ,GAAG,GAAG;aAC7C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sCAAsC;IACtC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC;IACpC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,IAAI,CAAC;IACd,CAAC;IAED,kBAAkB;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAE9B,2BAA2B;IAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7C,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IACpE,CAAC;SAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;IACvE,CAAC;SAAM,CAAC;QACN,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;QAC/B,IAAI;QACJ,MAAM,EAAE;YACN,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;YACtB,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;YACxB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;SAC3B;QACD,UAAU,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;KACxD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAqB,EACrB,cAAsB,EAAE;IAExB,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,iDAAiD;IACjD,gEAAgE;IAChE,IAAI,IAAI,GAAG,QAAQ,CAAC;IACpB,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC;IAErB,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC;IACjC,MAAM,cAAc,GAAG,IAAI,GAAG,YAAY,GAAG,GAAG,CAAC,CAAC,+BAA+B;IACjF,MAAM,iBAAiB,GAAG,IAAI,GAAG,YAAY,GAAG,GAAG,CAAC,CAAC,iCAAiC;IAEtF,0CAA0C;IAC1C,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5C,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAEvD,8BAA8B;QAC9B,IAAI,OAAO,GAAG,cAAc,IAAI,OAAO,GAAG,iBAAiB,EAAE,CAAC;YAC5D,SAAS;QACX,CAAC;QAED,gDAAgD;QAChD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,2CAA2C;QAC3C,IAAI,EAAE,CAAC,SAAS,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QAED,oCAAoC;QACpC,MAAM,YAAY,GAAG;YACnB,uCAAuC;YACvC,0EAA0E;YAC1E,OAAO,EAAE,eAAe;YACxB,iBAAiB,EAAE,6BAA6B;YAChD,OAAO,EAAE,kBAAkB;YAC3B,MAAM,EAAE,gBAAgB;YACxB,cAAc,EAAE,gBAAgB;YAChC,QAAQ,EAAE,iBAAiB;SAC5B,CAAC;QAEF,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;YAC5D,SAAS;QACX,CAAC;QAED,6CAA6C;QAC7C,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7D,SAAS;QACX,CAAC;QAED,iEAAiE;QACjE,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5F,gFAAgF;YAChF,IAAI,WAAW,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBAC5B,SAAS;YACX,CAAC;QACH,CAAC;QAED,8BAA8B;QAC9B,MAAM,GAAG,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEd,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE;gBACN,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI;gBACpB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;gBAClB,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK;gBACtB,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM;aACzB;YACD,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;YACnC,MAAM;QACR,CAAC;IACH,CAAC;IAED,4CAA4C;IAC5C,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAErD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,QAAqB,EACrB,WAAmB,EACnB,YAAoB;IAEpB,MAAM,eAAe,GAA2B,EAAE,CAAC;IACnD,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,iBAAiB;QACjB,eAAe,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAEzE,2BAA2B;QAC3B,IAAI,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,gBAAgB,EAAE,CAAC;QACrB,CAAC;QAED,2BAA2B;QAC3B,IAAI,EAAE,CAAC,SAAS,EAAE,CAAC;YACjB,iBAAiB,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO;QACL,WAAW;QACX,YAAY;QACZ,aAAa,EAAE,QAAQ,CAAC,MAAM;QAC9B,gBAAgB;QAChB,iBAAiB;QACjB,eAAe;KAChB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAwB;IAC3D,MAAM,KAAK,GAAa;QACtB,cAAc;QACd,SAAS,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,YAAY,EAAE;QACxD,SAAS,QAAQ,CAAC,aAAa,EAAE;QACjC,YAAY,QAAQ,CAAC,gBAAgB,EAAE;QACvC,UAAU,QAAQ,CAAC,iBAAiB,EAAE;QACtC,EAAE;QACF,gBAAgB;KACjB,CAAC;IAEF,gCAAgC;IAChC,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;SAC3D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEhB,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;QAC/C,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS,KAAK,KAAK,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAA4B;IAC5D,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED,OAAO;QACL,cAAc;QACd,OAAO,IAAI,CAAC,KAAK,EAAE;QACnB,QAAQ,IAAI,CAAC,IAAI,GAAG;QACpB,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG;QAC7F,QAAQ,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE;KACtF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,QAAuB;IACpD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,qBAAqB,CAAC;IAC/B,CAAC;IAED,MAAM,KAAK,GAAa;QACtB,cAAc;QACd,MAAM,QAAQ,CAAC,MAAM,MAAM;QAC3B,EAAE;KACH,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACzH,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,QAAqB,EACrB,YAAoB,EAAE;IAEtB,MAAM,OAAO,GAIR,EAAE,CAAC;IAER,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnC,gCAAgC;QAChC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,qDAAqD;QACrD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9D,SAAS;QACX,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,WAAW,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,WAAW;YAChG,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,MAAM,EAAE;gBACN,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI;gBACpB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;gBAClB,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK;gBACtB,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM;aACzB;SACF,CAAC,CAAC;IACL,CAAC;IAED,4BAA4B;IAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAEpD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAInC;IACA,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED,MAAM,KAAK,GAAa;QACtB,gBAAgB;QAChB,KAAK,QAAQ,CAAC,MAAM,IAAI;QACxB,EAAE;KACH,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAC7H,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/package.json

@@ -1,10 +1,13 @@
 {
   "name": "shellx-ai",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "type": "module",
   "description": "A powerful TypeScript library for Android device automation and UI control via WebSocket",
   "workspaces": [
-    "example"
+    "example",
+    "phone-agent",
+    "shellx-cli",
+    "gui-agent"
   ],
   "keywords": [
     "android",
@@ -44,8 +47,11 @@
   ],
   "scripts": {
     "build": "tsc",
+    "build:gui-agent": "cd gui-agent && npm run build:all",
     "build:phone-agent": "cd phone-agent && npm run build:all",
-    "build:all": "npm run build && npm run build:phone-agent",
+    "build:shellx-cli": "cd shellx-cli && npm run build",
+    "build:example": "cd example && npm run build",
+    "build:all": "npm run build && npm run build:gui-agent && npm run build:phone-agent && npm run build:shellx-cli && npm run build:example",
     "build:watch": "tsc --watch",
     "type-check": "tsc --noEmit",
     "lint": "eslint src --ext .ts",
@@ -55,11 +61,13 @@
     "test": "jest",
     "test:coverage": "jest --coverage",
     "test:watch": "jest --watch",
-    "clean": "rimraf dist && cd phone-agent && rimraf dist",
+    "clean": "rimraf dist && cd gui-agent && rimraf dist && cd ../phone-agent && rimraf dist && cd ../shellx-cli && rimraf dist && cd ../example && rimraf dist",
     "prepublishOnly": "npm run clean && npm run build && npm run type-check",
     "prepare": "husky install || echo 'Husky not available'"
   },
   "dependencies": {
+    "@mariozechner/pi-agent-core": "^0.65.2",
+    "@mariozechner/pi-ai": "^0.65.2",
     "cbor": "^9.0.0",
     "dotenv": "^16.4.5",
     "ofetch": "^1.4.1",
@@ -72,18 +80,31 @@
     "ws": "^8.14.2"
   },
   "devDependencies": {
+    "@eslint/plugin-kit": "^0.4.1",
     "@types/jest": "^30.0.0",
     "@types/node": "^24.0.13",
+    "@types/react": "^19.2.11",
+    "@types/uuid": "^9.0.7",
     "@types/ws": "^8.18.1",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
+    "autocomplete-cli": "^0.0.2",
+    "chalk": "^5.6.2",
+    "esbuild": "^0.27.4",
+    "esbuild-node-externals": "^1.20.1",
     "eslint": "^9.0.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.2.0",
+    "ink": "^6.6.0",
+    "ink-spinner": "^5.0.0",
+    "ink-text-input": "^6.0.0",
     "jest": "^30.0.4",
     "prettier": "^3.3.0",
+    "prettier-linter-helpers": "^1.0.1",
+    "react": "^19.2.4",
     "rimraf": "^6.0.0",
     "ts-jest": "^29.4.0",
+    "tsx": "^4.19.2",
     "typescript": "^5.8.3"
   },
   "engines": {

package/dist/shell/output-buffer.d.ts

@@ -1,152 +0,0 @@
-/**
- * OutputBuffer - A module for managing shell command output buffering
- *
- * This module handles buffering, processing, and managing output from shell commands
- * across multiple sessions, with support for output cleaning and combination.
- */
-/**
- * Options for shell command execution
- */
-export interface ShellCommandOptions {
-    /** Title for the command (used for logging) */
-    title?: string;
-    /** Timeout in milliseconds */
-    timeout?: number;
-    /** Wait time after command execution in milliseconds */
-    waitAfterMs?: number;
-    /** Callback for handling output chunks as they arrive */
-    onOutput?: (output: string) => void;
-    /** Callback for handling error messages */
-    onError?: (error: string) => void;
-    /** Expected output pattern (string or RegExp) */
-    expectedOutput?: string | RegExp;
-    /** Success pattern to match in output (string or RegExp) */
-    successPattern?: string | RegExp;
-    /** Whether to allow returning partial results on timeout */
-    allowPartialResult?: boolean;
-    /** Whether to automatically set Android PATH (default: true) */
-    setAndroidPath?: boolean;
-}
-/**
- * Result of a shell command execution
- */
-export interface ShellCommandResult {
-    /** Whether the command executed successfully */
-    success: boolean;
-    /** The command output */
-    output: string;
-    /** Error message if command failed */
-    error?: string;
-    /** Exit code of the command */
-    exitCode?: number;
-    /** Duration of command execution in milliseconds */
-    duration: number;
-}
-/**
- * Internal promise representation for a pending shell command
- */
-interface CommandPromise {
-    /** Resolve function for the promise */
-    resolve: (result: ShellCommandResult) => void;
-    /** Reject function for the promise */
-    reject: (error: Error) => void;
-    /** Start time of the command */
-    startTime: number;
-    /** Execution options */
-    options: ShellCommandOptions;
-    /** Accumulated output */
-    output: string;
-    /** Session-specific outputs */
-    sessionOutputs: Map<number, string>;
-    /** Original command string */
-    command: string;
-}
-/**
- * OutputBuffer class manages shell command output buffering and processing
- *
- * This class handles:
- * - Buffering output from multiple shell command sessions
- * - Managing pending command promises
- * - Cleaning and combining output from different sessions
- * - Providing access to accumulated command results
- */
-export declare class OutputBuffer {
-    /** Map of pending command promises keyed by command ID */
-    private commandPromises;
-    /**
-     * Register a new command promise for tracking
-     *
-     * @param commandId - Unique identifier for the command
-     * @param resolve - Promise resolve function
-     * @param reject - Promise reject function
-     * @param startTime - Command start timestamp
-     * @param options - Command execution options
-     * @param command - Original command string
-     */
-    registerCommand(commandId: string, resolve: (result: ShellCommandResult) => void, reject: (error: Error) => void, startTime: number, options: ShellCommandOptions, command: string): void;
-    /**
-     * Unregister and resolve a command promise
-     *
-     * @param commandId - Unique identifier for the command
-     * @returns The command promise data if found, undefined otherwise
-     */
-    unregisterCommand(commandId: string): CommandPromise | undefined;
-    /**
-     * Get a registered command promise
-     *
-     * @param commandId - Unique identifier for the command
-     * @returns The command promise data if found, undefined otherwise
-     */
-    getCommand(commandId: string): CommandPromise | undefined;
-    /**
-     * Check if a command is registered
-     *
-     * @param commandId - Unique identifier for the command
-     * @returns True if the command is registered, false otherwise
-     */
-    hasCommand(commandId: string): boolean;
-    /**
-     * Get the number of pending commands
-     *
-     * @returns The number of currently registered commands
-     */
-    getPendingCount(): number;
-    /**
-     * Handle incoming shell output chunks
-     *
-     * This method processes output from shell sessions and distributes it
-     * to all waiting commands.
-     *
-     * @param sessionId - The session ID for the output
-     * @param chunks - Array of Uint8Array data chunks
-     */
-    handleShellOutput(sessionId: number, chunks: Uint8Array[]): void;
-    /**
-     * Clean command output by removing the command itself
-     *
-     * @param output - The raw output string
-     * @param command - The command string to remove
-     * @returns Cleaned output string
-     */
-    cleanCommandOutput(output: string, command: string): string;
-    /**
-     * Combine outputs from multiple sessions into a single string
-     *
-     * @param sessionOutputs - Map of session IDs to their outputs
-     * @returns Combined output string
-     */
-    private combineSessionOutputs;
-    /**
-     * Clear all registered commands
-     *
-     * This method is useful for cleanup or reset scenarios
-     */
-    clear(): void;
-    /**
-     * Get all registered command IDs
-     *
-     * @returns Array of command IDs
-     */
-    getCommandIds(): string[];
-}
-export {};