shellx-ai 1.0.8 → 1.0.10

package/dist/index.d.ts

@@ -30,8 +30,9 @@ export interface PendingTask {
 /**
  * Enhanced WebSocket Task Client with protocol-aware task methods
  */
-export declare class WebSocketTaskClient {
+export declare class ConnectionTaskClient {
     private wsUrl;
+    private deviceId;
     private config;
     ws: any;
     private shellxConnected;
@@ -43,8 +44,10 @@ export declare class WebSocketTaskClient {
     private pingIntervalId;
     private initializationPromise;
     private shellx;
-    constructor(wsUrl: string, config?: Partial<TaskClientConfig>);
+    constructor(deviceId: string, config?: Partial<TaskClientConfig>);
     private init;
+    private authenticateDevice;
+    getFetch(): (url: string, options?: any) => Promise<any>;
     /**
      * 等待WebSocket初始化完成
      */
@@ -56,7 +59,7 @@ export declare class WebSocketTaskClient {
     private handleMessage;
     private processServerMessage;
     private handleJsonDataResponse;
-    private sendMessage;
+    sendMessage(message: WsClient, taskType?: string, commandType?: string): Promise<any>;
     /**
      * Find UI elements on the screen
      */
@@ -175,6 +178,7 @@ export declare class WebSocketTaskClient {
      */
     completeTasksByType(taskType: string, result?: any, success?: boolean): number;
 }
-export default WebSocketTaskClient;
+export default ConnectionTaskClient;
 export type { UIElement, ElementSelector, ActionSequence, WsClient, WsServer, ScreenShotOptions, FindOptions, WaitOptions, UIHierarchy, WAITElement, ScreenShotResponse, ScreenInfoResponse, AppListResponse } from './protocol';
+export type { Click, Input, Swipe, Key, Wait, Find, Command, AppInfo, Screenshot, ActionResult, Element, FindResult, CommandResult, Action, Actions, } from './types';
 export { createShellXWithShellMonitoring, ShellX } from './shellx';

package/dist/index.js

@@ -42,7 +42,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ShellX = exports.createShellXWithShellMonitoring = exports.WebSocketTaskClient = exports.DEFAULT_CONFIG = void 0;
+exports.ShellX = exports.createShellXWithShellMonitoring = exports.ConnectionTaskClient = exports.DEFAULT_CONFIG = void 0;
 const uuid_1 = require("uuid");
 const utils_1 = require("./utils");
 // 定义默认配置
@@ -84,8 +84,8 @@ function getWebSocket() {
 /**
  * Enhanced WebSocket Task Client with protocol-aware task methods
  */
-class WebSocketTaskClient {
-    constructor(wsUrl, config = {}) {
+class ConnectionTaskClient {
+    constructor(deviceId, config = {}) {
         this.ws = null; // Use any to avoid WebSocket type issues
         this.shellxConnected = false; // ShellX.ai 连接状态
         this.wsConnected = false; // WebSocket 连接状态
@@ -96,7 +96,7 @@ class WebSocketTaskClient {
         this.pingIntervalId = null;
         this.initializationPromise = null;
         this.shellx = null; // 关联的 ShellX 实例
-        this.wsUrl = wsUrl;
+        this.deviceId = deviceId;
         this.config = Object.assign(Object.assign({}, exports.DEFAULT_CONFIG), config);
         this.initializationPromise = this.init();
     }
@@ -104,6 +104,7 @@ class WebSocketTaskClient {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b;
             try {
+                this.wsUrl = yield this.authenticateDevice(this.deviceId);
                 console.log('Initializing WebSocket client...' + this.wsUrl);
                 // 获取适合当前环境的WebSocket构造函数
                 const WebSocketConstructor = yield getWebSocket();
@@ -124,7 +125,10 @@ class WebSocketTaskClient {
                 };
                 this.ws.onclose = (event) => {
                     var _a, _b;
-                    console.log('❌ [ShellX] WebSocket连接已关闭，正在尝试重新连接...' + event.code + ' ' + this.wsUrl);
+                    console.log('❌ [ShellX] WebSocket连接已关闭，正在尝试重新连接...' +
+                        event.code +
+                        ' ' +
+                        this.wsUrl);
                     this.wsConnected = false;
                     this.shellxConnected = false;
                     this.authenticated = false;
@@ -144,6 +148,89 @@ class WebSocketTaskClient {
             }
         });
     }
+    authenticateDevice(deviceId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // 1. 优先检测本地服务
+            try {
+                // fetch超时实现
+                const fetchWithTimeout = (url, options, timeout = 1000) => Promise.race([
+                    fetch(url, options),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), timeout)),
+                ]);
+                const localResp = (yield fetchWithTimeout('http://127.0.0.1:9091/info', { method: 'GET', headers: { Accept: 'application/json' } }, 1000));
+                if (localResp.ok) {
+                    const info = yield localResp.json();
+                    if (info && (info.status === 'ok' || info.status === 1) && info.uuid) {
+                        if (deviceId == info.uuid) {
+                            const localUuid = info.uuid;
+                            const fallbackUrl = `ws://127.0.0.1:9091/api/s/${localUuid}`;
+                            console.log('✅ [Auth] 本地ShellX服务可用，使用本地 /info 返回的 uuid:', localUuid, '，服务地址:', fallbackUrl);
+                            return fallbackUrl;
+                        }
+                    }
+                }
+            }
+            catch (e) {
+                // 本地不可用，继续走远程
+            }
+            const fallbackUrl = `ws://127.0.0.1:9091/api/s/${deviceId}`;
+            // 2. 远程认证逻辑
+            authKey = (0, utils_1.getEnvVar)('SHELLX_AUTH_KEY');
+            if (!authKey) {
+                throw new Error('SHELLX_AUTH_KEY environment variable is required');
+            }
+            try {
+                console.log('🔑 [Auth] 正在认证设备...');
+                const featGlobal = this.getFetch();
+                const jsonData = yield featGlobal(`https://shellx.ai/api/device/${deviceId}`, {
+                    method: 'GET',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                });
+                console.log('ShellX.ai设备认证响应:', jsonData);
+                // const jsonData = JSON.parse(data);
+                console.log('✅ [Auth] ShellX.ai设备认证成功');
+                console.log(`📡 [Auth] 设备ID: ${jsonData.authenticate}`);
+                console.log(`📡 [Auth] ShellX.ai服务地址: ${jsonData.machine}`);
+                console.log(`📡 [Auth] 注册时间: ${jsonData.registered_at}`);
+                console.log(`📡 [Auth] 最后更新: ${jsonData.last_updated}`);
+                return jsonData.machine + '/api/s/' + jsonData.authenticate;
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                console.warn('⚠️ [Auth] ShellX.ai在线认证失败，使用本地服务:', JSON.stringify(errorMessage));
+                return fallbackUrl;
+            }
+        });
+    }
+    getFetch() {
+        return (url, options) => __awaiter(this, void 0, void 0, function* () {
+            // 尝试使用全局fetch
+            if (typeof globalThis.fetch !== 'undefined') {
+                console.log(`🌐 [Fetch] 请求: ${(options === null || options === void 0 ? void 0 : options.method) || 'GET'} ${url}`);
+                const response = yield globalThis.fetch(url, options);
+                console.log(`📡 [Fetch] 响应: ${response.status} ${response.statusText}`);
+                if (!response.ok) {
+                    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                }
+                return response.json();
+            }
+            try {
+                const { default: fetch } = yield Promise.resolve().then(() => __importStar(require('node-fetch')));
+                console.log(`🌐 [Fetch] 请求: ${(options === null || options === void 0 ? void 0 : options.method) || 'GET'} ${url}`);
+                const response = yield fetch(url, options);
+                console.log(`📡 [Fetch] 响应: ${response.status} ${response.statusText}`);
+                if (!response.ok) {
+                    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                }
+                return response.json();
+            }
+            catch (fetchError) {
+                throw new Error(`Fetch not available: ${fetchError.message}`);
+            }
+        });
+    }
     /**
      * 等待WebSocket初始化完成
      */
@@ -308,36 +395,45 @@ class WebSocketTaskClient {
         }
     }
     sendMessage(message, taskType, commandType) {
-        return new Promise((resolve, reject) => {
-            const taskId = (0, uuid_1.v4)();
-            if (taskType) {
-                const timer = setTimeout(() => {
-                    this.pendingTasks.delete(taskId);
-                    reject(new Error(`Task ${taskType} timeout (${this.config.timeout}ms)`));
-                }, this.config.timeout);
-                this.pendingTasks.set(taskId, { resolve, reject, timer, type: taskType, commandType });
-                console.log(`📋 [ShellX] 创建任务: ${taskId}, 类型: ${taskType}}`);
-            }
-            if (this.shellxConnected && this.ws) {
-                try {
-                    console.log('📤 [ShellX] 发送消息:', message);
-                    this.ws.send((0, cbor_1.encode)(message));
-                    if (!taskType)
-                        resolve(undefined); // For fire-and-forget messages
-                }
-                catch (error) {
-                    if (taskType) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return new Promise((resolve, reject) => {
+                const taskId = (0, uuid_1.v4)();
+                if (taskType) {
+                    const timer = setTimeout(() => {
                         this.pendingTasks.delete(taskId);
+                        reject(new Error(`Task ${taskType} timeout (${this.config.timeout}ms)`));
+                    }, this.config.timeout);
+                    this.pendingTasks.set(taskId, {
+                        resolve,
+                        reject,
+                        timer,
+                        type: taskType,
+                        commandType,
+                    });
+                    console.log(`📋 [ShellX] 创建任务: ${taskId}, 类型: ${taskType}}`);
+                }
+                if (this.shellxConnected && this.ws) {
+                    try {
+                        console.log('📤 [ShellX] 发送消息:', JSON.stringify(message));
+                        this.ws.send((0, cbor_1.encode)(message));
+                        if (!taskType)
+                            resolve(undefined); // For fire-and-forget messages
+                    }
+                    catch (error) {
+                        if (taskType) {
+                            this.pendingTasks.delete(taskId);
+                        }
+                        reject(error);
                     }
-                    reject(error);
                 }
-            }
-            else {
-                console.log('⏳ [ShellX] 连接未就绪，消息已加入队列');
-                this.messageQueue.push(message);
-                if (!taskType)
-                    resolve(undefined);
-            }
+                else {
+                    console.log('⏳ [ShellX] 连接未就绪，消息已加入队列');
+                    this.reconnect();
+                    this.messageQueue.push(message);
+                    if (!taskType)
+                        resolve(undefined);
+                }
+            });
         });
     }
     // ===== PROTOCOL-AWARE TASK METHODS =====
@@ -349,7 +445,7 @@ class WebSocketTaskClient {
             const findAction = {
                 type: 'find',
                 selector,
-                options
+                options,
             };
             return this.sendMessage({ findElement: findAction }, 'findElement');
         });
@@ -362,7 +458,7 @@ class WebSocketTaskClient {
             const waitAction = {
                 type: 'wait',
                 selector,
-                options
+                options,
             };
             return this.sendMessage({ waitElement: waitAction }, 'waitElement');
         });
@@ -372,17 +468,18 @@ class WebSocketTaskClient {
      */
     screenShot(options) {
         return __awaiter(this, void 0, void 0, function* () {
-            return this.sendMessage({ screenShot: options || {
-                    "format": "png",
-                    "quality": 30,
-                    "scale": 1.0,
-                    "region": {
-                        "left": 0,
-                        "top": 0,
-                        "width": 1080,
-                        "height": 2340
-                    }
-                }
+            return this.sendMessage({
+                screenShot: options || {
+                    format: 'png',
+                    quality: 30,
+                    scale: 1.0,
+                    region: {
+                        left: 0,
+                        top: 0,
+                        width: 1080,
+                        height: 2340,
+                    },
+                },
             }, 'screenShot');
         });
     }
@@ -432,7 +529,7 @@ class WebSocketTaskClient {
     screenChange(options) {
         return __awaiter(this, void 0, void 0, function* () {
             return this.sendMessage({
-                screenChange: options || { enable: true }
+                screenChange: options || { enable: true },
             });
         });
     }
@@ -499,13 +596,13 @@ class WebSocketTaskClient {
                     resolve: () => { },
                     reject: () => { },
                     timer,
-                    type: taskType
+                    type: taskType,
                 });
                 console.log(`📋 [ShellX] 创建可手动控制的任务: ${taskId}, 类型: ${taskType}}`);
             }
             if (this.shellxConnected && this.ws) {
                 try {
-                    console.log('📤 [ShellX] 发送消息:', message);
+                    console.log(Date.now() + ' 📤 [ShellX] 发送消息:', JSON.stringify(message));
                     this.ws.send((0, cbor_1.encode)(message));
                     const promise = new Promise((promiseResolve, promiseReject) => {
                         if (taskType) {
@@ -543,7 +640,7 @@ class WebSocketTaskClient {
         console.log('🏓 [ShellX] 开始心跳检测...');
         this.pingIntervalId = setInterval(() => {
             if (this.ws && this.shellxConnected) {
-                this.ws.send((0, cbor_1.encode)({ ping: BigInt(Date.now()) }));
+                this.ws.send((0, cbor_1.encode)({ ping: Date.now() }));
             }
         }, this.config.pingInterval);
     }
@@ -557,8 +654,10 @@ class WebSocketTaskClient {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b;
             // TODO: 本地的需要重连
-            console.log(`🔄 [ShellX] 重连中... (${this.reconnectAttempts}/${this.config.reconnectMaxAttempts})` + this.wsUrl);
+            console.log(`🔄 [ShellX] 重连中... (${this.reconnectAttempts}/${this.config.reconnectMaxAttempts})` +
+                this.wsUrl);
             if (this.reconnectAttempts >= this.config.reconnectMaxAttempts) {
+                console.error('❌ [ShellX] 重连失败，达到最大重连次数');
                 (_b = (_a = this.config).onReconnectFailed) === null || _b === void 0 ? void 0 : _b.call(_a);
                 return;
             }
@@ -662,7 +761,7 @@ class WebSocketTaskClient {
         }
         return {
             type: task.type,
-            commandType: task.commandType
+            commandType: task.commandType,
         };
     }
     /**
@@ -685,9 +784,8 @@ class WebSocketTaskClient {
         return completedCount;
     }
 }
-exports.WebSocketTaskClient = WebSocketTaskClient;
-exports.default = WebSocketTaskClient;
-// PromptFlow 模块已移至 examples/ 目录作为示例实现
+exports.ConnectionTaskClient = ConnectionTaskClient;
+exports.default = ConnectionTaskClient;
 var shellx_1 = require("./shellx");
 Object.defineProperty(exports, "createShellXWithShellMonitoring", { enumerable: true, get: function () { return shellx_1.createShellXWithShellMonitoring; } });
 Object.defineProperty(exports, "ShellX", { enumerable: true, get: function () { return shellx_1.ShellX; } });

package/dist/protocol.d.ts

@@ -43,6 +43,8 @@ export type ScreenShotResponse = {
     dimensions: ScreenDimensions;
     success?: boolean;
     errorMessage?: string;
+    isVideo?: boolean;
+    videoPath?: string;
 };
 export type ScreenInfoResponse = {
     displayId?: number;
@@ -101,6 +103,7 @@ export interface ActionSequence {
 }
 export type JSONData = {
     findElement?: UIHierarchy;
+    findAllElement?: string;
     waitElement?: WAITElement;
     screenShot?: ScreenShotResponse;
     screenInfo?: ScreenInfoResponse;
@@ -158,6 +161,9 @@ export type ScreenShotOptions = {
     quality?: number;
     scale?: number;
     region?: ScreenRegion;
+    isRecording?: boolean;
+    saveToFile?: boolean;
+    duration?: number;
 };
 /** Client message type, see the Rust version. */
 export type WsClient = {
@@ -205,6 +211,17 @@ export interface ShellCommandAction {
     activity?: string;
     command: string;
 }
+export interface ScreenShotAction {
+    title?: string;
+    type: "screenShot";
+    activity?: string;
+    format: 'png' | 'jpeg';
+    quality?: number;
+    scale?: number;
+    region?: ScreenRegion;
+    isRecording?: boolean;
+    duration?: number;
+}
 export interface getAppInfoAction {
     title?: string;
     type: "get_app_info";

package/dist/shellx.d.ts

@@ -1,9 +1,8 @@
-import type { ElementSelector, ActionSequence, UIElement, ScreenShotOptions, WsServer } from './protocol';
-import WebSocketTaskClient from './index';
-export type { UIElement, ElementSelector, ActionSequence };
-export { ShellX as AutomationHelpers };
-export { createShellX as createHelpers };
-export { createShellXWithShellMonitoring as createHelpersWithShellMonitoring };
+import type { ElementSelector, ActionSequence as LegacyActionSequence, UIElement, ScreenShotOptions, WsServer } from './protocol';
+import type { ActionResult, Click, Input, Swipe, Key, Wait, Find, Command, AppInfo, Screenshot, Point, Element, FindResult, CommandResult, Action, Actions } from './types';
+import ConnectionTaskClient from './index';
+export type { UIElement, ElementSelector, LegacyActionSequence };
+export type { ActionResult, Click, Input, Swipe, Key, Wait, Find, Command, AppInfo, Screenshot, Point, Element, FindResult, CommandResult, Action, Actions };
 interface ShellCommandResult {
     success: boolean;
     output: string;
@@ -28,11 +27,11 @@ export declare class ShellX {
     private client;
     private shellOutputBuffers;
     private shellCommandPromises;
-    constructor(client: WebSocketTaskClient);
+    constructor(client: ConnectionTaskClient);
     /**
      * Get the WebSocket client instance
      */
-    getClient(): WebSocketTaskClient;
+    getClient(): ConnectionTaskClient;
     /**
      * 设置 shell 输出监听器（需要在创建 WebSocketTaskClient 时调用）
      */
@@ -77,6 +76,58 @@ export declare class ShellX {
      * 生成命令唯一标识
      */
     private generateCommandKey;
+    /**
+     * 通用重试机制
+     */
+    private withRetry;
+    /**
+     * 转换精简选择器为原始选择器
+     */
+    private convertSelector;
+    /**
+     * 转换精简元素为原始元素
+     */
+    private convertElement;
+    /**
+     * 点击操作 - 支持元素ID、坐标或选择器
+     */
+    click(clickData: Click): Promise<ActionResult>;
+    /**
+     * 输入操作 - 支持元素ID、资源ID或选择器
+     */
+    input(inputData: Input): Promise<ActionResult>;
+    /**
+     * 滑动操作 - 支持坐标点
+     */
+    swipe(swipeData: Swipe): Promise<ActionResult>;
+    /**
+     * 按键操作
+     */
+    pressKey(keyData: Key): Promise<ActionResult>;
+    /**
+     * 等待操作 - 等待元素出现或消失
+     */
+    wait(waitData: Wait): Promise<ActionResult>;
+    /**
+     * 查找操作 - 查找元素
+     */
+    find(findData: Find): Promise<FindResult>;
+    /**
+     * 执行命令 - 兼容现有的 shell 命令执行
+     */
+    executeCommand(commandData: Command): Promise<CommandResult>;
+    /**
+     * 获取应用信息
+     */
+    getAppInfo(appInfoData: AppInfo): Promise<ActionResult>;
+    /**
+     * 截图操作
+     */
+    takeScreenshot(screenshotData?: Screenshot): Promise<ActionResult>;
+    /**
+     * 执行操作序列 - 支持多个操作连续执行
+     */
+    executeActions(actions: Array<Click | Input | Swipe | Key | Wait | Find | Command | AppInfo | Screenshot>): Promise<ActionResult[]>;
     /**
      * Smart element finder with retry logic
      */
@@ -108,10 +159,6 @@ export declare class ShellX {
         clear?: boolean;
         hideKeyboard?: boolean;
     }): Promise<boolean>;
-    /**
-     * Swipe in a direction
-     */
-    swipe(direction: 'up' | 'down' | 'left' | 'right', distance?: number, duration?: number): Promise<void>;
     /**
      * Take screenshot and save info
      */
@@ -175,12 +222,9 @@ export declare class ShellX {
 /**
  * Create ShellX instance
  */
-export declare function createShellX(client: WebSocketTaskClient): ShellX;
+export declare function createShellX(client: ConnectionTaskClient): ShellX;
 /**
  * Create ShellX instance with automatic authentication and shell output monitoring
  * 自动处理ShellX.ai认证和连接，无需外部提供连接地址
  */
-export declare function createShellXWithShellMonitoring(config?: any): Promise<{
-    client: WebSocketTaskClient;
-    shellx: ShellX;
-}>;
+export declare function createShellXWithShellMonitoring(config?: any): Promise<ShellX>;