shellx-ai 1.0.0

package/dist/index.js

@@ -0,0 +1,398 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createShellXWithShellMonitoring = exports.createShellX = exports.ShellX = exports.WebSocketTaskClient = void 0;
+const uuid_1 = require("uuid");
+const constants_1 = require("./constants");
+const utils_1 = require("./utils");
+const cbor_x_1 = require("cbor-x");
+/**
+ * Enhanced WebSocket Task Client with protocol-aware task methods
+ */
+class WebSocketTaskClient {
+    constructor(wsUrl, config = {}) {
+        this.ws = null;
+        this.connected = false;
+        this.pendingTasks = new Map();
+        this.messageQueue = [];
+        this.reconnectAttempts = 0;
+        this.pingIntervalId = null;
+        this.wsUrl = wsUrl;
+        this.config = Object.assign(Object.assign({}, constants_1.DEFAULT_CONFIG), config);
+        this.init();
+    }
+    init() {
+        console.log('Initializing WebSocket client...');
+        this.ws = new WebSocket(this.wsUrl);
+        this.ws.binaryType = 'arraybuffer';
+        this.ws.onopen = () => {
+            var _a, _b;
+            console.log('Connected to WebSocket server');
+            this.connected = true;
+            this.reconnectAttempts = 0;
+            (_b = (_a = this.config).onOpen) === null || _b === void 0 ? void 0 : _b.call(_a);
+            this.flushQueue();
+            this.startPing();
+        };
+        this.ws.onmessage = (event) => {
+            this.handleMessage(event);
+        };
+        this.ws.onclose = (event) => {
+            var _a, _b;
+            this.connected = false;
+            (_b = (_a = this.config).onClose) === null || _b === void 0 ? void 0 : _b.call(_a, event);
+            this.stopPing();
+            this.reconnect();
+        };
+        this.ws.onerror = (error) => {
+            var _a, _b;
+            (_b = (_a = this.config).onError) === null || _b === void 0 ? void 0 : _b.call(_a, error);
+        };
+    }
+    handleMessage(event) {
+        let serverMessage;
+        try {
+            // Handle CBOR decoding
+            let binaryData;
+            if (event.data instanceof ArrayBuffer) {
+                binaryData = new Uint8Array(event.data);
+            }
+            else if (event.data instanceof Uint8Array) {
+                binaryData = event.data;
+            }
+            else {
+                // JSON fallback
+                serverMessage = JSON.parse(event.data);
+                this.processServerMessage(serverMessage);
+                return;
+            }
+            serverMessage = (0, cbor_x_1.decode)(binaryData);
+            this.processServerMessage(serverMessage);
+        }
+        catch (cborError) {
+            console.log('CBOR decode failed, trying JSON fallback:', cborError);
+            try {
+                let textData;
+                if (event.data instanceof ArrayBuffer) {
+                    textData = new TextDecoder().decode(event.data);
+                }
+                else {
+                    textData = event.data;
+                }
+                serverMessage = JSON.parse(textData);
+                this.processServerMessage(serverMessage);
+            }
+            catch (jsonError) {
+                console.error('Failed to parse message:', { cborError, jsonError });
+            }
+        }
+    }
+    processServerMessage(message) {
+        var _a, _b;
+        console.log('Received server message:', message);
+        // Call user-defined message handler
+        (_b = (_a = this.config).onMessage) === null || _b === void 0 ? void 0 : _b.call(_a, message);
+        // Handle specific message types and resolve pending tasks
+        if (message.jsonData) {
+            this.handleJsonDataResponse(message.jsonData);
+        }
+        // Handle other server message types (hello, users, shells, etc.)
+        if (message.hello !== undefined) {
+            console.log('Server hello, user ID:', message.hello);
+        }
+        if (message.pong !== undefined) {
+            console.log('Received pong:', message.pong);
+        }
+        if (message.error) {
+            console.error('Server error:', message.error);
+        }
+    }
+    handleJsonDataResponse(jsonData) {
+        // Match responses to pending tasks based on response type
+        for (const [taskId, task] of this.pendingTasks.entries()) {
+            let responseData = null;
+            let shouldResolve = false;
+            switch (task.type) {
+                case 'findElement':
+                    if (jsonData.findElement) {
+                        responseData = jsonData.findElement;
+                        shouldResolve = true;
+                    }
+                    break;
+                case 'waitElement':
+                    if (jsonData.waitElement) {
+                        responseData = jsonData.waitElement;
+                        shouldResolve = true;
+                    }
+                    break;
+                case 'screenShot':
+                    if (jsonData.screenShot) {
+                        responseData = jsonData.screenShot;
+                        shouldResolve = true;
+                    }
+                    break;
+                case 'screenInfo':
+                    if (jsonData.screenInfo) {
+                        responseData = jsonData.screenInfo;
+                        shouldResolve = true;
+                    }
+                    break;
+                case 'appList':
+                    if (jsonData.appList) {
+                        responseData = jsonData.appList;
+                        shouldResolve = true;
+                    }
+                    break;
+                case 'action':
+                    if (jsonData.action_event) {
+                        responseData = jsonData.action_event;
+                        shouldResolve = true;
+                    }
+                    break;
+            }
+            if (shouldResolve) {
+                clearTimeout(task.timer);
+                this.pendingTasks.delete(taskId);
+                if ((responseData === null || responseData === void 0 ? void 0 : responseData.success) === false) {
+                    task.reject(new Error(responseData.errorMessage || 'Task failed'));
+                }
+                else {
+                    task.resolve(responseData);
+                }
+                break; // Only resolve the first matching task
+            }
+        }
+    }
+    sendMessage(message, taskType) {
+        return new Promise((resolve, reject) => {
+            const taskId = (0, uuid_1.v4)();
+            if (taskType) {
+                const timer = setTimeout(() => {
+                    this.pendingTasks.delete(taskId);
+                    reject(new Error(`Task ${taskType} timeout (${this.config.timeout}ms)`));
+                }, this.config.timeout);
+                this.pendingTasks.set(taskId, { resolve, reject, timer, type: taskType });
+            }
+            if (this.connected && this.ws) {
+                try {
+                    console.log('Sending message:', message);
+                    this.ws.send((0, cbor_x_1.encode)(message));
+                    if (!taskType)
+                        resolve(undefined); // For fire-and-forget messages
+                }
+                catch (error) {
+                    if (taskType) {
+                        this.pendingTasks.delete(taskId);
+                    }
+                    reject(error);
+                }
+            }
+            else {
+                this.messageQueue.push(message);
+                if (!taskType)
+                    resolve(undefined);
+            }
+        });
+    }
+    // ===== PROTOCOL-AWARE TASK METHODS =====
+    /**
+     * Find UI elements on the screen
+     */
+    findElement(selector, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const findAction = {
+                type: 'find',
+                selector,
+                options
+            };
+            return this.sendMessage({ findElement: findAction }, 'findElement');
+        });
+    }
+    /**
+     * Wait for UI element to appear
+     */
+    waitElement(selector, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const waitAction = {
+                type: 'wait',
+                selector,
+                options
+            };
+            return this.sendMessage({ waitElement: waitAction }, 'waitElement');
+        });
+    }
+    /**
+     * Take a screenshot
+     */
+    screenShot(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({ screenShot: options || { format: 'png' } }, 'screenShot');
+        });
+    }
+    /**
+     * Get screen information
+     */
+    getScreenInfo() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({ screenInfo: {} }, 'screenInfo');
+        });
+    }
+    /**
+     * Get list of installed apps
+     */
+    getAppList(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({ appList: options || {} }, 'appList');
+        });
+    }
+    /**
+     * Get specific app information
+     */
+    getAppInfo(packageName) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({ appInfo: { packageName } }, 'appInfo');
+        });
+    }
+    /**
+     * Execute an action sequence
+     */
+    executeAction(actionSequence) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({ actions: actionSequence }, 'action');
+        });
+    }
+    /**
+     * Execute promptflow actions
+     */
+    executePromptFlow(actionSequence) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({ promptflowx: actionSequence });
+        });
+    }
+    /**
+     * Listen for display changes
+     */
+    screenChange(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({
+                screenChange: options || { enable: true }
+            });
+        });
+    }
+    /**
+     * Switch to a different node
+     */
+    switchNode(nodeId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({ switchNode: nodeId });
+        });
+    }
+    /**
+     * Send authentication data
+     */
+    authenticate(authData) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({ authenticate: authData });
+        });
+    }
+    /**
+     * Set user name
+     */
+    setName(name) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({ setName: name });
+        });
+    }
+    /**
+     * Send chat message
+     */
+    sendChat(message) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage({ chat: message });
+        });
+    }
+    /**
+     * Send raw message (for custom integrations)
+     */
+    sendRawMessage(message) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.sendMessage(message);
+        });
+    }
+    // ===== CONNECTION MANAGEMENT =====
+    startPing() {
+        this.stopPing();
+        console.log('Starting ping...');
+        this.pingIntervalId = setInterval(() => {
+            if (this.ws && this.connected) {
+                this.ws.send((0, cbor_x_1.encode)({ ping: BigInt(Date.now()) }));
+            }
+        }, this.config.pingInterval);
+    }
+    stopPing() {
+        if (this.pingIntervalId !== null) {
+            clearInterval(this.pingIntervalId);
+            this.pingIntervalId = null;
+        }
+    }
+    reconnect() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            if (this.reconnectAttempts >= this.config.reconnectMaxAttempts) {
+                (_b = (_a = this.config).onReconnectFailed) === null || _b === void 0 ? void 0 : _b.call(_a);
+                return;
+            }
+            this.reconnectAttempts++;
+            yield (0, utils_1.wait)(this.config.reconnectInterval * this.reconnectAttempts);
+            this.init();
+        });
+    }
+    flushQueue() {
+        while (this.messageQueue.length > 0 && this.ws) {
+            const message = this.messageQueue.shift();
+            if (message) {
+                try {
+                    console.log('flushQueue Sending message:', message);
+                    this.ws.send((0, cbor_x_1.encode)(message));
+                }
+                catch (error) {
+                    console.error('Failed to send queued message:', error);
+                }
+            }
+        }
+    }
+    close() {
+        var _a;
+        this.connected = false;
+        (_a = this.ws) === null || _a === void 0 ? void 0 : _a.close();
+        this.pendingTasks.clear();
+        this.messageQueue = [];
+        this.stopPing();
+    }
+    // Getters for debugging/monitoring
+    get isConnected() {
+        return this.connected;
+    }
+    get pendingTaskCount() {
+        return this.pendingTasks.size;
+    }
+    get queuedMessageCount() {
+        return this.messageQueue.length;
+    }
+}
+exports.WebSocketTaskClient = WebSocketTaskClient;
+exports.default = WebSocketTaskClient;
+// Export ShellX automation utilities
+var shellx_1 = require("./shellx");
+Object.defineProperty(exports, "ShellX", { enumerable: true, get: function () { return shellx_1.ShellX; } });
+Object.defineProperty(exports, "createShellX", { enumerable: true, get: function () { return shellx_1.createShellX; } });
+Object.defineProperty(exports, "createShellXWithShellMonitoring", { enumerable: true, get: function () { return shellx_1.createShellXWithShellMonitoring; } });
+// PromptFlow 模块已移至 examples/ 目录作为示例实现 

package/dist/protocol.d.ts

@@ -0,0 +1,319 @@
+type Sid = number;
+type Uid = number;
+/** UI Element bounds */
+export type UIBounds = {
+    left: number;
+    top: number;
+    right: number;
+    bottom: number;
+};
+/** UI Element information */
+export type UIElement = {
+    elementId: string;
+    className: string;
+    resourceId: string;
+    text: string;
+    describe: string;
+    bounds: UIBounds;
+    visible: boolean;
+    clickable: boolean;
+};
+/** UI Hierarchy information */
+export type UIHierarchy = {
+    elements: UIElement[];
+    timestamp: number;
+    found: boolean;
+    totalMatches?: number;
+    success?: boolean;
+    errorMessage?: string;
+};
+export type WAITElement = {
+    element: UIElement;
+    timestamp: bigint;
+    elapsedTimeMs: boolean;
+    success?: boolean;
+    errorMessage?: string;
+};
+/** Screen capture response */
+export type ScreenShotResponse = {
+    imageData: string;
+    format: 'png' | 'jpeg';
+    timestamp: bigint;
+    dimensions: ScreenDimensions;
+    success?: boolean;
+    errorMessage?: string;
+};
+export type ScreenInfoResponse = {
+    displayId?: number;
+    width?: number;
+    height?: number;
+    density?: number;
+    name?: string;
+    visible?: boolean;
+    foregroundPackageName?: string;
+    foregroundActivityName?: string;
+    screenOn?: boolean;
+    screenUnlocked?: boolean;
+    accurateForegroundActivity?: string;
+    accurateForegroundApp?: string;
+    timestamp: bigint;
+    success?: boolean;
+    errorMessage?: string;
+};
+export type App = {
+    packageName: string;
+    appName: string;
+    versionName: string;
+    versionCode: number;
+    isSystemApp: boolean;
+    isEnabled: boolean;
+    firstInstallTime: number;
+    lastUpdateTime: number;
+    sourceDir: string;
+};
+export type AppListResponse = {
+    apps: App[];
+    totalCount: number;
+    systemAppCount: number;
+    userAppCount: number;
+    timestamp: bigint;
+    success?: boolean;
+    errorMessage?: string;
+};
+/** Screen change event response */
+export type ScreenChangeEvent = {
+    eventType: 'SCREEN_CHANGED';
+    timestamp: bigint;
+    changeRatio: number;
+    imageData?: string;
+};
+export interface ActionSequence {
+    title?: string;
+    description?: string;
+    actions: Array<ShellCommandAction | ClickAction | InputAction | SwipeAction | KeyAction | WaitAction | findAction | getAppInfoAction | completeAction>;
+    options?: {
+        continueOnError?: boolean;
+        timeoutMs?: number;
+    };
+    deleted?: boolean;
+}
+export type JSONData = {
+    findElement?: UIHierarchy;
+    waitElement?: WAITElement;
+    screenShot?: ScreenShotResponse;
+    screenInfo?: ScreenInfoResponse;
+    screenChange?: ScreenChangeEvent;
+    appList?: AppListResponse;
+    promptflowx?: ActionSequence[];
+    action_event?: ClickAction | KeyAction | SwipeAction;
+};
+/** Position and size of a window, see the Rust version. */
+export type WsWinsize = {
+    x: number;
+    y: number;
+    rows: number;
+    cols: number;
+    mode?: number;
+};
+/** Information about a user, see the Rust version */
+export type WsUser = {
+    name: string;
+    cursor: [number, number] | null;
+    focus: number | null;
+};
+/** Screen capture dimensions */
+export type ScreenDimensions = {
+    width: number;
+    height: number;
+};
+/** Server message type, see the Rust version. */
+export type WsServer = {
+    input?: string;
+    hello?: Uid;
+    invalidAuth?: [];
+    users?: [Uid, WsUser][];
+    userDiff?: [Uid, WsUser | null];
+    shells?: [Sid, WsWinsize][];
+    chunks?: [Sid, number, Uint8Array[]];
+    hear?: [Uid, string, string];
+    shellLatency?: number | bigint;
+    pong?: number | bigint;
+    error?: string;
+    jsonData?: JSONData;
+    uiClick?: [string, number, number];
+    uiRefresh?: boolean;
+};
+/** Screen capture region */
+export type ScreenRegion = {
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+};
+/** Screen capture options */
+export type ScreenShotOptions = {
+    format: 'png' | 'jpeg';
+    quality?: number;
+    scale?: number;
+    region?: ScreenRegion;
+};
+/** Client message type, see the Rust version. */
+export type WsClient = {
+    authenticate?: Uint8Array;
+    setName?: string;
+    setCursor?: [number, number] | null;
+    setFocus?: number | null;
+    create?: [number, number];
+    close?: Sid;
+    move?: [Sid, WsWinsize | null];
+    data?: [Sid, Uint8Array, bigint];
+    subscribe?: [Sid, number];
+    chat?: string;
+    ping?: bigint;
+    findElement?: findAction;
+    waitElement?: WaitAction;
+    screenShot?: ScreenShotOptions;
+    screenInfo?: {};
+    screenChange?: {
+        interval?: number;
+        compareThreshold?: number;
+        includeScreenshot?: boolean;
+        enable?: boolean;
+    };
+    action?: ShellXActions;
+    actions?: ActionSequence;
+    appInfo?: {
+        packageName: string;
+    };
+    appList?: {
+        includeSystemApps?: boolean;
+        includeDisabledApps?: boolean;
+    };
+    promptflowx?: ActionSequence;
+    switchNode?: string;
+};
+export type ShellXActions = {
+    title?: string;
+    thought?: string;
+    action: ShellCommandAction | ClickAction | InputAction | SwipeAction | KeyAction | WaitAction | findAction | getAppInfoAction | completeAction;
+};
+interface ShellCommandAction {
+    title?: string;
+    type: "command";
+    command: string;
+}
+interface getAppInfoAction {
+    title?: string;
+    type: "get_app_info";
+    packageName: string;
+}
+interface ClickAction {
+    title?: string;
+    type: 'click';
+    target: {
+        type: "elementId";
+        value: string;
+    } | {
+        type: "resourceId";
+        value: string;
+    } | {
+        type: "coordinate";
+        value: {
+            "x": number;
+            "y": number;
+        };
+    };
+    options?: {
+        timeoutMs?: number;
+        durationMs?: number;
+        waitAfterMs?: number;
+        clickType?: 'normal' | 'long' | 'double';
+    };
+}
+interface InputAction {
+    title?: string;
+    type: "input";
+    text: string;
+    target: {
+        type?: "elementId";
+        value?: string;
+    } | {
+        type: "resourceId";
+        value: string;
+    };
+    options?: {
+        waitAfterMs?: number;
+        replaceExisting?: boolean;
+        hideKeyboardAfter?: boolean;
+    };
+}
+interface SwipeAction {
+    title?: string;
+    type: 'swipe';
+    "from": {
+        "x": number;
+        "y": number;
+    };
+    "to": {
+        "x": number;
+        "y": number;
+    };
+    options?: {
+        durationMs?: number;
+        steps?: number;
+        inertia?: boolean;
+        waitAfterMs?: number;
+        replaceExisting?: boolean;
+        hideKeyboardAfter?: boolean;
+    };
+}
+interface KeyAction {
+    title?: string;
+    type: "key";
+    keyCode?: string;
+    options?: {
+        longPress?: boolean;
+    };
+}
+export interface WaitAction {
+    title?: string;
+    type?: "wait";
+    selector: ElementSelector;
+    options?: WaitOptions;
+    condition?: string;
+}
+export interface findAction {
+    title?: string;
+    type?: "find";
+    selector: ElementSelector;
+    options?: FindOptions;
+}
+export type completeAction = {
+    title?: string;
+    type: "complete";
+    result?: string;
+};
+export type ElementSelector = {
+    elementId?: string;
+    className?: string;
+    resourceId?: string;
+    text?: string;
+    textContains?: string;
+    textMatches?: string;
+    visible?: boolean;
+    clickable?: boolean;
+};
+export type WaitOptions = {
+    timeoutMs?: number;
+    intervalMs?: number;
+    failOnTimeout?: boolean;
+};
+export type FindOptions = {
+    timeout?: number;
+    visibleOnly?: boolean;
+    clickableOnly?: boolean;
+    multiple?: boolean;
+    waitForVisible?: boolean;
+    maxResults?: number;
+};
+export {};

package/dist/protocol.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });