shellx-ai 1.1.1 → 1.1.2

package/dist/protocol.d.ts

@@ -161,7 +161,7 @@ export type JSONData = {
     findElement?: UIHierarchy;
     findAllElement?: string;
     waitElement?: WAITElement;
-    screenShot?: ScreenShotResponse;
+    screenshot?: ScreenShotResponse;
     screenInfo?: ScreenInfoResponse;
     screenChange?: ScreenChangeEvent;
     appList?: AppListResponse;
@@ -170,6 +170,17 @@ export type JSONData = {
     action_event?: ClickAction | KeyAction | SwipeAction;
     /** Forwarded openclaw gateway message */
     gatewayMsg?: GatewayMsg;
+    /** Data module response */
+    dataRequest?: {
+        requestType: string;
+        result: unknown;
+    };
+    /** Shell command execution result (ptyType=0) */
+    command?: {
+        success: boolean;
+        result?: string;
+        errorMessage?: string;
+    };
 };
 /** Position and size of a window, see the Rust version. */
 export type WsWinsize = {
@@ -206,6 +217,8 @@ export type WsServer = {
     jsonData?: JSONData;
     uiClick?: [string, number, number];
     uiRefresh?: boolean;
+    /** Gateway message is sent as JSON string from Java, needs parsing in SDK */
+    gatewayMsg?: string;
 };
 /** Screen capture region */
 export type ScreenRegion = {
@@ -240,7 +253,7 @@ export type WsClient = {
     ping?: bigint;
     findElement?: findAction;
     waitElement?: WaitAction;
-    screenShot?: ScreenShotOptions;
+    screenshot?: ScreenShotOptions;
     screenInfo?: {
         keepScreenOn?: boolean;
         wakeApp?: boolean;
@@ -264,6 +277,11 @@ export type WsClient = {
     switchNode?: string;
     asrControl?: AsrControl;
     ttsControl?: TtsControl;
+    /** Data module request */
+    dataRequest?: {
+        requestType: string;
+        params: unknown;
+    };
     /** Openclaw gateway request to forward to the gateway */
     gatewayReq?: {
         type: "req";
@@ -276,7 +294,7 @@ export type ShellXActions = {
     taskId?: string;
     title?: string;
     thought?: string;
-    action: ShellCommandAction | ClickAction | InputAction | SwipeAction | KeyAction | WaitAction | findAction | ClipboardAction | getAppInfoAction | completeAction;
+    action: ShellCommandAction | ClickAction | InputAction | SwipeAction | KeyAction | WaitAction | findAction | ClipboardAction | getAppInfoAction | ScreenShotAction | completeAction;
 };
 export interface ShellCommandAction {
     title?: string;
@@ -285,10 +303,12 @@ export interface ShellCommandAction {
     thinking?: string;
     note?: string;
     command: string;
+    ptyType?: 0 | 1;
+    terminalId?: number;
 }
 export interface ScreenShotAction {
     title?: string;
-    type: "screenShot";
+    type: "screenshot";
     activity?: string;
     thinking?: string;
     note?: string;
@@ -301,7 +321,7 @@ export interface ScreenShotAction {
 }
 export interface getAppInfoAction {
     title?: string;
-    type: "get_app_info";
+    type: "getAppInfo";
     activity?: string;
     thinking?: string;
     note?: string;
@@ -462,5 +482,7 @@ export type AsrControl = {
 export type TtsControl = {
     action: "tts_speak" | "tts_stop";
     text?: string;
+    voice_type?: string;
+    volume?: number;
 };
 export {};

package/dist/shellx.d.ts

@@ -7,14 +7,13 @@
  * @module shellx
  */
 import type { ElementSelector, UIElement, WsServer } from "./protocol.js";
-import type { Actions, AppInfo, AppInfoResult, BaseResult, Click, ClickResult, Clipboard, ClipboardResult, Command, CommandResult as TypesCommandResult, Element, ExecuteActionsResult, Find, FindResult as TypesFindResult, Input, InputResult, Key, KeyResult, ScreenInfoResult, Screenshot, ScreenshotResult, Swipe, SwipeResult, Wait, WaitResult } from "./types.js";
+import type { Actions, AppInfo, AppInfoResult, Asr, AsrResult, BaseResult, Click, ClickResult, Clipboard, ClipboardResult, Command, CommandResult as TypesCommandResult, Element, ExecuteActionsResult, Find, FindResult as TypesFindResult, Input, InputResult, Key, KeyResult, ScreenInfoResult, Screenshot, ScreenshotResult, Swipe, SwipeResult, Tts, TtsResult, Wait, WaitResult } from "./types.js";
 import ConnectionClient from "./index.js";
 import { type LogLevel } from "./logger.js";
 import type { WsClient } from "./protocol.js";
 export type { UIElement, ElementSelector };
 export type { LogLevel } from "./logger.js";
-export type { Click, Input, Swipe, Key, Wait, Find, Command, AppInfo, Screenshot, Clipboard, Actions, ClickResult, InputResult, SwipeResult, KeyResult, WaitResult, TypesFindResult as FindResult, TypesCommandResult as CommandResult, ClipboardResult, AppInfoResult, ScreenshotResult, ScreenInfoResult, ExecuteActionsResult, BaseResult, Element, };
-export type { ShellCommandResult, ShellCommandOptions } from "./shell/output-buffer.js";
+export type { Click, Input, Swipe, Key, Wait, Find, Command, AppInfo, Screenshot, Clipboard, Asr, AsrResult, Tts, TtsResult, Actions, ClickResult, InputResult, SwipeResult, KeyResult, WaitResult, TypesFindResult as FindResult, TypesCommandResult as CommandResult, ClipboardResult, AppInfoResult, ScreenshotResult, ScreenInfoResult, ExecuteActionsResult, BaseResult, Element, };
 /**
  * Configuration options for ShellX instance
  */
@@ -43,6 +42,23 @@ export interface ShellXOptions {
     onReconnectFailed?: () => void;
     /** Callback for WebSocket messages */
     onMessage?: (message: WsServer) => void;
+    onGatewayMessage?: (message: any) => void;
+    /**
+     * Called every time the device sends an ASR recognition result.
+     * Use this to receive speech-to-text output without handling raw messages.
+     *
+     * @example
+     * ```typescript
+     * const shellx = new ShellX({
+     *   deviceId: 'your-device-id',
+     *   onAsrResult: (text) => console.log('Recognised:', text)
+     * });
+     * await shellx.startAsr();
+     * // ...user speaks...
+     * await shellx.stopAsr();
+     * ```
+     */
+    onAsrResult?: (text: string) => void;
 }
 /**
  * ShellX - High-level Android automation SDK (Recommended API)
@@ -96,8 +112,6 @@ export declare class ShellX {
     private uiActionHandler;
     /** Element finder for locating UI elements */
     private elementFinder;
-    /** Shell command executor for running shell commands */
-    private shellCommandExecutor;
     /** Connection client for device communication */
     private client;
     /**
@@ -349,7 +363,7 @@ export declare class ShellX {
      *
      * @example
      * ```typescript
-     * // Simplified - execute command
+     * // Simplified - execute command (ptyType=0, shell execution with result)
      * const result = await shellx.command('ls -la');
      *
      * // Simplified - execute with timeout
@@ -358,6 +372,12 @@ export declare class ShellX {
      * // Full - execute with all options
      * const result = await shellx.command({ cmd: 'ls -la', timeout: 5000 });
      *
+     * // PTY execution (ptyType=1, one-way execution without result)
+     * const result = await shellx.command({ cmd: 'ls -la', ptyType: 1 });
+     *
+     * // PTY execution with specific terminal (ptyType=1 with terminalId)
+     * const result = await shellx.command({ cmd: 'ls -la', ptyType: 1, terminalId: 12345 });
+     *
      * console.log(result.output);
      * ```
      */
@@ -542,14 +562,6 @@ export declare class ShellX {
         visibleOnly?: boolean;
         clickableOnly?: boolean;
     }): Promise<UIElement[]>;
-    /**
-     * Wait for any of multiple elements to appear
-     * @deprecated Use waitAnyElement() instead for consistency
-     */
-    waitForAnyElement(selectors: ElementSelector[], timeout?: number): Promise<{
-        element: UIElement;
-        selectorIndex: number;
-    } | null>;
     /**
      * Wait for any of multiple elements to appear
      *
@@ -591,25 +603,6 @@ export declare class ShellX {
      * ```
      */
     scrollToFindElement(selector: ElementSelector, maxScrolls?: number, direction?: "up" | "down"): Promise<UIElement | null>;
-    /**
-     * Handle shell output from WebSocket messages
-     *
-     * This method should be called in the WebSocket message handler.
-     *
-     * @param chunks - Shell output chunks
-     *
-     * @example
-     * ```typescript
-     * const client = new ConnectionClient(deviceId, {
-     *   onMessage: (message) => {
-     *     if (message.chunks) {
-     *       shellx.handleShellOutput(message.chunks);
-     *     }
-     *   }
-     * });
-     * ```
-     */
-    handleShellOutput(chunks: [number, number, Uint8Array[]]): void;
     /**
      * Send raw WebSocket message (for advanced use cases)
      *
@@ -649,41 +642,65 @@ export declare class ShellX {
      */
     sendRawMessage(message: WsClient): Promise<unknown>;
     /**
-     * Start ASR (Automatic Speech Recognition)
-     * @returns Promise that resolves when speech recognition has started
+     * Start ASR (Automatic Speech Recognition) on the device.
+     *
+     * The device will begin capturing microphone audio and streaming recognition
+     * results back. Each result is delivered via the `onAsrResult` callback
+     * configured in `ShellXOptions`, or you can listen to raw messages via
+     * `onMessage` and check `message.jsonData?.asr`.
+     *
+     * @param options - Optional ASR options (e.g. per-call `onResult` callback)
+     * @returns Promise that resolves when the start command has been sent
      *
      * @example
      * ```typescript
-     * await shellx.startSpeechRecognition();
-     * console.log('Speech recognition started');
+     * const shellx = new ShellX({
+     *   deviceId: 'your-device-id',
+     *   onAsrResult: (text) => console.log('ASR:', text)
+     * });
+     * await shellx.ready();
+     * await shellx.startAsr();
+     * // ...user speaks into device microphone...
+     * await shellx.stopAsr();
      * ```
      */
-    startSpeechRecognition(): Promise<void>;
+    startAsr(options?: Asr): Promise<void>;
     /**
-     * Stop ASR (Automatic Speech Recognition)
-     * @returns Promise that resolves when speech recognition has stopped
+     * Stop ASR (Automatic Speech Recognition) on the device.
+     *
+     * @returns Promise that resolves when the stop command has been sent
      *
      * @example
      * ```typescript
-     * await shellx.stopSpeechRecognition();
-     * console.log('Speech recognition stopped');
+     * await shellx.stopAsr();
      * ```
      */
-    stopSpeechRecognition(): Promise<void>;
+    stopAsr(): Promise<void>;
     /**
-     * Speak text aloud using TTS (Text-to-Speech)
-     * @param text - The text to speak
-     * @returns Promise that resolves when TTS has started speaking
+     * Speak text aloud using TTS (Text-to-Speech) on the device.
+     *
+     * @param text - The text to synthesise and play
+     * @param options - Optional TTS options (e.g. voice type override)
+     * @returns Promise resolving to TtsResult when the command is sent
      *
      * @example
      * ```typescript
+     * // Simple usage
      * await shellx.speak('Hello, world!');
+     *
+     * // With voice type
+     * await shellx.speak('你好，世界！', { voiceType: 'BV002_streaming' });
+     *
+     * // Or pass a Tts object
+     * await shellx.speak({ text: '你好！', voiceType: 'BV002_streaming' });
      * ```
      */
-    speak(text: string): Promise<void>;
+    speak(text: string, options?: Omit<Tts, "text">): Promise<TtsResult>;
+    speak(ttsData: Tts): Promise<TtsResult>;
     /**
-     * Stop TTS (Text-to-Speech)
-     * @returns Promise that resolves when TTS has stopped speaking
+     * Stop TTS (Text-to-Speech) on the device.
+     *
+     * @returns Promise that resolves when the stop command has been sent
      *
      * @example
      * ```typescript
@@ -692,16 +709,82 @@ export declare class ShellX {
      */
     stopSpeaking(): Promise<void>;
     /**
-     * Static factory method to create shell output handler
-     * @deprecated This method is kept for backward compatibility but is no longer needed
-     * @param shellx - The ShellX instance
-     * @returns Message handler function
+     * Close a terminal session by ID
+     *
+     * This method closes a PTY (pseudo-terminal) session and releases its resources.
+     * The terminal ID is the same as the one used in the `terminalId` parameter
+     * when executing commands.
+     *
+     * @param terminalId - Terminal ID (sid) to close
+     * @returns Promise that resolves when the close command has been sent
      *
      * @example
      * ```typescript
-     * const shellx = new ShellX({ deviceId: 'device-id' });
-     * const handler = ShellX.createShellOutputHandler(shellx);
+     * // Execute command in a specific terminal
+     * await shellx.command({ cmd: 'pwd', ptyType: 1, terminalId: 12345 });
+     *
+     * // Close the terminal when done
+     * await shellx.closeTerminal(12345);
+     * ```
+     */
+    closeTerminal(terminalId: number): Promise<void>;
+    /**
+     * Close the WebSocket connection and cleanup resources
+     *
+     * This method closes the underlying WebSocket connection to the device.
+     * All pending tasks will be cancelled with a connection-closed result.
+     *
+     * After calling this method, the ShellX instance cannot be used for further
+     * operations until a new connection is established.
+     *
+     * @example
+     * ```typescript
+     * // Close the connection when done
+     * await shellx.close();
+     * ```
+     */
+    close(): void;
+    /**
+     * Disconnect from the device (alias for close)
+     *
+     * @example
+     * ```typescript
+     * await shellx.disconnect();
+     * ```
+     */
+    disconnect(): void;
+    /**
+     * Send a data request to the device
+     *
+     * This is a generic method that allows sending data requests to the device
+     * and receiving typed responses. It can be used to implement custom data
+     * access functionality beyond what's provided by the ShellX API.
+     *
+     * @template TResponse - Type of the response
+     * @param requestType - The type of data request (e.g., "addAlarm", "getCalls", etc.)
+     * @param params - Request parameters
+     * @returns Promise resolving to the typed response
+     *
+     * @example
+     * ```typescript
+     * // Add an alarm
+     * const alarmResult = await shellx.sendDataRequest<AddAlarmResponse>(
+     *   "addAlarm",
+     *   { hour: 7, minute: 30, label: "Wake up", enabled: true }
+     * );
+     *
+     * // Get call records
+     * const callsResult = await shellx.sendDataRequest<GetCallsResponse>(
+     *   "getCalls",
+     *   { limit: 20 }
+     * );
+     *
+     * // Search contacts
+     * const contactsResult = await shellx.sendDataRequest<SearchContactsResponse>(
+     *   "searchContacts",
+     *   { query: "张三", limit: 10 }
+     * );
      * ```
      */
-    static createShellOutputHandler(shellx: ShellX): (message: WsServer) => void;
+    sendDataRequest<TResponse>(requestType: string, params: unknown): Promise<TResponse>;
 }