shellx-ai 1.0.11 → 1.1.0

package/dist/automation/ui-action-handler.d.ts

@@ -0,0 +1,330 @@
+/**
+ * UIActionHandler - A module for handling UI automation actions
+ *
+ * This module provides functionality for executing various UI actions such as
+ * clicking, inputting text, swiping, pressing keys, and more.
+ */
+import type { ElementSelector, ActionSequence as LegacyActionSequence, ScreenShotOptions, ScreenInfoResponse, ScreenShotResponse } from "../protocol.js";
+import type { Click, ClickResult, Input, InputResult, Swipe, SwipeResult, Key, KeyResult, Wait, WaitResult, Clipboard, ClipboardResult, AppInfo, AppInfoResult, Screenshot, ScreenshotResult } from "../types.js";
+/**
+ * Interface for ConnectionClient methods used by UIActionHandler
+ */
+interface IConnectionClient {
+    findElement(selector: ElementSelector, options?: unknown): Promise<{
+        elements?: unknown[];
+    } | undefined>;
+    executeAction(action: LegacyActionSequence): Promise<void>;
+    sendMessageWithTaskId<T = unknown>(message: unknown, type: string, options?: {
+        timeout?: number;
+        taskId?: string;
+    }): Promise<{
+        promise: Promise<T>;
+    }>;
+    screenShot(options: ScreenShotOptions): Promise<ScreenShotResponse>;
+    getScreenInfo(): Promise<ScreenInfoResponse>;
+}
+/**
+ * App list response types
+ */
+interface DeviceAppInfo {
+    packageName: string;
+    appName: string;
+    versionName: string;
+    versionCode: number;
+    isSystemApp: boolean;
+    isEnabled: boolean;
+    firstInstallTime: number;
+    lastUpdateTime: number;
+    sourceDir: string;
+}
+interface AppListResult {
+    apps: DeviceAppInfo[];
+    totalCount: number;
+    systemAppCount: number;
+    userAppCount: number;
+    timestamp: number;
+    success: boolean;
+    error?: string;
+}
+/**
+ * UIActionHandler class handles UI automation operations
+ *
+ * This class provides methods to:
+ * - Execute UI actions (click, input, swipe, key press)
+ * - Wait for elements
+ * - Take screenshots
+ * - Execute action sequences
+ * - Navigate through app UI
+ */
+export declare class UIActionHandler {
+    private client;
+    private elementFinder;
+    private logger;
+    /**
+     * Creates a UIActionHandler instance
+     *
+     * @param client - The ConnectionClient instance for UI operations
+     */
+    constructor(client: IConnectionClient);
+    /**
+     * Validate that client is initialized
+     *
+     * @returns true if client is valid, false otherwise
+     */
+    private validateClient;
+    /**
+     * Convert simplified selector to protocol selector
+     *
+     * @param selector - Simplified selector object
+     * @returns Protocol ElementSelector object
+     */
+    private convertSelector;
+    /**
+     * Execute action and verify it succeeded
+     *
+     * @param action - The action sequence to execute
+     */
+    private executeAndVerifyAction;
+    /**
+     * Execute operation with retry and unified error handling
+     *
+     * @param operation - Async operation to execute
+     * @param startTime - Operation start time
+     * @param options - Retry and error handling options
+     * @returns Promise resolving to operation result
+     */
+    private executeWithRetry;
+    /**
+     * Execute a click action
+     *
+     * @param clickData - Click configuration
+     * @returns Promise resolving to ClickResult
+     *
+     * @example
+     * ```typescript
+     * const result = await actionHandler.click({
+     *   elementId: 'element123',
+     *   clickType: 'single',
+     *   wait: 3000
+     * });
+     * ```
+     */
+    click(clickData: Click): Promise<ClickResult>;
+    /**
+     * Execute an input action
+     *
+     * @param inputData - Input configuration
+     * @returns Promise resolving to ActionResult
+     *
+     * @example
+     * ```typescript
+     * const result = await actionHandler.input({
+     *   elementId: 'element123',
+     *   text: 'Hello World',
+     *   clear: true,
+     *   hideKeyboard: false,
+     *   wait: 500
+     * });
+     * ```
+     */
+    input(inputData: Input): Promise<InputResult>;
+    /**
+     * Execute a swipe action
+     *
+     * @param swipeData - Swipe configuration
+     * @returns Promise resolving to ActionResult
+     *
+     * @example
+     * ```typescript
+     * const result = await actionHandler.swipe({
+     *   fromX: 500,
+     *   fromY: 1000,
+     *   toX: 500,
+     *   toY: 500,
+     *   duration: 800,
+     *   wait: 500
+     * });
+     * ```
+     */
+    swipe(swipeData: Swipe): Promise<SwipeResult>;
+    /**
+     * Execute a key press action
+     *
+     * @param keyData - Key configuration
+     * @returns Promise resolving to ActionResult
+     *
+     * @example
+     * ```typescript
+     * const result = await actionHandler.pressKey({
+     *   key: 'KEYCODE_BACK',
+     *   longPress: false,
+     *   wait: 500
+     * });
+     * ```
+     */
+    pressKey(keyData: Key): Promise<KeyResult>;
+    /**
+     * Wait for an element condition
+     *
+     * @param waitData - Wait configuration
+     * @returns Promise resolving to ActionResult
+     *
+     * @example
+     * ```typescript
+     * // Wait for element to appear
+     * const result = await actionHandler.wait({
+     *   text: 'Submit',
+     *   condition: 'visible',
+     *   timeout: 10000
+     * });
+     *
+     * // Wait for element to disappear
+     * const result = await actionHandler.wait({
+     *   text: 'Loading',
+     *   condition: 'gone',
+     *   timeout: 10000
+     * });
+     * ```
+     */
+    wait(waitData: Wait): Promise<WaitResult>;
+    /**
+     * Execute a clipboard action
+     *
+     * @param clipboardData - Clipboard configuration
+     * @returns Promise resolving to ActionResult
+     *
+     * @example
+     * ```typescript
+     * // Get clipboard content
+     * const result = await actionHandler.clipboard({ get: true });
+     *
+     * // Set clipboard content
+     * const result = await actionHandler.clipboard({
+     *   text: 'Hello',
+     *   paste: false
+     * });
+     *
+     * // Paste clipboard content
+     * const result = await actionHandler.clipboard({ paste: true });
+     * ```
+     */
+    clipboard(clipboardData: Clipboard): Promise<ClipboardResult>;
+    /**
+     * Take a screenshot
+     *
+     * @param screenshotData - Screenshot configuration
+     * @returns Promise resolving to ActionResult
+     *
+     * @example
+     * ```typescript
+     * const result = await actionHandler.takeScreenshot({
+     *   format: 'png',
+     *   quality: 100,
+     *   scale: 1,
+     *   saveToFile: true
+     * });
+     * ```
+     */
+    takeScreenshot(screenshotData?: Screenshot): Promise<ScreenshotResult>;
+    /**
+     * Get application info
+     *
+     * @param appInfoData - App info configuration
+     * @returns Promise resolving to ActionResult
+     *
+     * @example
+     * ```typescript
+     * const result = await actionHandler.getAppInfo({
+     *   package: 'com.example.app',
+     *   retry: 3
+     * });
+     * ```
+     */
+    getAppInfo(appInfoData: AppInfo): Promise<AppInfoResult>;
+    /**
+     * Get screen information
+     *
+     * @returns Promise resolving to ScreenInfoResponse
+     *
+     * @example
+     * ```typescript
+     * const screenInfo = await actionHandler.getScreenInfo();
+     * console.log(`Screen size: ${screenInfo.width}x${screenInfo.height}`);
+     * ```
+     */
+    getScreenInfo(): Promise<ScreenInfoResponse>;
+    /**
+     * Get list of installed applications
+     *
+     * @param options - Options for filtering app list
+     * @returns Promise resolving to app list response
+     *
+     * @example
+     * ```typescript
+     * const result = await actionHandler.getAppList({
+     *   includeSystemApps: false,
+     *   includeDisabledApps: false
+     * });
+     * console.log(`Found ${result.userAppCount} user apps`);
+     * ```
+     */
+    getAppList(options?: {
+        includeSystemApps?: boolean;
+        includeDisabledApps?: boolean;
+    }): Promise<AppListResult>;
+    /**
+     * Navigate through app using a series of text-based clicks
+     *
+     * @param textPath - Array of text strings to click in sequence
+     * @returns Promise resolving to boolean indicating success
+     *
+     * @example
+     * ```typescript
+     * const success = await actionHandler.navigateByPath([
+     *   'Settings',
+     *   'Accounts',
+     *   'Add Account'
+     * ]);
+     * ```
+     */
+    navigateByPath(textPath: string[]): Promise<boolean>;
+    /**
+     * Click element by text content
+     *
+     * @param text - Text content to search for
+     * @param exact - Whether to match exact text (default: false)
+     * @returns Promise resolving to boolean indicating success
+     *
+     * @example
+     * ```typescript
+     * // Click element containing text
+     * await actionHandler.clickByText('Submit');
+     *
+     * // Click element with exact text match
+     * await actionHandler.clickByText('Submit', true);
+     * ```
+     */
+    clickByText(text: string, exact?: boolean): Promise<boolean>;
+    /**
+     * Input text into a field
+     *
+     * @param selector - Element selector
+     * @param text - Text to input
+     * @param options - Additional options
+     * @returns Promise resolving to boolean indicating success
+     *
+     * @example
+     * ```typescript
+     * await actionHandler.inputText(
+     *   { resourceId: 'username_field' },
+     *   'john.doe',
+     *   { clear: true, hideKeyboard: false }
+     * );
+     * ```
+     */
+    inputText(selector: ElementSelector, text: string, options?: {
+        clear?: boolean;
+        hideKeyboard?: boolean;
+    }): Promise<boolean>;
+}
+export {};