tauri-plugin-android-accessibility-api 0.1.0 → 0.2.2

package/README.md

@@ -8,6 +8,9 @@ A mobile plugin based on Tauri v2 for Android accessibility bridging, providing
 - Jump to the system accessibility settings page
 - Get a UI tree snapshot of the current foreground window
 - Perform click/long press/focus by node ID (supports falling back to a clickable parent node)
+- Simulate gestures (tap, long press, swipe/drag, multi-touch)
+- Trigger global system actions (back, home, recents, notifications, etc.)
+- Perform generic node actions (scroll, focus navigation, selection)
 
 ## 1. Functional Description
 
@@ -26,6 +29,14 @@ Implementation entry points:
 
 ## 2. Usage in Tauri App
 
+Directly add:
+
+```bash
+bun tauri add android-accessibility
+```
+
+Or add it manually as follows：
+
 Register the plugin on the application side:
 
 ```rust
@@ -50,6 +61,9 @@ import {
         openAccessibilitySettings,
         getFrontmostUiTree,
         clickNode,
+        performGesture,
+        performGlobalAction,
+        performNodeAction,
 } from 'tauri-plugin-android-accessibility-api'
 
 const status = await checkAccessibilityEnabled()
@@ -68,49 +82,116 @@ await clickNode({
         action: 'click',
         fallbackToClickableParent: true,
 })
+
+await performGesture({
+        strokes: [
+                {
+                        points: [
+                                { x: 540, y: 1500 },
+                                { x: 540, y: 700 },
+                        ],
+                        durationMs: 320,
+                },
+        ],
+})
+
+await performGlobalAction({ action: 'back' })
+
+await performNodeAction({
+        nodeId: '0.1.0',
+        action: 'scrollForward',
+        fallbackToScrollableParent: true,
+})
 ```
 
 ## 3. API List
 
-### checkAccessibilityEnabled
+### `checkAccessibilityEnabled`
 
 Returns:
 
-- enabled: Whether the current app's accessibility service is enabled
-- serviceId: Current service component ID
-- enabledServices: List of accessibility services currently enabled by the system
+- `enabled`: Whether the current app's accessibility service is enabled
+- `serviceId`: Current service component ID
+- `enabledServices`: List of accessibility services currently enabled by the system
+
+### `openAccessibilitySettings`
+
+Opens the system accessibility settings page, returns `opened`.
+
+### `getFrontmostUiTree`
 
-### openAccessibilitySettings
+Parameters:
+
+- `maxDepth`: Tree depth limit
+- `maxChildrenPerNode`: Maximum number of child nodes per node
+- `includeNonClickable`: Whether to include non-clickable leaf nodes
+
+Returns:
+
+- `timestampMs`
+- `packageName`
+- `root` (recursive UI node)
+
+### `clickNode`
+
+Parameters:
+
+- `nodeId`: Node path ID in the UI tree (e.g., 0.1.2)
+- `action`: `click` | `longClick` | `focus`
+- `fallbackToClickableParent`: Whether to fall back to a parent node click if the target fails
+
+Returns:
+
+- `success`
+- `performedOnNodeId`
+- `message`
+
+### `performGesture`
+
+Parameters:
+
+- `strokes`: Array of gesture strokes
+- `strokes[].points`: Path points as `[{ x, y }, ...]`
+- `strokes[].startTimeMs`: Relative start time for this stroke (optional)
+- `strokes[].durationMs`: Duration of the stroke in milliseconds
+- `strokes[].willContinue`: Whether the stroke should continue (optional)
+
+Notes:
+
+- Tap: 1 point + short duration
+- Long press: 1 point + long duration
+- Swipe/drag: 2 or more points
+- Multi-touch: multiple strokes in one request
+
+Returns:
 
-Opens the system accessibility settings page, returns opened.
+- `success`
+- `message`
 
-### getFrontmostUiTree
+### `performGlobalAction`
 
 Parameters:
 
-- maxDepth: Tree depth limit
-- maxChildrenPerNode: Maximum number of child nodes per node
-- includeNonClickable: Whether to include non-clickable leaf nodes
+- `action`: `back` | `home` | `recents` | `notifications` | `quickSettings` | `powerDialog` | `lockScreen` | `takeScreenshot`
 
 Returns:
 
-- timestampMs
-- packageName
-- root (recursive UI node)
+- `success`
+- `message`
 
-### clickNode
+### `performNodeAction`
 
 Parameters:
 
-- nodeId: Node path ID in the UI tree (e.g., 0.1.2)
-- action: click | longClick | focus
-- fallbackToClickableParent: Whether to fall back to a parent node click if the target fails
+- `nodeId`: Node path ID in the UI tree
+- `action`: `click` | `longClick` | `focus` | `clearFocus` | `select` | `clearSelection` | `scrollForward` | `scrollBackward`
+- `fallbackToScrollableParent`: Whether to fallback to a scrollable parent for failed scroll actions
 
 Returns:
 
-- success
-- performedOnNodeId
-- message
+- `success`
+- `performedOnNodeId`
+- `message`
 
 ## 4. Android Specifications and Limitations
 

package/dist-js/index.cjs

@@ -25,9 +25,33 @@ async function clickNode(payload) {
         payload,
     });
 }
+async function performGesture(payload) {
+    return await core.invoke('plugin:android-accessibility|perform_gesture', {
+        payload,
+    });
+}
+async function performGlobalAction(payload) {
+    return await core.invoke('plugin:android-accessibility|perform_global_action', {
+        payload,
+    });
+}
+async function performNodeAction(payload) {
+    return await core.invoke('plugin:android-accessibility|perform_node_action', {
+        payload,
+    });
+}
+async function typeText(payload) {
+    return await core.invoke('plugin:android-accessibility|type_text', {
+        payload,
+    });
+}
 
 exports.checkAccessibilityEnabled = checkAccessibilityEnabled;
 exports.clickNode = clickNode;
 exports.getFrontmostUiTree = getFrontmostUiTree;
 exports.openAccessibilitySettings = openAccessibilitySettings;
+exports.performGesture = performGesture;
+exports.performGlobalAction = performGlobalAction;
+exports.performNodeAction = performNodeAction;
 exports.ping = ping;
+exports.typeText = typeText;

package/dist-js/index.d.ts

@@ -52,7 +52,55 @@ export interface ClickNodeResponse {
 export interface OpenSettingsResponse {
     opened: boolean;
 }
+export interface GesturePoint {
+    x: number;
+    y: number;
+}
+export interface GestureStroke {
+    points: GesturePoint[];
+    startTimeMs?: number;
+    durationMs: number;
+    willContinue?: boolean;
+}
+export interface PerformGestureRequest {
+    strokes: GestureStroke[];
+}
+export interface PerformGestureResponse {
+    success: boolean;
+    message?: string | null;
+}
+export type GlobalActionType = 'back' | 'home' | 'recents' | 'notifications' | 'quickSettings' | 'powerDialog' | 'lockScreen' | 'takeScreenshot';
+export interface GlobalActionRequest {
+    action: GlobalActionType;
+}
+export interface GlobalActionResponse {
+    success: boolean;
+    message?: string | null;
+}
+export type NodeActionType = 'click' | 'longClick' | 'focus' | 'clearFocus' | 'select' | 'clearSelection' | 'scrollForward' | 'scrollBackward';
+export interface NodeActionRequest {
+    nodeId: string;
+    action: NodeActionType;
+    fallbackToScrollableParent?: boolean;
+}
+export interface NodeActionResponse {
+    success: boolean;
+    performedOnNodeId?: string | null;
+    message?: string | null;
+}
+export interface TypeTextRequest {
+    nodeId: string;
+    text: string;
+}
+export interface TypeTextResponse {
+    success: boolean;
+    message?: string | null;
+}
 export declare function checkAccessibilityEnabled(): Promise<AccessibilityPermissionStatus>;
 export declare function openAccessibilitySettings(): Promise<OpenSettingsResponse>;
 export declare function getFrontmostUiTree(payload?: UiTreeRequest): Promise<UiTreeResponse>;
 export declare function clickNode(payload: ClickNodeRequest): Promise<ClickNodeResponse>;
+export declare function performGesture(payload: PerformGestureRequest): Promise<PerformGestureResponse>;
+export declare function performGlobalAction(payload: GlobalActionRequest): Promise<GlobalActionResponse>;
+export declare function performNodeAction(payload: NodeActionRequest): Promise<NodeActionResponse>;
+export declare function typeText(payload: TypeTextRequest): Promise<TypeTextResponse>;

package/dist-js/index.js

@@ -23,5 +23,25 @@ async function clickNode(payload) {
         payload,
     });
 }
+async function performGesture(payload) {
+    return await invoke('plugin:android-accessibility|perform_gesture', {
+        payload,
+    });
+}
+async function performGlobalAction(payload) {
+    return await invoke('plugin:android-accessibility|perform_global_action', {
+        payload,
+    });
+}
+async function performNodeAction(payload) {
+    return await invoke('plugin:android-accessibility|perform_node_action', {
+        payload,
+    });
+}
+async function typeText(payload) {
+    return await invoke('plugin:android-accessibility|type_text', {
+        payload,
+    });
+}
 
-export { checkAccessibilityEnabled, clickNode, getFrontmostUiTree, openAccessibilitySettings, ping };
+export { checkAccessibilityEnabled, clickNode, getFrontmostUiTree, openAccessibilitySettings, performGesture, performGlobalAction, performNodeAction, ping, typeText };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tauri-plugin-android-accessibility-api",
-  "version": "0.1.0",
+  "version": "0.2.2",
   "author": "ShizukuAqua",
   "description": "A Tauri plugin that supports Android accessibility features, allowing Tauri applications to interact with Android's accessibility services for enhanced functionality and user experience.",
   "license": "MIT",