tauri-plugin-android-accessibility-api 0.1.0

package/README.md

@@ -0,0 +1,127 @@
+# Tauri Plugin Android Accessibility
+
+[简体中文](./README_zh-cn.md)
+
+A mobile plugin based on Tauri v2 for Android accessibility bridging, providing the following capabilities:
+
+- Check if accessibility services are enabled
+- Jump to the system accessibility settings page
+- Get a UI tree snapshot of the current foreground window
+- Perform click/long press/focus by node ID (supports falling back to a clickable parent node)
+
+## 1. Functional Description
+
+This plugin is designed for Android and consists of a Kotlin accessibility service and a Rust/JS bridge.
+
+- Rust plugin name: android-accessibility
+- Kotlin plugin class: AndroidAccessibilityPlugin
+- Kotlin accessibility service: TauriAccessibilityService
+
+Implementation entry points:
+
+- Rust commands are defined in [src/commands.rs](src/commands.rs)
+- Mobile bridging is in [src/mobile.rs](src/mobile.rs)
+- Android native implementation is in [android/src/main/java/com/tauri/plugin/androidaccessibility/AndroidAccessibilityPlugin.kt](android/src/main/java/com/tauri/plugin/androidaccessibility/AndroidAccessibilityPlugin.kt)
+- Accessibility service is in [android/src/main/java/com/tauri/plugin/androidaccessibility/TauriAccessibilityService.kt](android/src/main/java/com/tauri/plugin/androidaccessibility/TauriAccessibilityService.kt)
+
+## 2. Usage in Tauri App
+
+Register the plugin on the application side:
+
+```rust
+tauri::Builder::default()
+        .plugin(tauri_plugin_android_accessibility::init())
+        .run(tauri::generate_context!())
+        .expect("error while running tauri application");
+```
+
+JS side (it is recommended to use bun for dependency installation and building):
+
+```bash
+bun install
+bun run build
+```
+
+Example call:
+
+```ts
+import {
+        checkAccessibilityEnabled,
+        openAccessibilitySettings,
+        getFrontmostUiTree,
+        clickNode,
+} from 'tauri-plugin-android-accessibility-api'
+
+const status = await checkAccessibilityEnabled()
+if (!status.enabled) {
+        await openAccessibilitySettings()
+}
+
+const tree = await getFrontmostUiTree({
+        maxDepth: 8,
+        maxChildrenPerNode: 40,
+        includeNonClickable: true,
+})
+
+await clickNode({
+        nodeId: '0.1.2',
+        action: 'click',
+        fallbackToClickableParent: true,
+})
+```
+
+## 3. API List
+
+### checkAccessibilityEnabled
+
+Returns:
+
+- enabled: Whether the current app's accessibility service is enabled
+- serviceId: Current service component ID
+- enabledServices: List of accessibility services currently enabled by the system
+
+### openAccessibilitySettings
+
+Opens the system accessibility settings page, returns opened.
+
+### getFrontmostUiTree
+
+Parameters:
+
+- maxDepth: Tree depth limit
+- maxChildrenPerNode: Maximum number of child nodes per node
+- includeNonClickable: Whether to include non-clickable leaf nodes
+
+Returns:
+
+- timestampMs
+- packageName
+- root (recursive UI node)
+
+### clickNode
+
+Parameters:
+
+- nodeId: Node path ID in the UI tree (e.g., 0.1.2)
+- action: click | longClick | focus
+- fallbackToClickableParent: Whether to fall back to a parent node click if the target fails
+
+Returns:
+
+- success
+- performedOnNodeId
+- message
+
+## 4. Android Specifications and Limitations
+
+- Accessibility services are highly sensitive capabilities and must be manually authorized by the user in the system settings.
+- This plugin does not attempt to bypass the system authorization process and will not silently enable accessibility.
+- Reading the foreground app's UI tree depends on the system's visible windows and Android version behavior; results may be empty.
+- Node clicks are affected by the target app's protection policies, dynamic UI states, and accessibility flags; 100% success is not guaranteed.
+- Before submitting to an app store, please clearly inform users of the collection scope and purpose, and follow the app market's privacy and accessibility policies.
+
+## 5. Permissions and Manifest
+
+- Accessibility service declaration is located in [android/src/main/AndroidManifest.xml](android/src/main/AndroidManifest.xml)
+- Service configuration is located in [android/src/main/res/xml/tauri_accessibility_service_config.xml](android/src/main/res/xml/tauri_accessibility_service_config.xml)
+- The default set of Tauri plugin command permissions is located in [permissions/default.toml](permissions/default.toml)

package/dist-js/index.cjs

@@ -0,0 +1,33 @@
+'use strict';
+
+var core = require('@tauri-apps/api/core');
+
+async function ping(value) {
+    return await core.invoke('plugin:android-accessibility|ping', {
+        payload: {
+            value,
+        },
+    }).then((r) => (r.value ? r.value : null));
+}
+async function checkAccessibilityEnabled() {
+    return await core.invoke('plugin:android-accessibility|check_accessibility_enabled');
+}
+async function openAccessibilitySettings() {
+    return await core.invoke('plugin:android-accessibility|open_accessibility_settings');
+}
+async function getFrontmostUiTree(payload = {}) {
+    return await core.invoke('plugin:android-accessibility|get_frontmost_ui_tree', {
+        payload,
+    });
+}
+async function clickNode(payload) {
+    return await core.invoke('plugin:android-accessibility|click_node', {
+        payload,
+    });
+}
+
+exports.checkAccessibilityEnabled = checkAccessibilityEnabled;
+exports.clickNode = clickNode;
+exports.getFrontmostUiTree = getFrontmostUiTree;
+exports.openAccessibilitySettings = openAccessibilitySettings;
+exports.ping = ping;

package/dist-js/index.d.ts

@@ -0,0 +1,58 @@
+export declare function ping(value: string): Promise<string | null>;
+export interface AccessibilityPermissionStatus {
+    enabled: boolean;
+    serviceId: string;
+    enabledServices: string[];
+}
+export interface Bounds {
+    left: number;
+    top: number;
+    right: number;
+    bottom: number;
+}
+export interface UiNode {
+    nodeId: string;
+    className?: string | null;
+    text?: string | null;
+    contentDescription?: string | null;
+    resourceId?: string | null;
+    packageName?: string | null;
+    bounds?: Bounds | null;
+    clickable: boolean;
+    enabled: boolean;
+    focusable: boolean;
+    focused: boolean;
+    scrollable: boolean;
+    selected: boolean;
+    checkable: boolean;
+    checked: boolean;
+    longClickable: boolean;
+    children: UiNode[];
+}
+export interface UiTreeRequest {
+    maxDepth?: number;
+    maxChildrenPerNode?: number;
+    includeNonClickable?: boolean;
+}
+export interface UiTreeResponse {
+    timestampMs: number;
+    packageName?: string | null;
+    root?: UiNode | null;
+}
+export interface ClickNodeRequest {
+    nodeId: string;
+    action?: 'click' | 'longClick' | 'focus';
+    fallbackToClickableParent?: boolean;
+}
+export interface ClickNodeResponse {
+    success: boolean;
+    performedOnNodeId?: string | null;
+    message?: string | null;
+}
+export interface OpenSettingsResponse {
+    opened: boolean;
+}
+export declare function checkAccessibilityEnabled(): Promise<AccessibilityPermissionStatus>;
+export declare function openAccessibilitySettings(): Promise<OpenSettingsResponse>;
+export declare function getFrontmostUiTree(payload?: UiTreeRequest): Promise<UiTreeResponse>;
+export declare function clickNode(payload: ClickNodeRequest): Promise<ClickNodeResponse>;

package/dist-js/index.js

@@ -0,0 +1,27 @@
+import { invoke } from '@tauri-apps/api/core';
+
+async function ping(value) {
+    return await invoke('plugin:android-accessibility|ping', {
+        payload: {
+            value,
+        },
+    }).then((r) => (r.value ? r.value : null));
+}
+async function checkAccessibilityEnabled() {
+    return await invoke('plugin:android-accessibility|check_accessibility_enabled');
+}
+async function openAccessibilitySettings() {
+    return await invoke('plugin:android-accessibility|open_accessibility_settings');
+}
+async function getFrontmostUiTree(payload = {}) {
+    return await invoke('plugin:android-accessibility|get_frontmost_ui_tree', {
+        payload,
+    });
+}
+async function clickNode(payload) {
+    return await invoke('plugin:android-accessibility|click_node', {
+        payload,
+    });
+}
+
+export { checkAccessibilityEnabled, clickNode, getFrontmostUiTree, openAccessibilitySettings, ping };

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "tauri-plugin-android-accessibility-api",
+  "version": "0.1.0",
+  "author": "ShizukuAqua",
+  "description": "A Tauri plugin that supports Android accessibility features, allowing Tauri applications to interact with Android's accessibility services for enhanced functionality and user experience.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zhangnjsd/tauri-plugin-android-accessibility.git"
+  },
+  "bugs": {
+    "url": "https://github.com/zhangnjsd/tauri-plugin-android-accessibility/issues"
+  },
+  "homepage": "https://github.com/zhangnjsd/tauri-plugin-android-accessibility#readme",
+  "keywords": [
+    "tauri",
+    "tauri-plugin",
+    "android",
+    "accessibility"
+  ],
+  "type": "module",
+  "types": "./dist-js/index.d.ts",
+  "main": "./dist-js/index.cjs",
+  "module": "./dist-js/index.js",
+  "exports": {
+    "types": "./dist-js/index.d.ts",
+    "import": "./dist-js/index.js",
+    "require": "./dist-js/index.cjs"
+  },
+  "files": [
+    "dist-js",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "rollup -c",
+    "prepublishOnly": "bun run build",
+    "pretest": "bun run build"
+  },
+  "dependencies": {
+    "@tauri-apps/api": "^2.0.0"
+  },
+  "devDependencies": {
+    "@rollup/plugin-typescript": "^12.0.0",
+    "rollup": "^4.9.6",
+    "typescript": "^5.3.3",
+    "tslib": "^2.6.2"
+  }
+}