shellx-ai 1.0.12 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +666 -0
- package/dist/automation/element-finder.d.ts +189 -0
- package/dist/automation/element-finder.js +322 -0
- package/dist/automation/element-finder.js.map +1 -0
- package/dist/automation/ui-action-handler.d.ts +330 -0
- package/dist/automation/ui-action-handler.js +873 -0
- package/dist/automation/ui-action-handler.js.map +1 -0
- package/dist/cbor-compat.d.ts +27 -0
- package/dist/cbor-compat.js +111 -0
- package/dist/cbor-compat.js.map +1 -0
- package/dist/domain-manager.d.ts +80 -0
- package/dist/domain-manager.js +161 -0
- package/dist/domain-manager.js.map +1 -0
- package/dist/error-handler.d.ts +87 -0
- package/dist/error-handler.js +151 -0
- package/dist/error-handler.js.map +1 -0
- package/dist/errors.d.ts +114 -0
- package/dist/errors.js +139 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +163 -54
- package/dist/index.js +678 -481
- package/dist/index.js.map +1 -0
- package/dist/logger.d.ts +81 -0
- package/dist/logger.js +128 -0
- package/dist/logger.js.map +1 -0
- package/dist/protocol.d.ts +147 -31
- package/dist/protocol.js +2 -2
- package/dist/protocol.js.map +1 -0
- package/dist/shell/output-buffer.d.ts +152 -0
- package/dist/shell/output-buffer.js +176 -0
- package/dist/shell/output-buffer.js.map +1 -0
- package/dist/shell/shell-command-executor.d.ts +182 -0
- package/dist/shell/shell-command-executor.js +404 -0
- package/dist/shell/shell-command-executor.js.map +1 -0
- package/dist/shellx.d.ts +681 -178
- package/dist/shellx.js +762 -1159
- package/dist/shellx.js.map +1 -0
- package/dist/types.d.ts +132 -57
- package/dist/types.js +4 -4
- package/dist/types.js.map +1 -0
- package/dist/utils/retry-helper.d.ts +73 -0
- package/dist/utils/retry-helper.js +95 -0
- package/dist/utils/retry-helper.js.map +1 -0
- package/dist/utils.d.ts +3 -3
- package/dist/utils.js +20 -23
- package/dist/utils.js.map +1 -0
- package/package.json +95 -62
package/dist/index.js
CHANGED
|
@@ -1,52 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
-
});
|
|
43
|
-
};
|
|
44
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
-
exports.ShellX = exports.createShellXWithShellMonitoring = exports.ConnectionTaskClient = exports.DEFAULT_CONFIG = void 0;
|
|
46
|
-
const uuid_1 = require("uuid");
|
|
47
|
-
const utils_1 = require("./utils");
|
|
48
|
-
// 定义默认配置
|
|
49
|
-
exports.DEFAULT_CONFIG = {
|
|
1
|
+
import { v4 as uuidv4 } from "uuid";
|
|
2
|
+
import { wait } from "./utils.js";
|
|
3
|
+
import { buildDeviceApiUrl } from "./domain-manager.js";
|
|
4
|
+
import { createLogger } from "./logger.js";
|
|
5
|
+
// Create logger for ConnectionClient
|
|
6
|
+
const logger = createLogger("ConnectionClient");
|
|
7
|
+
// Execution log store for debugging
|
|
8
|
+
const executionLogStore = [];
|
|
9
|
+
// Define default configuration
|
|
10
|
+
export const DEFAULT_CONFIG = {
|
|
50
11
|
timeout: 5000,
|
|
51
12
|
reconnect: true,
|
|
52
13
|
reconnectMaxAttempts: 5,
|
|
@@ -57,230 +18,316 @@ exports.DEFAULT_CONFIG = {
|
|
|
57
18
|
onError: () => { },
|
|
58
19
|
onReconnectFailed: () => { },
|
|
59
20
|
};
|
|
60
|
-
|
|
61
|
-
// 安全地获取环境变量,兼容浏览器和Node.js环境
|
|
62
|
-
let authKey = (0, utils_1.getEnvVar)('SHELLX_AUTH_KEY');
|
|
21
|
+
import { initCbor, cborEncode, cborDecode } from "./cbor-compat.js";
|
|
63
22
|
/**
|
|
64
|
-
*
|
|
23
|
+
* Get WebSocket constructor for current environment
|
|
65
24
|
*/
|
|
66
|
-
function getWebSocket() {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
});
|
|
25
|
+
async function getWebSocket() {
|
|
26
|
+
// Check if global WebSocket exists (browser environment or Node.js 20+)
|
|
27
|
+
if (typeof globalThis.WebSocket !== "undefined") {
|
|
28
|
+
return globalThis.WebSocket;
|
|
29
|
+
}
|
|
30
|
+
// Dynamically import ws module in Node.js environment
|
|
31
|
+
try {
|
|
32
|
+
const wsModule = await import("ws");
|
|
33
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
|
|
34
|
+
return (wsModule.default || wsModule);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
logger.error("❌ [WebSocket] ws module is not available, please install: npm install ws");
|
|
38
|
+
throw new Error("WebSocket not available. Please install ws module: npm install ws");
|
|
39
|
+
}
|
|
82
40
|
}
|
|
83
41
|
function isPendingTask(taskType) {
|
|
84
|
-
return taskType && taskType !== "command";
|
|
42
|
+
return taskType && taskType !== "command" && taskType !== "oneway";
|
|
85
43
|
}
|
|
86
44
|
/**
|
|
87
|
-
*
|
|
45
|
+
* Low-level WebSocket Client for direct protocol communication
|
|
46
|
+
*
|
|
47
|
+
* @warning This is a low-level API intended for advanced use cases only.
|
|
48
|
+
* Most users should use the ShellX class instead, which provides a simpler,
|
|
49
|
+
* more intuitive interface with automatic error handling and retry logic.
|
|
50
|
+
*
|
|
51
|
+
* @example
|
|
52
|
+
* ```typescript
|
|
53
|
+
* // For most users, use ShellX instead:
|
|
54
|
+
* import { ShellX } from './shellx';
|
|
55
|
+
* const shellx = new ShellX({ deviceId: 'your-device-id' });
|
|
56
|
+
* await shellx.connect();
|
|
57
|
+
* await shellx.click({ text: 'Submit' });
|
|
58
|
+
* ```
|
|
59
|
+
*
|
|
60
|
+
* @see Use ShellX for a high-level API unless you need low-level WebSocket access
|
|
88
61
|
*/
|
|
89
|
-
class
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
62
|
+
export class ConnectionClient {
|
|
63
|
+
wsUrl;
|
|
64
|
+
deviceId;
|
|
65
|
+
config;
|
|
66
|
+
ws = null;
|
|
67
|
+
shellxConnected = false; // ShellX.ai connection status
|
|
68
|
+
wsConnected = false; // WebSocket connection status
|
|
69
|
+
authenticated = false; // Authentication status
|
|
70
|
+
pendingTasks = new Map();
|
|
71
|
+
messageQueue = [];
|
|
72
|
+
reconnectAttempts = 0;
|
|
73
|
+
pingIntervalId = null;
|
|
74
|
+
initializationPromise = null;
|
|
75
|
+
shellx = null; // Associated ShellX instance
|
|
76
|
+
executionLogsRef = executionLogStore; // Current execution log
|
|
77
|
+
logger;
|
|
78
|
+
constructor(deviceId, config = {}, logger) {
|
|
101
79
|
this.deviceId = deviceId;
|
|
102
|
-
this.config =
|
|
80
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
81
|
+
// Use provided logger or create default one
|
|
82
|
+
this.logger = logger || createLogger("ConnectionClient");
|
|
103
83
|
this.initializationPromise = this.init();
|
|
104
84
|
}
|
|
105
|
-
init() {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
if (!
|
|
85
|
+
async init() {
|
|
86
|
+
await initCbor(); // load the right CBOR codec for this environment
|
|
87
|
+
return new Promise((resolve, reject) => {
|
|
88
|
+
this.authenticateDevice()
|
|
89
|
+
.then((wsUrl) => {
|
|
90
|
+
if (!wsUrl) {
|
|
111
91
|
this.config.onError(new Event("connection"));
|
|
92
|
+
void this.reconnect();
|
|
93
|
+
reject(new Error("Failed to authenticate device"));
|
|
112
94
|
return;
|
|
113
95
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
const
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
96
|
+
// ✅ Store the WebSocket URL for later use
|
|
97
|
+
this.wsUrl = wsUrl;
|
|
98
|
+
const separator = wsUrl.includes("?") ? "&" : "?";
|
|
99
|
+
const wsUrlWithSessionId = `${wsUrl}${separator}session_id=${this.deviceId}`;
|
|
100
|
+
this.logger.info("Initializing ShellX client wsUrl:" + wsUrl + " deviceId: " + this.deviceId);
|
|
101
|
+
this.logger.info("WebSocket URL with session_id: " + wsUrlWithSessionId);
|
|
102
|
+
// Get WebSocket constructor for current environment
|
|
103
|
+
getWebSocket()
|
|
104
|
+
.then((WebSocketConstructor) => {
|
|
105
|
+
try {
|
|
106
|
+
this.ws = new WebSocketConstructor(wsUrlWithSessionId);
|
|
107
|
+
// Set binary type (if supported)
|
|
108
|
+
if (this.ws.binaryType !== undefined) {
|
|
109
|
+
this.ws.binaryType = "arraybuffer";
|
|
110
|
+
}
|
|
111
|
+
this.ws.onopen = () => {
|
|
112
|
+
this.logger.info("🔗 [ShellX] WebSocket connection established:" + this.wsUrl);
|
|
113
|
+
this.wsConnected = true;
|
|
114
|
+
this.reconnectAttempts = 0;
|
|
115
|
+
// Mark as connected after WebSocket open
|
|
116
|
+
this.shellxConnected = true;
|
|
117
|
+
this.config.onOpen?.(this.deviceId);
|
|
118
|
+
void this.flushQueue();
|
|
119
|
+
void this.startPing();
|
|
120
|
+
// Resolve initialization promise after WebSocket is connected
|
|
121
|
+
resolve();
|
|
122
|
+
// Check if we're in Android sandbox environment
|
|
123
|
+
const isAndroidSandbox = process.env["SANDBOX"] === "android";
|
|
124
|
+
if (isAndroidSandbox) {
|
|
125
|
+
void this.getScreenInfo();
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
void this.getAndWakeScreen();
|
|
129
|
+
}
|
|
130
|
+
};
|
|
131
|
+
this.ws.onmessage = (event) => {
|
|
132
|
+
void this.handleMessage(event);
|
|
133
|
+
};
|
|
134
|
+
this.ws.onclose = (event) => {
|
|
135
|
+
this.logger.info("❌ [ShellX] WebSocket connection closed, attempting to reconnect..." +
|
|
136
|
+
event.code +
|
|
137
|
+
" " +
|
|
138
|
+
this.wsUrl);
|
|
139
|
+
this.wsConnected = false;
|
|
140
|
+
this.shellxConnected = false;
|
|
141
|
+
this.authenticated = false;
|
|
142
|
+
this.config.onClose?.(event);
|
|
143
|
+
this.stopPing();
|
|
144
|
+
void this.reconnect();
|
|
145
|
+
};
|
|
146
|
+
this.ws.onerror = (error) => {
|
|
147
|
+
this.logger.error("❌ [ShellX] WebSocket error:", error);
|
|
148
|
+
this.config.onError?.(error);
|
|
149
|
+
reject(new Error("WebSocket connection failed"));
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
this.logger.error("❌ [WebSocket] Initialization failed:", error);
|
|
154
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
155
|
+
this.config.onError?.(err);
|
|
156
|
+
reject(err);
|
|
157
|
+
}
|
|
158
|
+
})
|
|
159
|
+
.catch((error) => {
|
|
160
|
+
this.logger.error("❌ [WebSocket] Failed to get constructor:", error);
|
|
161
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
162
|
+
reject(err);
|
|
163
|
+
});
|
|
164
|
+
})
|
|
165
|
+
.catch((error) => {
|
|
166
|
+
this.logger.error("❌ [Auth] Device authentication failed:", error);
|
|
167
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
168
|
+
reject(err);
|
|
169
|
+
});
|
|
155
170
|
});
|
|
156
171
|
}
|
|
157
|
-
authenticateDevice(
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
if (
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
}
|
|
172
|
+
async authenticateDevice() {
|
|
173
|
+
let fallbackUrl = undefined;
|
|
174
|
+
// 1. Prioritize local service detection
|
|
175
|
+
try {
|
|
176
|
+
// fetch timeout implementation
|
|
177
|
+
const fetchWithTimeout = (url, options, timeout = 1000) => Promise.race([
|
|
178
|
+
fetch(url, options),
|
|
179
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), timeout)),
|
|
180
|
+
]);
|
|
181
|
+
const localResp = await fetchWithTimeout("http://127.0.0.1:9091/info", { method: "GET", headers: { Accept: "application/json" } }, 1000);
|
|
182
|
+
if (localResp.ok) {
|
|
183
|
+
const info = (await localResp.json());
|
|
184
|
+
if (info && (info.status === "ok" || info.status === 1) && info.uuid) {
|
|
185
|
+
if (this.deviceId === undefined || this.deviceId === info.uuid) {
|
|
186
|
+
const localUuid = info.uuid;
|
|
187
|
+
this.deviceId = localUuid;
|
|
188
|
+
fallbackUrl = `ws://127.0.0.1:9091/api/s/${localUuid}`;
|
|
189
|
+
this.logger.info("✅ [Auth] Local ShellX service available, using local /info returned uuid:", localUuid, ", Service address:", fallbackUrl);
|
|
190
|
+
return fallbackUrl;
|
|
177
191
|
}
|
|
178
192
|
}
|
|
179
193
|
}
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
console.error('❌ [Auth] ShellX.ai设备未注册或认证信息无效');
|
|
207
|
-
return `wss://shellx.ai/api/s/${deviceId}`;
|
|
208
|
-
}
|
|
209
|
-
// const jsonData = JSON.parse(data);
|
|
210
|
-
console.log('✅ [Auth] ShellX.ai设备认证成功');
|
|
211
|
-
console.log(`📡 [Auth] 设备ID: ${jsonData.authenticate}`);
|
|
212
|
-
console.log(`📡 [Auth] ShellX.ai服务地址: ${jsonData.machine}`);
|
|
213
|
-
console.log(`📡 [Auth] 注册时间: ${jsonData.registered_at}`);
|
|
214
|
-
console.log(`📡 [Auth] 最后更新: ${jsonData.last_updated}`);
|
|
215
|
-
return jsonData.machine + '/api/s/' + jsonData.authenticate;
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
// Local unavailable, proceeding with remote
|
|
197
|
+
}
|
|
198
|
+
if (this.deviceId === undefined) {
|
|
199
|
+
this.logger.warn("❌ [Auth] Device ID not set, local USB not connected, please set environment variable SHELLX_DEVICE_ID or connect USB device");
|
|
200
|
+
return undefined;
|
|
201
|
+
}
|
|
202
|
+
// 2. Remote authentication logic
|
|
203
|
+
try {
|
|
204
|
+
this.logger.info("🔑 [Auth] Authenticating device...");
|
|
205
|
+
const featGlobal = this.getFetch();
|
|
206
|
+
const deviceApiUrl = buildDeviceApiUrl(this.deviceId);
|
|
207
|
+
const jsonData = (await featGlobal(deviceApiUrl, {
|
|
208
|
+
method: "GET",
|
|
209
|
+
headers: {
|
|
210
|
+
"Content-Type": "application/json",
|
|
211
|
+
},
|
|
212
|
+
}));
|
|
213
|
+
this.logger.info("ShellX.ai device authentication response:", jsonData);
|
|
214
|
+
// Check if device is not registered
|
|
215
|
+
if (jsonData?.status === "not_registered") {
|
|
216
|
+
this.logger.error("❌ [Auth] Device not registered");
|
|
217
|
+
this.logger.error(`📝 [Auth] Status: ${jsonData.message || "Device not registered"}`);
|
|
218
|
+
this.logger.error("💡 [Auth] Please register this device on ShellX.ai platform first");
|
|
219
|
+
throw new Error(`Device "${this.deviceId}" not registered on ShellX.ai platform. Please visit https://shellx.ai to register the device first.`);
|
|
216
220
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
+
// Verify authentication response is valid
|
|
222
|
+
// If interface returns null or missing required fields, device is not registered
|
|
223
|
+
if (!jsonData || jsonData === null || !jsonData.machine || !jsonData.authenticate) {
|
|
224
|
+
this.logger.error("❌ [Auth] ShellX.ai device not registered or invalid authentication information");
|
|
225
|
+
throw new Error(`Device "${this.deviceId}" has invalid authentication information. Please check device ID or re-register on ShellX.ai platform.`);
|
|
221
226
|
}
|
|
222
|
-
|
|
227
|
+
// const jsonData = JSON.parse(data);
|
|
228
|
+
this.logger.info("✅ [Auth] ShellX.ai device authentication successful");
|
|
229
|
+
this.logger.info(`📡 [Auth] Device ID: ${jsonData.authenticate}`);
|
|
230
|
+
this.logger.info(`📡 [Auth] ShellX.ai service address: ${jsonData.machine}`);
|
|
231
|
+
this.logger.info(`📡 [Auth] Registration time: ${jsonData.registered_at}`);
|
|
232
|
+
this.logger.info(`📡 [Auth] Last update: ${jsonData.last_updated}`);
|
|
233
|
+
this.logger.info(`📡 [Auth] Device status: ${jsonData.status || "registered"}`);
|
|
234
|
+
// UUID is the deviceId, no need to extract from URL
|
|
235
|
+
const wsUrl = `${jsonData.machine}/api/s/${this.deviceId}`;
|
|
236
|
+
this.logger.info(`🔗 [Auth] WebSocket URL: ${wsUrl}`);
|
|
237
|
+
return wsUrl;
|
|
238
|
+
}
|
|
239
|
+
catch (error) {
|
|
240
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
241
|
+
this.logger.warn("⚠️ [Auth] Online authentication failed, using local service:", JSON.stringify(errorMessage));
|
|
242
|
+
return fallbackUrl;
|
|
243
|
+
}
|
|
223
244
|
}
|
|
224
245
|
getFetch() {
|
|
225
|
-
return (url, options) =>
|
|
226
|
-
//
|
|
227
|
-
if (typeof globalThis.fetch !==
|
|
228
|
-
|
|
229
|
-
const response =
|
|
230
|
-
|
|
246
|
+
return async (url, options) => {
|
|
247
|
+
// Try using global fetch
|
|
248
|
+
if (typeof globalThis.fetch !== "undefined") {
|
|
249
|
+
this.logger.info(`🌐 [Fetch] Request: ${options?.method || "GET"} ${url}`);
|
|
250
|
+
const response = await globalThis.fetch(url, options);
|
|
251
|
+
this.logger.info(`📡 [Fetch] Response: ${response.status} ${response.statusText}`);
|
|
231
252
|
if (!response.ok) {
|
|
232
253
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
233
254
|
}
|
|
234
255
|
return response.json();
|
|
235
256
|
}
|
|
236
257
|
try {
|
|
237
|
-
const { default: fetch } =
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
258
|
+
const { default: fetch } = await import("node-fetch");
|
|
259
|
+
this.logger.info(`🌐 [Fetch] Request: ${options?.method || "GET"} ${url}`);
|
|
260
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
261
|
+
const response = await fetch(url, options);
|
|
262
|
+
this.logger.info(`📡 [Fetch] Response: ${response.status} ${response.statusText}`);
|
|
241
263
|
if (!response.ok) {
|
|
242
264
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
243
265
|
}
|
|
244
266
|
return response.json();
|
|
245
267
|
}
|
|
246
268
|
catch (fetchError) {
|
|
247
|
-
|
|
269
|
+
const errorMessage = fetchError instanceof Error ? fetchError.message : String(fetchError);
|
|
270
|
+
throw new Error(`Fetch not available: ${errorMessage}`);
|
|
248
271
|
}
|
|
249
|
-
}
|
|
272
|
+
};
|
|
250
273
|
}
|
|
251
274
|
/**
|
|
252
|
-
*
|
|
275
|
+
* Wait for WebSocket initialization complete
|
|
253
276
|
*/
|
|
254
|
-
waitForInitialization() {
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
}
|
|
259
|
-
});
|
|
277
|
+
async waitForInitialization() {
|
|
278
|
+
if (this.initializationPromise) {
|
|
279
|
+
await this.initializationPromise;
|
|
280
|
+
}
|
|
260
281
|
}
|
|
261
282
|
/**
|
|
262
|
-
*
|
|
283
|
+
* Ensure connection is ready before executing device commands
|
|
284
|
+
* If not connected, wait for connection with timeout
|
|
285
|
+
*
|
|
286
|
+
* @param timeout - Timeout in milliseconds (default: 10000ms)
|
|
287
|
+
* @throws Error if connection timeout or connection failed
|
|
288
|
+
*
|
|
289
|
+
* @example
|
|
290
|
+
* ```typescript
|
|
291
|
+
* await client.ensureConnected(10000);
|
|
292
|
+
* console.log('Connection ready');
|
|
293
|
+
* ```
|
|
263
294
|
*/
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
295
|
+
async ensureConnected(timeout = 10000) {
|
|
296
|
+
// If already connected, return immediately
|
|
297
|
+
if (this.shellxConnected && this.wsConnected) {
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
this.logger.info(`⏳ [ConnectionClient] Waiting for connection to be ready (timeout: ${timeout}ms)...`);
|
|
301
|
+
const startTime = Date.now();
|
|
302
|
+
const checkInterval = 100; // Check every 100ms
|
|
303
|
+
return new Promise((resolve, reject) => {
|
|
304
|
+
const intervalId = setInterval(() => {
|
|
305
|
+
const elapsed = Date.now() - startTime;
|
|
306
|
+
// Check if connected
|
|
307
|
+
if (this.shellxConnected && this.wsConnected) {
|
|
308
|
+
clearInterval(intervalId);
|
|
309
|
+
this.logger.info(`✅ [ConnectionClient] Connection ready (took ${elapsed}ms)`);
|
|
310
|
+
resolve();
|
|
311
|
+
return;
|
|
271
312
|
}
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
313
|
+
// Check timeout
|
|
314
|
+
if (elapsed >= timeout) {
|
|
315
|
+
clearInterval(intervalId);
|
|
316
|
+
const errorMsg = `Connection timeout after ${timeout}ms. ` +
|
|
317
|
+
`Device "${this.deviceId}" is not connected. ` +
|
|
318
|
+
`Please check: ` +
|
|
319
|
+
`1. Device is online and registered on ShellX.ai ` +
|
|
320
|
+
`2. Network connection is stable ` +
|
|
321
|
+
`3. Device ID is correct`;
|
|
322
|
+
this.logger.error(`❌ [ConnectionClient] ${errorMsg}`);
|
|
323
|
+
reject(new Error(errorMsg));
|
|
324
|
+
return;
|
|
276
325
|
}
|
|
277
|
-
|
|
278
|
-
|
|
326
|
+
// Log progress every 2 seconds
|
|
327
|
+
if (elapsed > 0 && elapsed % 2000 < checkInterval) {
|
|
328
|
+
this.logger.info(`⏳ [ConnectionClient] Still waiting for connection... (${elapsed}/${timeout}ms)`);
|
|
279
329
|
}
|
|
280
|
-
}
|
|
281
|
-
catch (error) {
|
|
282
|
-
console.error('❌ [Auth] 发送认证消息失败:', error);
|
|
283
|
-
}
|
|
330
|
+
}, checkInterval);
|
|
284
331
|
});
|
|
285
332
|
}
|
|
286
333
|
handleMessage(event) {
|
|
@@ -300,11 +347,11 @@ class ConnectionTaskClient {
|
|
|
300
347
|
this.processServerMessage(serverMessage);
|
|
301
348
|
return;
|
|
302
349
|
}
|
|
303
|
-
serverMessage = (
|
|
350
|
+
serverMessage = cborDecode(binaryData);
|
|
304
351
|
this.processServerMessage(serverMessage);
|
|
305
352
|
}
|
|
306
353
|
catch (cborError) {
|
|
307
|
-
|
|
354
|
+
this.logger.info("CBOR decode failed, trying JSON fallback:", cborError);
|
|
308
355
|
try {
|
|
309
356
|
let textData;
|
|
310
357
|
if (event.data instanceof ArrayBuffer) {
|
|
@@ -317,316 +364,358 @@ class ConnectionTaskClient {
|
|
|
317
364
|
this.processServerMessage(serverMessage);
|
|
318
365
|
}
|
|
319
366
|
catch (jsonError) {
|
|
320
|
-
|
|
367
|
+
this.logger.error("Failed to parse message:", { cborError, jsonError });
|
|
321
368
|
}
|
|
322
369
|
}
|
|
323
370
|
}
|
|
324
371
|
processServerMessage(message) {
|
|
325
|
-
|
|
326
|
-
//console.log('📨 [ShellX] 收到服务器消息:', message);
|
|
327
|
-
this.authenticated = true;
|
|
328
|
-
// 只有在认证成功后才处理其他消息并认为连接成功
|
|
329
|
-
if (this.authenticated && !this.shellxConnected) {
|
|
330
|
-
this.shellxConnected = true;
|
|
331
|
-
console.log('✅ [ShellX] ShellX.ai服务连接成功!');
|
|
332
|
-
(_b = (_a = this.config).onOpen) === null || _b === void 0 ? void 0 : _b.call(_a);
|
|
333
|
-
this.flushQueue();
|
|
334
|
-
this.startPing();
|
|
335
|
-
}
|
|
372
|
+
//console.log('📨 [ShellX] Received server message:', message);
|
|
336
373
|
// Call user-defined message handler
|
|
337
|
-
|
|
374
|
+
this.config.onMessage?.(message);
|
|
338
375
|
// Handle specific message types and resolve pending tasks
|
|
339
376
|
if (message.jsonData) {
|
|
340
377
|
this.handleJsonDataResponse(message.jsonData);
|
|
341
378
|
}
|
|
342
379
|
// Handle other server message types (hello, users, shells, etc.)
|
|
343
380
|
if (message.hello !== undefined) {
|
|
344
|
-
|
|
381
|
+
this.logger.info("👋 [ShellX] Server greeting, user ID:", message.hello);
|
|
345
382
|
}
|
|
346
383
|
if (message.pong !== undefined) {
|
|
347
|
-
//console.log('🏓 [ShellX]
|
|
384
|
+
//console.log('🏓 [ShellX] Heartbeat response:', message.pong);
|
|
348
385
|
}
|
|
349
386
|
if (message.error) {
|
|
350
|
-
|
|
387
|
+
this.logger.error("❌ [ShellX] Server error:", message.error);
|
|
351
388
|
}
|
|
352
389
|
}
|
|
390
|
+
matchResponseByType(jsonData, taskType) {
|
|
391
|
+
switch (taskType) {
|
|
392
|
+
case "findElement":
|
|
393
|
+
if (jsonData.findElement) {
|
|
394
|
+
return jsonData.findElement;
|
|
395
|
+
}
|
|
396
|
+
break;
|
|
397
|
+
case "waitElement":
|
|
398
|
+
if (jsonData.waitElement) {
|
|
399
|
+
return jsonData.waitElement;
|
|
400
|
+
}
|
|
401
|
+
break;
|
|
402
|
+
case "screenShot":
|
|
403
|
+
if (jsonData.screenShot) {
|
|
404
|
+
return jsonData.screenShot;
|
|
405
|
+
}
|
|
406
|
+
break;
|
|
407
|
+
case "screenInfo":
|
|
408
|
+
if (jsonData.screenInfo) {
|
|
409
|
+
return jsonData.screenInfo;
|
|
410
|
+
}
|
|
411
|
+
break;
|
|
412
|
+
case "appList":
|
|
413
|
+
logger.debug("appList:", jsonData);
|
|
414
|
+
if (jsonData.appList) {
|
|
415
|
+
return jsonData.appList;
|
|
416
|
+
}
|
|
417
|
+
break;
|
|
418
|
+
case "action":
|
|
419
|
+
if (jsonData.action_event) {
|
|
420
|
+
return jsonData.action_event;
|
|
421
|
+
}
|
|
422
|
+
break;
|
|
423
|
+
case "promptflowx":
|
|
424
|
+
if (jsonData.promptflowx) {
|
|
425
|
+
return jsonData.promptflowx;
|
|
426
|
+
}
|
|
427
|
+
break;
|
|
428
|
+
case "clipboard":
|
|
429
|
+
if (jsonData.clipboard) {
|
|
430
|
+
return jsonData.clipboard;
|
|
431
|
+
}
|
|
432
|
+
break;
|
|
433
|
+
case "asrControl":
|
|
434
|
+
// ASR control doesn't expect a specific response
|
|
435
|
+
return { success: true };
|
|
436
|
+
}
|
|
437
|
+
return undefined;
|
|
438
|
+
}
|
|
353
439
|
handleJsonDataResponse(jsonData) {
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
break;
|
|
365
|
-
case 'waitElement':
|
|
366
|
-
if (jsonData.waitElement) {
|
|
367
|
-
responseData = jsonData.waitElement;
|
|
368
|
-
shouldResolve = true;
|
|
369
|
-
}
|
|
370
|
-
break;
|
|
371
|
-
case 'screenShot':
|
|
372
|
-
if (jsonData.screenShot) {
|
|
373
|
-
responseData = jsonData.screenShot;
|
|
374
|
-
shouldResolve = true;
|
|
375
|
-
}
|
|
376
|
-
break;
|
|
377
|
-
case 'screenInfo':
|
|
378
|
-
if (jsonData.screenInfo) {
|
|
379
|
-
responseData = jsonData.screenInfo;
|
|
380
|
-
shouldResolve = true;
|
|
440
|
+
const responseTaskId = jsonData?.taskId;
|
|
441
|
+
if (responseTaskId) {
|
|
442
|
+
const task = this.pendingTasks.get(responseTaskId);
|
|
443
|
+
if (task) {
|
|
444
|
+
const responseData = this.matchResponseByType(jsonData, task.type);
|
|
445
|
+
if (responseData !== undefined) {
|
|
446
|
+
clearTimeout(task.timer);
|
|
447
|
+
this.pendingTasks.delete(responseTaskId);
|
|
448
|
+
if (task.type != "findElement") {
|
|
449
|
+
this.logger.info("✅ [ShellX] Response processing complete:", responseTaskId, jsonData);
|
|
381
450
|
}
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
responseData = jsonData.action_event;
|
|
392
|
-
shouldResolve = true;
|
|
393
|
-
}
|
|
394
|
-
break;
|
|
395
|
-
case 'actions':
|
|
396
|
-
if (jsonData.actions) {
|
|
397
|
-
responseData = jsonData.actions;
|
|
398
|
-
shouldResolve = true;
|
|
399
|
-
}
|
|
400
|
-
break;
|
|
451
|
+
task.resolve(responseData);
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
else {
|
|
455
|
+
this.logger.warn(`⚠️ [ShellX] Task ${responseTaskId} (type: ${task.type}) found but no matching response data in:`, jsonData);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
else {
|
|
459
|
+
this.logger.warn(`⚠️ [ShellX] Received taskId ${responseTaskId} but no pending task found`);
|
|
401
460
|
}
|
|
402
|
-
|
|
461
|
+
}
|
|
462
|
+
else {
|
|
463
|
+
this.logger.debug(`📨 [ShellX] Received response without taskId, trying fallback match:`, jsonData);
|
|
464
|
+
}
|
|
465
|
+
// Fallback to legacy matching when taskId is missing or not tracked
|
|
466
|
+
for (const [taskId, task] of this.pendingTasks.entries()) {
|
|
467
|
+
const responseData = this.matchResponseByType(jsonData, task.type);
|
|
468
|
+
if (responseData !== undefined) {
|
|
403
469
|
clearTimeout(task.timer);
|
|
404
470
|
this.pendingTasks.delete(taskId);
|
|
405
|
-
|
|
471
|
+
this.logger.info(`✅ [ShellX] Fallback match: completed task ${taskId} (type: ${task.type})`);
|
|
406
472
|
task.resolve(responseData);
|
|
407
|
-
break;
|
|
473
|
+
break;
|
|
408
474
|
}
|
|
409
475
|
}
|
|
410
476
|
}
|
|
411
|
-
sendMessage(message, taskType
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
this.pendingTasks.delete(taskId);
|
|
418
|
-
// 超时时返回 undefined 而不是 reject
|
|
419
|
-
console.log(`⏰ [ShellX] 任务超时: ${taskType}, 返回 undefined`);
|
|
420
|
-
resolve(undefined);
|
|
421
|
-
}, timeout ? timeout : this.config.timeout);
|
|
422
|
-
this.pendingTasks.set(taskId, {
|
|
423
|
-
resolve,
|
|
424
|
-
reject,
|
|
425
|
-
timer,
|
|
426
|
-
type: taskType
|
|
427
|
-
});
|
|
428
|
-
console.log(`📋 [ShellX] 创建任务: ${taskId}, 类型: ${taskType}}`);
|
|
429
|
-
}
|
|
430
|
-
if (this.shellxConnected && this.ws) {
|
|
431
|
-
try {
|
|
432
|
-
console.log('📤 [ShellX] 发送消息:', JSON.stringify(message));
|
|
433
|
-
this.ws.send((0, cbor_1.encode)(message));
|
|
434
|
-
if (!taskType)
|
|
435
|
-
resolve(undefined); // For fire-and-forget messages
|
|
436
|
-
}
|
|
437
|
-
catch (error) {
|
|
438
|
-
if (taskType) {
|
|
439
|
-
this.pendingTasks.delete(taskId);
|
|
440
|
-
}
|
|
441
|
-
// 发送失败时返回 undefined 而不是 reject
|
|
442
|
-
console.log(`❌ [ShellX] 发送消息失败,返回 undefined:`, error);
|
|
443
|
-
resolve(undefined);
|
|
444
|
-
}
|
|
445
|
-
}
|
|
446
|
-
else {
|
|
447
|
-
console.log('⏳ [ShellX] 连接未就绪,消息已加入队列');
|
|
448
|
-
this.reconnect();
|
|
449
|
-
this.messageQueue.push(message);
|
|
450
|
-
if (!taskType)
|
|
451
|
-
resolve(undefined);
|
|
452
|
-
}
|
|
453
|
-
});
|
|
454
|
-
});
|
|
477
|
+
async sendMessage(message, taskType) {
|
|
478
|
+
if (!taskType) {
|
|
479
|
+
taskType = "oneway";
|
|
480
|
+
}
|
|
481
|
+
const { promise } = await this.sendMessageWithTaskId(message, taskType);
|
|
482
|
+
return await promise;
|
|
455
483
|
}
|
|
456
484
|
// ===== PROTOCOL-AWARE TASK METHODS =====
|
|
457
485
|
/**
|
|
458
486
|
* Find UI elements on the screen
|
|
459
487
|
*/
|
|
460
|
-
findElement(selector, options) {
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
return this.sendMessage({ findElement: findAction }, 'findElement');
|
|
468
|
-
});
|
|
488
|
+
async findElement(selector, options) {
|
|
489
|
+
const findAction = {
|
|
490
|
+
type: "find",
|
|
491
|
+
selector,
|
|
492
|
+
options,
|
|
493
|
+
};
|
|
494
|
+
return this.sendMessage({ findElement: findAction }, "findElement");
|
|
469
495
|
}
|
|
470
496
|
/**
|
|
471
497
|
* Wait for UI element to appear
|
|
472
498
|
*/
|
|
473
|
-
waitElement(selector, options) {
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
return this.sendMessage({ waitElement: waitAction }, 'waitElement');
|
|
481
|
-
});
|
|
499
|
+
async waitElement(selector, options) {
|
|
500
|
+
const waitAction = {
|
|
501
|
+
type: "wait",
|
|
502
|
+
selector,
|
|
503
|
+
options,
|
|
504
|
+
};
|
|
505
|
+
return this.sendMessage({ waitElement: waitAction }, "waitElement");
|
|
482
506
|
}
|
|
483
507
|
/**
|
|
484
508
|
* Take a screenshot
|
|
509
|
+
* @deprecated Use takeScreenshot() instead for consistency
|
|
485
510
|
*/
|
|
486
|
-
screenShot(options) {
|
|
487
|
-
return
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
511
|
+
async screenShot(options) {
|
|
512
|
+
return this.takeScreenshot(options);
|
|
513
|
+
}
|
|
514
|
+
/**
|
|
515
|
+
* Take a screenshot
|
|
516
|
+
*/
|
|
517
|
+
async takeScreenshot(options) {
|
|
518
|
+
return this.sendMessage({
|
|
519
|
+
screenShot: options || {
|
|
520
|
+
format: "png",
|
|
521
|
+
quality: 30,
|
|
522
|
+
scale: 1.0,
|
|
523
|
+
region: {
|
|
524
|
+
left: 0,
|
|
525
|
+
top: 0,
|
|
526
|
+
width: 1080,
|
|
527
|
+
height: 2340,
|
|
499
528
|
},
|
|
500
|
-
},
|
|
501
|
-
});
|
|
529
|
+
},
|
|
530
|
+
}, "screenShot");
|
|
502
531
|
}
|
|
503
532
|
/**
|
|
504
533
|
* Get screen information
|
|
505
534
|
*/
|
|
506
|
-
getScreenInfo() {
|
|
507
|
-
return
|
|
508
|
-
return this.sendMessage({ screenInfo: {} }, 'screenInfo');
|
|
509
|
-
});
|
|
535
|
+
async getScreenInfo() {
|
|
536
|
+
return this.sendMessage({ screenInfo: { keepScreenOn: false, wakeApp: true } }, "screenInfo");
|
|
510
537
|
}
|
|
511
538
|
/**
|
|
512
|
-
* Get
|
|
539
|
+
* Get screen information
|
|
540
|
+
* @deprecated Use wakeScreenAndGetInfo() instead for clarity
|
|
513
541
|
*/
|
|
514
|
-
|
|
515
|
-
return
|
|
516
|
-
return this.sendMessage({ appList: options || {} }, 'appList');
|
|
517
|
-
});
|
|
542
|
+
async getAndWakeScreen() {
|
|
543
|
+
return this.wakeScreenAndGetInfo();
|
|
518
544
|
}
|
|
519
545
|
/**
|
|
520
|
-
*
|
|
546
|
+
* Wake screen and get screen information
|
|
521
547
|
*/
|
|
522
|
-
|
|
523
|
-
return
|
|
524
|
-
return this.sendMessage({ appInfo: { packageName } }, 'appInfo');
|
|
525
|
-
});
|
|
548
|
+
async wakeScreenAndGetInfo() {
|
|
549
|
+
return this.sendMessage({ screenInfo: { keepScreenOn: true, wakeApp: true } }, "screenInfo");
|
|
526
550
|
}
|
|
527
551
|
/**
|
|
528
552
|
* Execute an action sequence
|
|
529
553
|
*/
|
|
530
|
-
executeAction(actionSequence, taskId,
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
});
|
|
554
|
+
async executeAction(actionSequence, taskId, timeout) {
|
|
555
|
+
const { promise } = await this.sendMessageWithTaskId({ actions: actionSequence }, "action", { taskId, timeout });
|
|
556
|
+
return await promise;
|
|
534
557
|
}
|
|
535
558
|
/**
|
|
536
559
|
* Execute promptflow actions
|
|
537
560
|
*/
|
|
538
|
-
executePromptFlow(actionSequence) {
|
|
539
|
-
return
|
|
540
|
-
return this.sendMessage({ promptflowx: actionSequence });
|
|
541
|
-
});
|
|
561
|
+
async executePromptFlow(actionSequence) {
|
|
562
|
+
return this.sendMessage({ promptflowx: actionSequence });
|
|
542
563
|
}
|
|
543
564
|
/**
|
|
544
565
|
* Listen for display changes
|
|
545
566
|
*/
|
|
546
|
-
screenChange(options) {
|
|
547
|
-
return
|
|
548
|
-
|
|
549
|
-
screenChange: options || { enable: true },
|
|
550
|
-
});
|
|
567
|
+
async screenChange(options) {
|
|
568
|
+
return this.sendMessage({
|
|
569
|
+
screenChange: options || { enable: true },
|
|
551
570
|
});
|
|
552
571
|
}
|
|
553
572
|
/**
|
|
554
573
|
* Switch to a different node
|
|
555
574
|
*/
|
|
556
|
-
switchNode(nodeId) {
|
|
557
|
-
return
|
|
558
|
-
return this.sendMessage({ switchNode: nodeId });
|
|
559
|
-
});
|
|
575
|
+
async switchNode(nodeId) {
|
|
576
|
+
return this.sendMessage({ switchNode: nodeId });
|
|
560
577
|
}
|
|
561
578
|
/**
|
|
562
579
|
* Send authentication data
|
|
563
580
|
*/
|
|
564
|
-
authenticate(authData) {
|
|
565
|
-
return
|
|
566
|
-
return this.sendMessage({ authenticate: authData });
|
|
567
|
-
});
|
|
581
|
+
async authenticate(authData) {
|
|
582
|
+
return this.sendMessage({ authenticate: authData });
|
|
568
583
|
}
|
|
569
584
|
/**
|
|
570
585
|
* Set user name
|
|
571
586
|
*/
|
|
572
|
-
setName(name) {
|
|
573
|
-
return
|
|
574
|
-
return this.sendMessage({ setName: name });
|
|
575
|
-
});
|
|
587
|
+
async setName(name) {
|
|
588
|
+
return this.sendMessage({ setName: name });
|
|
576
589
|
}
|
|
577
590
|
/**
|
|
578
591
|
* Send chat message
|
|
579
592
|
*/
|
|
580
|
-
sendChat(message) {
|
|
581
|
-
return
|
|
582
|
-
|
|
583
|
-
|
|
593
|
+
async sendChat(message) {
|
|
594
|
+
return this.sendMessage({ chat: message });
|
|
595
|
+
}
|
|
596
|
+
/**
|
|
597
|
+
* Start ASR (Automatic Speech Recognition)
|
|
598
|
+
*/
|
|
599
|
+
async startAsr() {
|
|
600
|
+
return this.sendMessage({ asrControl: { action: "start_asr" } }, "oneway");
|
|
601
|
+
}
|
|
602
|
+
/**
|
|
603
|
+
* Stop ASR (Automatic Speech Recognition)
|
|
604
|
+
*/
|
|
605
|
+
async stopAsr() {
|
|
606
|
+
return this.sendMessage({ asrControl: { action: "stop_asr" } }, "oneway");
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
609
|
+
* Speak text using TTS (Text-to-Speech)
|
|
610
|
+
*/
|
|
611
|
+
async speak(text) {
|
|
612
|
+
return this.sendMessage({ ttsControl: { action: "tts_speak", text } }, "oneway");
|
|
613
|
+
}
|
|
614
|
+
/**
|
|
615
|
+
* Stop TTS (Text-to-Speech)
|
|
616
|
+
*/
|
|
617
|
+
async stopTts() {
|
|
618
|
+
return this.sendMessage({ ttsControl: { action: "tts_stop" } }, "oneway");
|
|
584
619
|
}
|
|
585
620
|
/**
|
|
586
621
|
* Send raw message (for custom integrations)
|
|
587
622
|
*/
|
|
588
|
-
sendRawMessage(message) {
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
623
|
+
async sendRawMessage(message) {
|
|
624
|
+
// Auto-detect task type from message content
|
|
625
|
+
const taskType = this.detectTaskType(message);
|
|
626
|
+
return this.sendMessage(message, taskType);
|
|
592
627
|
}
|
|
593
628
|
/**
|
|
594
|
-
*
|
|
595
|
-
* @param message 要发送的消息
|
|
596
|
-
* @param taskType 任务类型
|
|
597
|
-
* @returns Promise<{taskId: string, promise: Promise<any>}>
|
|
629
|
+
* Detect task type from message content
|
|
598
630
|
*/
|
|
599
|
-
|
|
631
|
+
detectTaskType(message) {
|
|
632
|
+
if (message.findElement)
|
|
633
|
+
return "findElement";
|
|
634
|
+
if (message.waitElement)
|
|
635
|
+
return "waitElement";
|
|
636
|
+
if (message.screenShot)
|
|
637
|
+
return "screenShot";
|
|
638
|
+
if (message.screenInfo)
|
|
639
|
+
return "screenInfo";
|
|
640
|
+
if (message.appList)
|
|
641
|
+
return "appList";
|
|
642
|
+
if (message.action)
|
|
643
|
+
return "action";
|
|
644
|
+
if (message.actions)
|
|
645
|
+
return "action";
|
|
646
|
+
if (message.promptflowx)
|
|
647
|
+
return "promptflowx";
|
|
648
|
+
if (message.asrControl)
|
|
649
|
+
return "asrControl";
|
|
650
|
+
// Oneway messages (fire-and-forget, no response expected)
|
|
651
|
+
if (message.ttsControl)
|
|
652
|
+
return "oneway";
|
|
653
|
+
if (message.screenChange)
|
|
654
|
+
return "oneway";
|
|
655
|
+
if (message.switchNode)
|
|
656
|
+
return "oneway";
|
|
657
|
+
if (message.authenticate)
|
|
658
|
+
return "oneway";
|
|
659
|
+
if (message.setName)
|
|
660
|
+
return "oneway";
|
|
661
|
+
if (message.chat)
|
|
662
|
+
return "oneway";
|
|
663
|
+
if (message.ping)
|
|
664
|
+
return "oneway";
|
|
665
|
+
// Default to oneway for unknown message types
|
|
666
|
+
return "oneway";
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* Send message and return taskId
|
|
670
|
+
* @param message Message to send
|
|
671
|
+
* @param taskType Task type
|
|
672
|
+
* @param options Optional parameters { timeout, taskId }
|
|
673
|
+
* @returns Promise<{taskId: string, promise: Promise<T>>}
|
|
674
|
+
*/
|
|
675
|
+
async sendMessageWithTaskId(message, taskType, options) {
|
|
600
676
|
return new Promise((resolve) => {
|
|
677
|
+
const { timeout, taskId: definedTaskId } = options || {};
|
|
601
678
|
let taskId = definedTaskId;
|
|
602
679
|
if (taskId == null) {
|
|
603
|
-
taskId = (
|
|
680
|
+
taskId = uuidv4();
|
|
604
681
|
}
|
|
682
|
+
this.logger.info(`📋 [ShellX] Created task: ${taskId}, Task type: ${taskType}`);
|
|
683
|
+
this.logger.info(message);
|
|
684
|
+
const outgoingMessage = taskId ? { ...message, taskId } : message;
|
|
605
685
|
if (isPendingTask(taskType)) {
|
|
606
686
|
const timer = setTimeout(() => {
|
|
607
687
|
if (taskId != null) {
|
|
688
|
+
const task = this.pendingTasks.get(taskId);
|
|
689
|
+
if (task) {
|
|
690
|
+
// Create a timeout result based on task type
|
|
691
|
+
const timeoutResult = this.createTimeoutResult(taskType, timeout ?? this.config.timeout);
|
|
692
|
+
// Resolve with timeout result instead of rejecting
|
|
693
|
+
task.resolve(timeoutResult);
|
|
694
|
+
}
|
|
608
695
|
this.pendingTasks.delete(taskId);
|
|
609
696
|
}
|
|
610
|
-
|
|
611
|
-
}, timeout
|
|
697
|
+
this.logger.info(`⏰ [ShellX] Task timeout: ${taskId}, Task type: ${taskType}`);
|
|
698
|
+
}, timeout ?? this.config.timeout);
|
|
612
699
|
this.pendingTasks.set(taskId, {
|
|
613
700
|
resolve: () => { },
|
|
614
701
|
reject: () => { },
|
|
615
702
|
timer,
|
|
616
703
|
type: taskType,
|
|
617
704
|
});
|
|
618
|
-
|
|
705
|
+
this.logger.info(`📋 [ShellX] Created manually controllable task: ${taskId}, Task type: ${taskType}}`);
|
|
619
706
|
}
|
|
620
707
|
if (this.shellxConnected && this.ws) {
|
|
621
708
|
try {
|
|
622
|
-
|
|
623
|
-
this.ws.send((
|
|
709
|
+
this.logger.info(" 📤 [ShellX] Sending message:", JSON.stringify(outgoingMessage));
|
|
710
|
+
this.ws.send(cborEncode(outgoingMessage));
|
|
624
711
|
const promise = new Promise((promiseResolve, promiseReject) => {
|
|
625
712
|
if (isPendingTask(taskType)) {
|
|
626
713
|
if (taskId != null) {
|
|
627
714
|
const task = this.pendingTasks.get(taskId);
|
|
628
715
|
if (task) {
|
|
716
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any
|
|
629
717
|
task.resolve = promiseResolve;
|
|
718
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any
|
|
630
719
|
task.reject = promiseReject;
|
|
631
720
|
}
|
|
632
721
|
}
|
|
@@ -641,12 +730,14 @@ class ConnectionTaskClient {
|
|
|
641
730
|
if (isPendingTask(taskType)) {
|
|
642
731
|
this.pendingTasks.delete(taskId);
|
|
643
732
|
}
|
|
644
|
-
|
|
733
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
734
|
+
resolve({ taskId, promise: Promise.reject(err) });
|
|
645
735
|
}
|
|
646
736
|
}
|
|
647
737
|
else {
|
|
648
|
-
|
|
649
|
-
this.messageQueue.push(
|
|
738
|
+
this.logger.info("⏳ [ShellX] Connection not ready, message queued");
|
|
739
|
+
this.messageQueue.push(outgoingMessage);
|
|
740
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
|
|
650
741
|
resolve({ taskId, promise: Promise.resolve(undefined) });
|
|
651
742
|
}
|
|
652
743
|
});
|
|
@@ -654,10 +745,10 @@ class ConnectionTaskClient {
|
|
|
654
745
|
// ===== CONNECTION MANAGEMENT =====
|
|
655
746
|
startPing() {
|
|
656
747
|
this.stopPing();
|
|
657
|
-
|
|
748
|
+
this.logger.info("🏓 [ShellX] Starting heartbeat detection...");
|
|
658
749
|
this.pingIntervalId = setInterval(() => {
|
|
659
750
|
if (this.ws && this.shellxConnected) {
|
|
660
|
-
this.ws.send((
|
|
751
|
+
this.ws.send(cborEncode({ ping: Date.now() }));
|
|
661
752
|
}
|
|
662
753
|
}, this.config.pingInterval);
|
|
663
754
|
}
|
|
@@ -667,32 +758,37 @@ class ConnectionTaskClient {
|
|
|
667
758
|
this.pingIntervalId = null;
|
|
668
759
|
}
|
|
669
760
|
}
|
|
670
|
-
reconnect() {
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
761
|
+
async reconnect() {
|
|
762
|
+
// TODO: Local needs reconnection
|
|
763
|
+
this.logger.info(`🔄 [ShellX] Reconnecting... (${this.reconnectAttempts}/${this.config.reconnectMaxAttempts})` +
|
|
764
|
+
this.wsUrl);
|
|
765
|
+
if (this.reconnectAttempts >= this.config.reconnectMaxAttempts) {
|
|
766
|
+
this.logger.error("❌ [ShellX] Reconnection failed, max attempts reached");
|
|
767
|
+
this.config.onReconnectFailed?.();
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
this.reconnectAttempts++;
|
|
771
|
+
// Exponential backoff strategy: baseInterval * 2^(attempts-1), with maximum delay cap and random jitter
|
|
772
|
+
const baseInterval = this.config.reconnectInterval || 1000;
|
|
773
|
+
const maxDelay = 30000; // Maximum delay 30 seconds
|
|
774
|
+
const exponentialDelay = baseInterval * Math.pow(2, this.reconnectAttempts - 1);
|
|
775
|
+
const delay = Math.min(exponentialDelay, maxDelay);
|
|
776
|
+
// Add random jitter (±20%) to avoid multiple clients reconnecting simultaneously
|
|
777
|
+
const jitter = delay * 0.2 * (Math.random() * 2 - 1);
|
|
778
|
+
const finalDelay = Math.max(0, delay + jitter);
|
|
779
|
+
await wait(Math.floor(finalDelay));
|
|
780
|
+
void this.init();
|
|
685
781
|
}
|
|
686
782
|
flushQueue() {
|
|
687
783
|
while (this.messageQueue.length > 0 && this.ws && this.shellxConnected) {
|
|
688
784
|
const message = this.messageQueue.shift();
|
|
689
785
|
if (message) {
|
|
690
786
|
try {
|
|
691
|
-
|
|
692
|
-
this.ws.send((
|
|
787
|
+
this.logger.info("flushQueue Sending message:", message);
|
|
788
|
+
this.ws.send(cborEncode(message));
|
|
693
789
|
}
|
|
694
790
|
catch (error) {
|
|
695
|
-
|
|
791
|
+
this.logger.error("Failed to send queued message:", error);
|
|
696
792
|
// Re-queue the message if sending fails
|
|
697
793
|
this.messageQueue.unshift(message);
|
|
698
794
|
break;
|
|
@@ -701,11 +797,10 @@ class ConnectionTaskClient {
|
|
|
701
797
|
}
|
|
702
798
|
}
|
|
703
799
|
close() {
|
|
704
|
-
var _a;
|
|
705
800
|
this.shellxConnected = false;
|
|
706
801
|
this.wsConnected = false;
|
|
707
802
|
this.authenticated = false;
|
|
708
|
-
|
|
803
|
+
this.ws?.close();
|
|
709
804
|
this.pendingTasks.clear();
|
|
710
805
|
this.messageQueue = [];
|
|
711
806
|
this.stopPing();
|
|
@@ -720,6 +815,12 @@ class ConnectionTaskClient {
|
|
|
720
815
|
get isAuthenticated() {
|
|
721
816
|
return this.authenticated;
|
|
722
817
|
}
|
|
818
|
+
/**
|
|
819
|
+
* Get the authenticated device ID (UUID)
|
|
820
|
+
*/
|
|
821
|
+
getDeviceId() {
|
|
822
|
+
return this.deviceId;
|
|
823
|
+
}
|
|
723
824
|
get pendingTaskCount() {
|
|
724
825
|
return this.pendingTasks.size;
|
|
725
826
|
}
|
|
@@ -727,36 +828,128 @@ class ConnectionTaskClient {
|
|
|
727
828
|
return this.messageQueue.length;
|
|
728
829
|
}
|
|
729
830
|
/**
|
|
730
|
-
*
|
|
831
|
+
* Get current execution log
|
|
832
|
+
*/
|
|
833
|
+
getExecutionLogs() {
|
|
834
|
+
return [...this.executionLogsRef];
|
|
835
|
+
}
|
|
836
|
+
/**
|
|
837
|
+
* Clear current execution log
|
|
838
|
+
*/
|
|
839
|
+
clearExecutionLogs() {
|
|
840
|
+
this.executionLogsRef.length = 0;
|
|
841
|
+
}
|
|
842
|
+
/**
|
|
843
|
+
* Manually append an execution log
|
|
844
|
+
*/
|
|
845
|
+
appendExecutionLog(log) {
|
|
846
|
+
this.logger.info("Append log:", log);
|
|
847
|
+
this.executionLogsRef.push(log);
|
|
848
|
+
}
|
|
849
|
+
/**
|
|
850
|
+
* Set associated ShellX instance
|
|
731
851
|
*/
|
|
732
852
|
setShellX(shellx) {
|
|
733
853
|
this.shellx = shellx;
|
|
734
|
-
|
|
854
|
+
this.logger.info("🔗 [ShellX] ShellX instance associated");
|
|
855
|
+
}
|
|
856
|
+
/**
|
|
857
|
+
* Create a timeout result object based on task type
|
|
858
|
+
* @param taskType Type of the task that timed out
|
|
859
|
+
* @param timeoutMs Timeout duration in milliseconds
|
|
860
|
+
* @returns A result object with success: false
|
|
861
|
+
*/
|
|
862
|
+
createTimeoutResult(taskType, timeoutMs) {
|
|
863
|
+
const baseResult = {
|
|
864
|
+
success: false,
|
|
865
|
+
error: `Task timed out after ${timeoutMs}ms`,
|
|
866
|
+
duration: timeoutMs,
|
|
867
|
+
timestamp: Date.now(),
|
|
868
|
+
};
|
|
869
|
+
// Return type-specific result based on task type
|
|
870
|
+
switch (taskType) {
|
|
871
|
+
case "action":
|
|
872
|
+
// For action tasks, return a basic result
|
|
873
|
+
return baseResult;
|
|
874
|
+
case "screenShot":
|
|
875
|
+
return {
|
|
876
|
+
...baseResult,
|
|
877
|
+
imageData: "",
|
|
878
|
+
imagePath: "",
|
|
879
|
+
format: "png",
|
|
880
|
+
};
|
|
881
|
+
case "screenInfo":
|
|
882
|
+
return {
|
|
883
|
+
...baseResult,
|
|
884
|
+
width: 0,
|
|
885
|
+
height: 0,
|
|
886
|
+
density: 0,
|
|
887
|
+
screenOn: false,
|
|
888
|
+
screenUnlocked: false,
|
|
889
|
+
};
|
|
890
|
+
case "findElement":
|
|
891
|
+
case "waitElement":
|
|
892
|
+
return {
|
|
893
|
+
...baseResult,
|
|
894
|
+
elements: [],
|
|
895
|
+
};
|
|
896
|
+
case "appList":
|
|
897
|
+
return {
|
|
898
|
+
...baseResult,
|
|
899
|
+
apps: [],
|
|
900
|
+
totalCount: 0,
|
|
901
|
+
systemAppCount: 0,
|
|
902
|
+
userAppCount: 0,
|
|
903
|
+
};
|
|
904
|
+
case "input":
|
|
905
|
+
return baseResult;
|
|
906
|
+
case "swipe":
|
|
907
|
+
return {
|
|
908
|
+
...baseResult,
|
|
909
|
+
fromX: 0,
|
|
910
|
+
fromY: 0,
|
|
911
|
+
toX: 0,
|
|
912
|
+
toY: 0,
|
|
913
|
+
};
|
|
914
|
+
case "press":
|
|
915
|
+
return {
|
|
916
|
+
...baseResult,
|
|
917
|
+
key: "",
|
|
918
|
+
};
|
|
919
|
+
case "clipboard":
|
|
920
|
+
return {
|
|
921
|
+
...baseResult,
|
|
922
|
+
text: "",
|
|
923
|
+
};
|
|
924
|
+
default:
|
|
925
|
+
// Default fallback result
|
|
926
|
+
return baseResult;
|
|
927
|
+
}
|
|
735
928
|
}
|
|
736
929
|
/**
|
|
737
|
-
*
|
|
930
|
+
* Get associated ShellX instance
|
|
738
931
|
*/
|
|
739
932
|
getShellX() {
|
|
740
933
|
return this.shellx;
|
|
741
934
|
}
|
|
742
935
|
/**
|
|
743
|
-
*
|
|
744
|
-
* @param taskId
|
|
745
|
-
* @param result
|
|
746
|
-
* @param success
|
|
936
|
+
* Manually complete task by taskId
|
|
937
|
+
* @param taskId Task ID
|
|
938
|
+
* @param result Task result
|
|
939
|
+
* @param success Success status
|
|
747
940
|
*/
|
|
748
941
|
completeTask(taskId, result, success = true) {
|
|
749
942
|
const task = this.pendingTasks.get(taskId);
|
|
750
943
|
if (!task) {
|
|
751
|
-
|
|
944
|
+
this.logger.info(`⚠️ [ShellX] Task not found: ${taskId}`);
|
|
752
945
|
return false;
|
|
753
946
|
}
|
|
754
|
-
|
|
755
|
-
//
|
|
947
|
+
this.logger.info(`✅ [ShellX] Manually completed task: ${taskId}, Task type: ${task.type}, Success: ${success}`);
|
|
948
|
+
// Clear timeout timer
|
|
756
949
|
clearTimeout(task.timer);
|
|
757
|
-
//
|
|
950
|
+
// Remove from pending tasks
|
|
758
951
|
this.pendingTasks.delete(taskId);
|
|
759
|
-
//
|
|
952
|
+
// Call appropriate callback based on success status
|
|
760
953
|
if (success) {
|
|
761
954
|
task.resolve(result || { success: true, taskId });
|
|
762
955
|
}
|
|
@@ -766,13 +959,13 @@ class ConnectionTaskClient {
|
|
|
766
959
|
return true;
|
|
767
960
|
}
|
|
768
961
|
/**
|
|
769
|
-
*
|
|
962
|
+
* Get all pending task IDs
|
|
770
963
|
*/
|
|
771
964
|
getPendingTaskIds() {
|
|
772
965
|
return Array.from(this.pendingTasks.keys());
|
|
773
966
|
}
|
|
774
967
|
/**
|
|
775
|
-
*
|
|
968
|
+
* Get task info by taskId
|
|
776
969
|
*/
|
|
777
970
|
getTaskInfo(taskId) {
|
|
778
971
|
const task = this.pendingTasks.get(taskId);
|
|
@@ -780,14 +973,14 @@ class ConnectionTaskClient {
|
|
|
780
973
|
return null;
|
|
781
974
|
}
|
|
782
975
|
return {
|
|
783
|
-
type: task.type
|
|
976
|
+
type: task.type,
|
|
784
977
|
};
|
|
785
978
|
}
|
|
786
979
|
/**
|
|
787
|
-
*
|
|
788
|
-
* @param taskType
|
|
789
|
-
* @param result
|
|
790
|
-
* @param success
|
|
980
|
+
* Batch complete tasks by type
|
|
981
|
+
* @param taskType Task type
|
|
982
|
+
* @param result Task result
|
|
983
|
+
* @param success Success status
|
|
791
984
|
*/
|
|
792
985
|
completeTasksByType(taskType, result, success = true) {
|
|
793
986
|
const taskIds = this.getPendingTaskIds();
|
|
@@ -799,12 +992,16 @@ class ConnectionTaskClient {
|
|
|
799
992
|
completedCount++;
|
|
800
993
|
}
|
|
801
994
|
}
|
|
802
|
-
|
|
995
|
+
this.logger.info(`📦 [ShellX] Batch completed ${completedCount} tasks of type ${taskType}`);
|
|
803
996
|
return completedCount;
|
|
804
997
|
}
|
|
805
998
|
}
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
999
|
+
export default ConnectionClient;
|
|
1000
|
+
export { ShellX } from "./shellx.js";
|
|
1001
|
+
// Error handling exports
|
|
1002
|
+
export { createErrorResponse, createOperationResult, ErrorCode, ShellXError, ConnectionError, OperationError, ElementError, ValidationError, } from "./errors.js";
|
|
1003
|
+
// Error handler utilities
|
|
1004
|
+
export { extractErrorMessage, createErrorResult, createSuccessResult, handleOperation, handleOperationWithRetry, validateRequired, validateOneOfRequired, wrapErrorWithContext, } from "./error-handler.js";
|
|
1005
|
+
// Domain management exports
|
|
1006
|
+
export { buildDeviceApiUrl, buildDeviceConfigUrl, buildDeviceRegisterUrl, buildTokenVerifyUrl, buildTokenRefreshUrl, getApiBaseUrl, getWsServerUrl, getCurrentDomainInfo, createDomainConfig, isChineseUser, } from "./domain-manager.js";
|
|
1007
|
+
//# sourceMappingURL=index.js.map
|