shellx-ai 1.0.12 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +586 -0
- package/dist/automation/element-finder.d.ts +189 -0
- package/dist/automation/element-finder.js +322 -0
- package/dist/automation/element-finder.js.map +1 -0
- package/dist/automation/ui-action-handler.d.ts +330 -0
- package/dist/automation/ui-action-handler.js +873 -0
- package/dist/automation/ui-action-handler.js.map +1 -0
- package/dist/cbor-compat.d.ts +27 -0
- package/dist/cbor-compat.js +108 -0
- package/dist/cbor-compat.js.map +1 -0
- package/dist/domain-manager.d.ts +80 -0
- package/dist/domain-manager.js +158 -0
- package/dist/domain-manager.js.map +1 -0
- package/dist/error-handler.d.ts +87 -0
- package/dist/error-handler.js +148 -0
- package/dist/error-handler.js.map +1 -0
- package/dist/errors.d.ts +114 -0
- package/dist/errors.js +139 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +163 -54
- package/dist/index.js +706 -480
- package/dist/index.js.map +1 -0
- package/dist/logger.d.ts +81 -0
- package/dist/logger.js +128 -0
- package/dist/logger.js.map +1 -0
- package/dist/protocol.d.ts +147 -31
- package/dist/protocol.js +2 -2
- package/dist/protocol.js.map +1 -0
- package/dist/shell/output-buffer.d.ts +152 -0
- package/dist/shell/output-buffer.js +163 -0
- package/dist/shell/output-buffer.js.map +1 -0
- package/dist/shell/shell-command-executor.d.ts +182 -0
- package/dist/shell/shell-command-executor.js +348 -0
- package/dist/shell/shell-command-executor.js.map +1 -0
- package/dist/shellx.d.ts +681 -178
- package/dist/shellx.js +762 -1159
- package/dist/shellx.js.map +1 -0
- package/dist/types.d.ts +132 -57
- package/dist/types.js +4 -4
- package/dist/types.js.map +1 -0
- package/dist/utils/retry-helper.d.ts +73 -0
- package/dist/utils/retry-helper.js +92 -0
- package/dist/utils/retry-helper.js.map +1 -0
- package/dist/utils.d.ts +3 -3
- package/dist/utils.js +17 -23
- package/dist/utils.js.map +1 -0
- package/package.json +95 -62
package/dist/index.js
CHANGED
|
@@ -1,52 +1,41 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
-
});
|
|
1
|
+
import { v4 as uuidv4 } from "uuid";
|
|
2
|
+
import { wait } from "./utils.js";
|
|
3
|
+
import { buildDeviceApiUrl } from "./domain-manager.js";
|
|
4
|
+
import { createLogger } from "./logger.js";
|
|
5
|
+
// Add readable timestamp to all logs
|
|
6
|
+
const formatTs = () => {
|
|
7
|
+
const now = new Date();
|
|
8
|
+
return now.toISOString();
|
|
9
|
+
};
|
|
10
|
+
const executionLogStore = [];
|
|
11
|
+
const formatLogArgs = (args) => args
|
|
12
|
+
.map((arg) => {
|
|
13
|
+
if (typeof arg === "string")
|
|
14
|
+
return arg;
|
|
15
|
+
try {
|
|
16
|
+
return JSON.stringify(arg);
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return String(arg);
|
|
20
|
+
}
|
|
21
|
+
})
|
|
22
|
+
.join(" ");
|
|
23
|
+
const pushExecutionLog = (ts, args) => {
|
|
24
|
+
executionLogStore.push(`[${ts}] ${formatLogArgs(args)}`);
|
|
25
|
+
};
|
|
26
|
+
const originalConsoleLog = console.log.bind(console);
|
|
27
|
+
const originalConsoleWarn = console.warn.bind(console);
|
|
28
|
+
const originalConsoleError = console.error.bind(console);
|
|
29
|
+
const wrapConsole = (originalFn) => (...args) => {
|
|
30
|
+
const ts = formatTs();
|
|
31
|
+
pushExecutionLog(ts, args);
|
|
32
|
+
originalFn(`[${ts}]`, ...args);
|
|
43
33
|
};
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
exports.DEFAULT_CONFIG = {
|
|
34
|
+
console.log = wrapConsole(originalConsoleLog);
|
|
35
|
+
console.warn = wrapConsole(originalConsoleWarn);
|
|
36
|
+
console.error = wrapConsole(originalConsoleError);
|
|
37
|
+
// Define default configuration
|
|
38
|
+
export const DEFAULT_CONFIG = {
|
|
50
39
|
timeout: 5000,
|
|
51
40
|
reconnect: true,
|
|
52
41
|
reconnectMaxAttempts: 5,
|
|
@@ -57,230 +46,316 @@ exports.DEFAULT_CONFIG = {
|
|
|
57
46
|
onError: () => { },
|
|
58
47
|
onReconnectFailed: () => { },
|
|
59
48
|
};
|
|
60
|
-
|
|
61
|
-
// 安全地获取环境变量,兼容浏览器和Node.js环境
|
|
62
|
-
let authKey = (0, utils_1.getEnvVar)('SHELLX_AUTH_KEY');
|
|
49
|
+
import { initCbor, cborEncode, cborDecode } from "./cbor-compat.js";
|
|
63
50
|
/**
|
|
64
|
-
*
|
|
51
|
+
* Get WebSocket constructor for current environment
|
|
65
52
|
*/
|
|
66
|
-
function getWebSocket() {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
});
|
|
53
|
+
async function getWebSocket() {
|
|
54
|
+
// Check if global WebSocket exists (browser environment or Node.js 20+)
|
|
55
|
+
if (typeof globalThis.WebSocket !== "undefined") {
|
|
56
|
+
return globalThis.WebSocket;
|
|
57
|
+
}
|
|
58
|
+
// Dynamically import ws module in Node.js environment
|
|
59
|
+
try {
|
|
60
|
+
const wsModule = await import("ws");
|
|
61
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
|
|
62
|
+
return (wsModule.default || wsModule);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
console.error("❌ [WebSocket] ws module is not available, please install: npm install ws");
|
|
66
|
+
throw new Error("WebSocket not available. Please install ws module: npm install ws");
|
|
67
|
+
}
|
|
82
68
|
}
|
|
83
69
|
function isPendingTask(taskType) {
|
|
84
|
-
return taskType && taskType !== "command";
|
|
70
|
+
return taskType && taskType !== "command" && taskType !== "oneway";
|
|
85
71
|
}
|
|
86
72
|
/**
|
|
87
|
-
*
|
|
73
|
+
* Low-level WebSocket Client for direct protocol communication
|
|
74
|
+
*
|
|
75
|
+
* @warning This is a low-level API intended for advanced use cases only.
|
|
76
|
+
* Most users should use the ShellX class instead, which provides a simpler,
|
|
77
|
+
* more intuitive interface with automatic error handling and retry logic.
|
|
78
|
+
*
|
|
79
|
+
* @example
|
|
80
|
+
* ```typescript
|
|
81
|
+
* // For most users, use ShellX instead:
|
|
82
|
+
* import { ShellX } from './shellx';
|
|
83
|
+
* const shellx = new ShellX({ deviceId: 'your-device-id' });
|
|
84
|
+
* await shellx.connect();
|
|
85
|
+
* await shellx.click({ text: 'Submit' });
|
|
86
|
+
* ```
|
|
87
|
+
*
|
|
88
|
+
* @see Use ShellX for a high-level API unless you need low-level WebSocket access
|
|
88
89
|
*/
|
|
89
|
-
class
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
90
|
+
export class ConnectionClient {
|
|
91
|
+
wsUrl;
|
|
92
|
+
deviceId;
|
|
93
|
+
config;
|
|
94
|
+
ws = null;
|
|
95
|
+
shellxConnected = false; // ShellX.ai connection status
|
|
96
|
+
wsConnected = false; // WebSocket connection status
|
|
97
|
+
authenticated = false; // Authentication status
|
|
98
|
+
pendingTasks = new Map();
|
|
99
|
+
messageQueue = [];
|
|
100
|
+
reconnectAttempts = 0;
|
|
101
|
+
pingIntervalId = null;
|
|
102
|
+
initializationPromise = null;
|
|
103
|
+
shellx = null; // Associated ShellX instance
|
|
104
|
+
executionLogsRef = executionLogStore; // Current execution log
|
|
105
|
+
logger;
|
|
106
|
+
constructor(deviceId, config = {}, logger) {
|
|
101
107
|
this.deviceId = deviceId;
|
|
102
|
-
this.config =
|
|
108
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
109
|
+
// Use provided logger or create default one
|
|
110
|
+
this.logger = logger || createLogger("ConnectionClient");
|
|
103
111
|
this.initializationPromise = this.init();
|
|
104
112
|
}
|
|
105
|
-
init() {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
if (!
|
|
113
|
+
async init() {
|
|
114
|
+
await initCbor(); // load the right CBOR codec for this environment
|
|
115
|
+
return new Promise((resolve, reject) => {
|
|
116
|
+
this.authenticateDevice()
|
|
117
|
+
.then((wsUrl) => {
|
|
118
|
+
if (!wsUrl) {
|
|
111
119
|
this.config.onError(new Event("connection"));
|
|
120
|
+
void this.reconnect();
|
|
121
|
+
reject(new Error("Failed to authenticate device"));
|
|
112
122
|
return;
|
|
113
123
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
const
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
124
|
+
// ✅ Store the WebSocket URL for later use
|
|
125
|
+
this.wsUrl = wsUrl;
|
|
126
|
+
const separator = wsUrl.includes("?") ? "&" : "?";
|
|
127
|
+
const wsUrlWithSessionId = `${wsUrl}${separator}session_id=${this.deviceId}`;
|
|
128
|
+
this.logger.info("Initializing ShellX client wsUrl:" + wsUrl + " deviceId: " + this.deviceId);
|
|
129
|
+
this.logger.info("WebSocket URL with session_id: " + wsUrlWithSessionId);
|
|
130
|
+
// Get WebSocket constructor for current environment
|
|
131
|
+
getWebSocket()
|
|
132
|
+
.then((WebSocketConstructor) => {
|
|
133
|
+
try {
|
|
134
|
+
this.ws = new WebSocketConstructor(wsUrlWithSessionId);
|
|
135
|
+
// Set binary type (if supported)
|
|
136
|
+
if (this.ws.binaryType !== undefined) {
|
|
137
|
+
this.ws.binaryType = "arraybuffer";
|
|
138
|
+
}
|
|
139
|
+
this.ws.onopen = () => {
|
|
140
|
+
this.logger.info("🔗 [ShellX] WebSocket connection established:" + this.wsUrl);
|
|
141
|
+
this.wsConnected = true;
|
|
142
|
+
this.reconnectAttempts = 0;
|
|
143
|
+
// Mark as connected after WebSocket open
|
|
144
|
+
this.shellxConnected = true;
|
|
145
|
+
this.config.onOpen?.(this.deviceId);
|
|
146
|
+
void this.flushQueue();
|
|
147
|
+
void this.startPing();
|
|
148
|
+
// Resolve initialization promise after WebSocket is connected
|
|
149
|
+
resolve();
|
|
150
|
+
// Check if we're in Android sandbox environment
|
|
151
|
+
const isAndroidSandbox = process.env["SANDBOX"] === "android";
|
|
152
|
+
if (isAndroidSandbox) {
|
|
153
|
+
void this.getScreenInfo();
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
void this.getAndWakeScreen();
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
this.ws.onmessage = (event) => {
|
|
160
|
+
void this.handleMessage(event);
|
|
161
|
+
};
|
|
162
|
+
this.ws.onclose = (event) => {
|
|
163
|
+
this.logger.info("❌ [ShellX] WebSocket connection closed, attempting to reconnect..." +
|
|
164
|
+
event.code +
|
|
165
|
+
" " +
|
|
166
|
+
this.wsUrl);
|
|
167
|
+
this.wsConnected = false;
|
|
168
|
+
this.shellxConnected = false;
|
|
169
|
+
this.authenticated = false;
|
|
170
|
+
this.config.onClose?.(event);
|
|
171
|
+
this.stopPing();
|
|
172
|
+
void this.reconnect();
|
|
173
|
+
};
|
|
174
|
+
this.ws.onerror = (error) => {
|
|
175
|
+
this.logger.error("❌ [ShellX] WebSocket error:", error);
|
|
176
|
+
this.config.onError?.(error);
|
|
177
|
+
reject(new Error("WebSocket connection failed"));
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
catch (error) {
|
|
181
|
+
this.logger.error("❌ [WebSocket] Initialization failed:", error);
|
|
182
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
183
|
+
this.config.onError?.(err);
|
|
184
|
+
reject(err);
|
|
185
|
+
}
|
|
186
|
+
})
|
|
187
|
+
.catch((error) => {
|
|
188
|
+
this.logger.error("❌ [WebSocket] Failed to get constructor:", error);
|
|
189
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
190
|
+
reject(err);
|
|
191
|
+
});
|
|
192
|
+
})
|
|
193
|
+
.catch((error) => {
|
|
194
|
+
this.logger.error("❌ [Auth] Device authentication failed:", error);
|
|
195
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
196
|
+
reject(err);
|
|
197
|
+
});
|
|
155
198
|
});
|
|
156
199
|
}
|
|
157
|
-
authenticateDevice(
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
if (
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
}
|
|
200
|
+
async authenticateDevice() {
|
|
201
|
+
let fallbackUrl = undefined;
|
|
202
|
+
// 1. Prioritize local service detection
|
|
203
|
+
try {
|
|
204
|
+
// fetch timeout implementation
|
|
205
|
+
const fetchWithTimeout = (url, options, timeout = 1000) => Promise.race([
|
|
206
|
+
fetch(url, options),
|
|
207
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), timeout)),
|
|
208
|
+
]);
|
|
209
|
+
const localResp = await fetchWithTimeout("http://127.0.0.1:9091/info", { method: "GET", headers: { Accept: "application/json" } }, 1000);
|
|
210
|
+
if (localResp.ok) {
|
|
211
|
+
const info = (await localResp.json());
|
|
212
|
+
if (info && (info.status === "ok" || info.status === 1) && info.uuid) {
|
|
213
|
+
if (this.deviceId === undefined || this.deviceId === info.uuid) {
|
|
214
|
+
const localUuid = info.uuid;
|
|
215
|
+
this.deviceId = localUuid;
|
|
216
|
+
fallbackUrl = `ws://127.0.0.1:9091/api/s/${localUuid}`;
|
|
217
|
+
this.logger.info("✅ [Auth] Local ShellX service available, using local /info returned uuid:", localUuid, ", Service address:", fallbackUrl);
|
|
218
|
+
return fallbackUrl;
|
|
177
219
|
}
|
|
178
220
|
}
|
|
179
221
|
}
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
console.error('❌ [Auth] ShellX.ai设备未注册或认证信息无效');
|
|
207
|
-
return `wss://shellx.ai/api/s/${deviceId}`;
|
|
208
|
-
}
|
|
209
|
-
// const jsonData = JSON.parse(data);
|
|
210
|
-
console.log('✅ [Auth] ShellX.ai设备认证成功');
|
|
211
|
-
console.log(`📡 [Auth] 设备ID: ${jsonData.authenticate}`);
|
|
212
|
-
console.log(`📡 [Auth] ShellX.ai服务地址: ${jsonData.machine}`);
|
|
213
|
-
console.log(`📡 [Auth] 注册时间: ${jsonData.registered_at}`);
|
|
214
|
-
console.log(`📡 [Auth] 最后更新: ${jsonData.last_updated}`);
|
|
215
|
-
return jsonData.machine + '/api/s/' + jsonData.authenticate;
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
// Local unavailable, proceeding with remote
|
|
225
|
+
}
|
|
226
|
+
if (this.deviceId === undefined) {
|
|
227
|
+
this.logger.warn("❌ [Auth] Device ID not set, local USB not connected, please set environment variable SHELLX_DEVICE_ID or connect USB device");
|
|
228
|
+
return undefined;
|
|
229
|
+
}
|
|
230
|
+
// 2. Remote authentication logic
|
|
231
|
+
try {
|
|
232
|
+
this.logger.info("🔑 [Auth] Authenticating device...");
|
|
233
|
+
const featGlobal = this.getFetch();
|
|
234
|
+
const deviceApiUrl = buildDeviceApiUrl(this.deviceId);
|
|
235
|
+
const jsonData = (await featGlobal(deviceApiUrl, {
|
|
236
|
+
method: "GET",
|
|
237
|
+
headers: {
|
|
238
|
+
"Content-Type": "application/json",
|
|
239
|
+
},
|
|
240
|
+
}));
|
|
241
|
+
this.logger.info("ShellX.ai device authentication response:", jsonData);
|
|
242
|
+
// Check if device is not registered
|
|
243
|
+
if (jsonData?.status === "not_registered") {
|
|
244
|
+
this.logger.error("❌ [Auth] Device not registered");
|
|
245
|
+
this.logger.error(`📝 [Auth] Status: ${jsonData.message || "Device not registered"}`);
|
|
246
|
+
this.logger.error("💡 [Auth] Please register this device on ShellX.ai platform first");
|
|
247
|
+
throw new Error(`Device "${this.deviceId}" not registered on ShellX.ai platform. Please visit https://shellx.ai to register the device first.`);
|
|
216
248
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
249
|
+
// Verify authentication response is valid
|
|
250
|
+
// If interface returns null or missing required fields, device is not registered
|
|
251
|
+
if (!jsonData || jsonData === null || !jsonData.machine || !jsonData.authenticate) {
|
|
252
|
+
this.logger.error("❌ [Auth] ShellX.ai device not registered or invalid authentication information");
|
|
253
|
+
throw new Error(`Device "${this.deviceId}" has invalid authentication information. Please check device ID or re-register on ShellX.ai platform.`);
|
|
221
254
|
}
|
|
222
|
-
|
|
255
|
+
// const jsonData = JSON.parse(data);
|
|
256
|
+
this.logger.info("✅ [Auth] ShellX.ai device authentication successful");
|
|
257
|
+
this.logger.info(`📡 [Auth] Device ID: ${jsonData.authenticate}`);
|
|
258
|
+
this.logger.info(`📡 [Auth] ShellX.ai service address: ${jsonData.machine}`);
|
|
259
|
+
this.logger.info(`📡 [Auth] Registration time: ${jsonData.registered_at}`);
|
|
260
|
+
this.logger.info(`📡 [Auth] Last update: ${jsonData.last_updated}`);
|
|
261
|
+
this.logger.info(`📡 [Auth] Device status: ${jsonData.status || "registered"}`);
|
|
262
|
+
// UUID is the deviceId, no need to extract from URL
|
|
263
|
+
const wsUrl = `${jsonData.machine}/api/s/${this.deviceId}`;
|
|
264
|
+
this.logger.info(`🔗 [Auth] WebSocket URL: ${wsUrl}`);
|
|
265
|
+
return wsUrl;
|
|
266
|
+
}
|
|
267
|
+
catch (error) {
|
|
268
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
269
|
+
this.logger.warn("⚠️ [Auth] Online authentication failed, using local service:", JSON.stringify(errorMessage));
|
|
270
|
+
return fallbackUrl;
|
|
271
|
+
}
|
|
223
272
|
}
|
|
224
273
|
getFetch() {
|
|
225
|
-
return (url, options) =>
|
|
226
|
-
//
|
|
227
|
-
if (typeof globalThis.fetch !==
|
|
228
|
-
|
|
229
|
-
const response =
|
|
230
|
-
|
|
274
|
+
return async (url, options) => {
|
|
275
|
+
// Try using global fetch
|
|
276
|
+
if (typeof globalThis.fetch !== "undefined") {
|
|
277
|
+
this.logger.info(`🌐 [Fetch] Request: ${options?.method || "GET"} ${url}`);
|
|
278
|
+
const response = await globalThis.fetch(url, options);
|
|
279
|
+
this.logger.info(`📡 [Fetch] Response: ${response.status} ${response.statusText}`);
|
|
231
280
|
if (!response.ok) {
|
|
232
281
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
233
282
|
}
|
|
234
283
|
return response.json();
|
|
235
284
|
}
|
|
236
285
|
try {
|
|
237
|
-
const { default: fetch } =
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
286
|
+
const { default: fetch } = await import("node-fetch");
|
|
287
|
+
this.logger.info(`🌐 [Fetch] Request: ${options?.method || "GET"} ${url}`);
|
|
288
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
289
|
+
const response = await fetch(url, options);
|
|
290
|
+
this.logger.info(`📡 [Fetch] Response: ${response.status} ${response.statusText}`);
|
|
241
291
|
if (!response.ok) {
|
|
242
292
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
243
293
|
}
|
|
244
294
|
return response.json();
|
|
245
295
|
}
|
|
246
296
|
catch (fetchError) {
|
|
247
|
-
|
|
297
|
+
const errorMessage = fetchError instanceof Error ? fetchError.message : String(fetchError);
|
|
298
|
+
throw new Error(`Fetch not available: ${errorMessage}`);
|
|
248
299
|
}
|
|
249
|
-
}
|
|
300
|
+
};
|
|
250
301
|
}
|
|
251
302
|
/**
|
|
252
|
-
*
|
|
303
|
+
* Wait for WebSocket initialization complete
|
|
253
304
|
*/
|
|
254
|
-
waitForInitialization() {
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
}
|
|
259
|
-
});
|
|
305
|
+
async waitForInitialization() {
|
|
306
|
+
if (this.initializationPromise) {
|
|
307
|
+
await this.initializationPromise;
|
|
308
|
+
}
|
|
260
309
|
}
|
|
261
310
|
/**
|
|
262
|
-
*
|
|
311
|
+
* Ensure connection is ready before executing device commands
|
|
312
|
+
* If not connected, wait for connection with timeout
|
|
313
|
+
*
|
|
314
|
+
* @param timeout - Timeout in milliseconds (default: 10000ms)
|
|
315
|
+
* @throws Error if connection timeout or connection failed
|
|
316
|
+
*
|
|
317
|
+
* @example
|
|
318
|
+
* ```typescript
|
|
319
|
+
* await client.ensureConnected(10000);
|
|
320
|
+
* console.log('Connection ready');
|
|
321
|
+
* ```
|
|
263
322
|
*/
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
323
|
+
async ensureConnected(timeout = 10000) {
|
|
324
|
+
// If already connected, return immediately
|
|
325
|
+
if (this.shellxConnected && this.wsConnected) {
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
this.logger.info(`⏳ [ConnectionClient] Waiting for connection to be ready (timeout: ${timeout}ms)...`);
|
|
329
|
+
const startTime = Date.now();
|
|
330
|
+
const checkInterval = 100; // Check every 100ms
|
|
331
|
+
return new Promise((resolve, reject) => {
|
|
332
|
+
const intervalId = setInterval(() => {
|
|
333
|
+
const elapsed = Date.now() - startTime;
|
|
334
|
+
// Check if connected
|
|
335
|
+
if (this.shellxConnected && this.wsConnected) {
|
|
336
|
+
clearInterval(intervalId);
|
|
337
|
+
this.logger.info(`✅ [ConnectionClient] Connection ready (took ${elapsed}ms)`);
|
|
338
|
+
resolve();
|
|
339
|
+
return;
|
|
271
340
|
}
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
341
|
+
// Check timeout
|
|
342
|
+
if (elapsed >= timeout) {
|
|
343
|
+
clearInterval(intervalId);
|
|
344
|
+
const errorMsg = `Connection timeout after ${timeout}ms. ` +
|
|
345
|
+
`Device "${this.deviceId}" is not connected. ` +
|
|
346
|
+
`Please check: ` +
|
|
347
|
+
`1. Device is online and registered on ShellX.ai ` +
|
|
348
|
+
`2. Network connection is stable ` +
|
|
349
|
+
`3. Device ID is correct`;
|
|
350
|
+
this.logger.error(`❌ [ConnectionClient] ${errorMsg}`);
|
|
351
|
+
reject(new Error(errorMsg));
|
|
352
|
+
return;
|
|
276
353
|
}
|
|
277
|
-
|
|
278
|
-
|
|
354
|
+
// Log progress every 2 seconds
|
|
355
|
+
if (elapsed > 0 && elapsed % 2000 < checkInterval) {
|
|
356
|
+
this.logger.info(`⏳ [ConnectionClient] Still waiting for connection... (${elapsed}/${timeout}ms)`);
|
|
279
357
|
}
|
|
280
|
-
}
|
|
281
|
-
catch (error) {
|
|
282
|
-
console.error('❌ [Auth] 发送认证消息失败:', error);
|
|
283
|
-
}
|
|
358
|
+
}, checkInterval);
|
|
284
359
|
});
|
|
285
360
|
}
|
|
286
361
|
handleMessage(event) {
|
|
@@ -300,11 +375,11 @@ class ConnectionTaskClient {
|
|
|
300
375
|
this.processServerMessage(serverMessage);
|
|
301
376
|
return;
|
|
302
377
|
}
|
|
303
|
-
serverMessage = (
|
|
378
|
+
serverMessage = cborDecode(binaryData);
|
|
304
379
|
this.processServerMessage(serverMessage);
|
|
305
380
|
}
|
|
306
381
|
catch (cborError) {
|
|
307
|
-
|
|
382
|
+
this.logger.info("CBOR decode failed, trying JSON fallback:", cborError);
|
|
308
383
|
try {
|
|
309
384
|
let textData;
|
|
310
385
|
if (event.data instanceof ArrayBuffer) {
|
|
@@ -317,316 +392,359 @@ class ConnectionTaskClient {
|
|
|
317
392
|
this.processServerMessage(serverMessage);
|
|
318
393
|
}
|
|
319
394
|
catch (jsonError) {
|
|
320
|
-
|
|
395
|
+
this.logger.error("Failed to parse message:", { cborError, jsonError });
|
|
321
396
|
}
|
|
322
397
|
}
|
|
323
398
|
}
|
|
324
399
|
processServerMessage(message) {
|
|
325
|
-
|
|
326
|
-
//console.log('📨 [ShellX] 收到服务器消息:', message);
|
|
327
|
-
this.authenticated = true;
|
|
328
|
-
// 只有在认证成功后才处理其他消息并认为连接成功
|
|
329
|
-
if (this.authenticated && !this.shellxConnected) {
|
|
330
|
-
this.shellxConnected = true;
|
|
331
|
-
console.log('✅ [ShellX] ShellX.ai服务连接成功!');
|
|
332
|
-
(_b = (_a = this.config).onOpen) === null || _b === void 0 ? void 0 : _b.call(_a);
|
|
333
|
-
this.flushQueue();
|
|
334
|
-
this.startPing();
|
|
335
|
-
}
|
|
400
|
+
//console.log('📨 [ShellX] Received server message:', message);
|
|
336
401
|
// Call user-defined message handler
|
|
337
|
-
|
|
402
|
+
this.config.onMessage?.(message);
|
|
338
403
|
// Handle specific message types and resolve pending tasks
|
|
339
404
|
if (message.jsonData) {
|
|
340
405
|
this.handleJsonDataResponse(message.jsonData);
|
|
341
406
|
}
|
|
342
407
|
// Handle other server message types (hello, users, shells, etc.)
|
|
343
408
|
if (message.hello !== undefined) {
|
|
344
|
-
|
|
409
|
+
this.logger.info("👋 [ShellX] Server greeting, user ID:", message.hello);
|
|
345
410
|
}
|
|
346
411
|
if (message.pong !== undefined) {
|
|
347
|
-
//console.log('🏓 [ShellX]
|
|
412
|
+
//console.log('🏓 [ShellX] Heartbeat response:', message.pong);
|
|
348
413
|
}
|
|
349
414
|
if (message.error) {
|
|
350
|
-
|
|
415
|
+
this.logger.error("❌ [ShellX] Server error:", message.error);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
matchResponseByType(jsonData, taskType) {
|
|
419
|
+
switch (taskType) {
|
|
420
|
+
case "findElement":
|
|
421
|
+
if (jsonData.findElement) {
|
|
422
|
+
return jsonData.findElement;
|
|
423
|
+
}
|
|
424
|
+
break;
|
|
425
|
+
case "waitElement":
|
|
426
|
+
if (jsonData.waitElement) {
|
|
427
|
+
return jsonData.waitElement;
|
|
428
|
+
}
|
|
429
|
+
break;
|
|
430
|
+
case "screenShot":
|
|
431
|
+
if (jsonData.screenShot) {
|
|
432
|
+
return jsonData.screenShot;
|
|
433
|
+
}
|
|
434
|
+
break;
|
|
435
|
+
case "screenInfo":
|
|
436
|
+
if (jsonData.screenInfo) {
|
|
437
|
+
return jsonData.screenInfo;
|
|
438
|
+
}
|
|
439
|
+
break;
|
|
440
|
+
case "appList":
|
|
441
|
+
console.log("appList");
|
|
442
|
+
console.log(jsonData);
|
|
443
|
+
if (jsonData.appList) {
|
|
444
|
+
return jsonData.appList;
|
|
445
|
+
}
|
|
446
|
+
break;
|
|
447
|
+
case "action":
|
|
448
|
+
if (jsonData.action_event) {
|
|
449
|
+
return jsonData.action_event;
|
|
450
|
+
}
|
|
451
|
+
break;
|
|
452
|
+
case "promptflowx":
|
|
453
|
+
if (jsonData.promptflowx) {
|
|
454
|
+
return jsonData.promptflowx;
|
|
455
|
+
}
|
|
456
|
+
break;
|
|
457
|
+
case "clipboard":
|
|
458
|
+
if (jsonData.clipboard) {
|
|
459
|
+
return jsonData.clipboard;
|
|
460
|
+
}
|
|
461
|
+
break;
|
|
462
|
+
case "asrControl":
|
|
463
|
+
// ASR control doesn't expect a specific response
|
|
464
|
+
return { success: true };
|
|
351
465
|
}
|
|
466
|
+
return undefined;
|
|
352
467
|
}
|
|
353
468
|
handleJsonDataResponse(jsonData) {
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
break;
|
|
365
|
-
case 'waitElement':
|
|
366
|
-
if (jsonData.waitElement) {
|
|
367
|
-
responseData = jsonData.waitElement;
|
|
368
|
-
shouldResolve = true;
|
|
369
|
-
}
|
|
370
|
-
break;
|
|
371
|
-
case 'screenShot':
|
|
372
|
-
if (jsonData.screenShot) {
|
|
373
|
-
responseData = jsonData.screenShot;
|
|
374
|
-
shouldResolve = true;
|
|
469
|
+
const responseTaskId = jsonData?.taskId;
|
|
470
|
+
if (responseTaskId) {
|
|
471
|
+
const task = this.pendingTasks.get(responseTaskId);
|
|
472
|
+
if (task) {
|
|
473
|
+
const responseData = this.matchResponseByType(jsonData, task.type);
|
|
474
|
+
if (responseData !== undefined) {
|
|
475
|
+
clearTimeout(task.timer);
|
|
476
|
+
this.pendingTasks.delete(responseTaskId);
|
|
477
|
+
if (task.type != "findElement") {
|
|
478
|
+
this.logger.info("✅ [ShellX] Response processing complete:", responseTaskId, jsonData);
|
|
375
479
|
}
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
responseData = jsonData.appList;
|
|
386
|
-
shouldResolve = true;
|
|
387
|
-
}
|
|
388
|
-
break;
|
|
389
|
-
case 'action':
|
|
390
|
-
if (jsonData.action_event) {
|
|
391
|
-
responseData = jsonData.action_event;
|
|
392
|
-
shouldResolve = true;
|
|
393
|
-
}
|
|
394
|
-
break;
|
|
395
|
-
case 'actions':
|
|
396
|
-
if (jsonData.actions) {
|
|
397
|
-
responseData = jsonData.actions;
|
|
398
|
-
shouldResolve = true;
|
|
399
|
-
}
|
|
400
|
-
break;
|
|
480
|
+
task.resolve(responseData);
|
|
481
|
+
return;
|
|
482
|
+
}
|
|
483
|
+
else {
|
|
484
|
+
this.logger.warn(`⚠️ [ShellX] Task ${responseTaskId} (type: ${task.type}) found but no matching response data in:`, jsonData);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
else {
|
|
488
|
+
this.logger.warn(`⚠️ [ShellX] Received taskId ${responseTaskId} but no pending task found`);
|
|
401
489
|
}
|
|
402
|
-
|
|
490
|
+
}
|
|
491
|
+
else {
|
|
492
|
+
this.logger.debug(`📨 [ShellX] Received response without taskId, trying fallback match:`, jsonData);
|
|
493
|
+
}
|
|
494
|
+
// Fallback to legacy matching when taskId is missing or not tracked
|
|
495
|
+
for (const [taskId, task] of this.pendingTasks.entries()) {
|
|
496
|
+
const responseData = this.matchResponseByType(jsonData, task.type);
|
|
497
|
+
if (responseData !== undefined) {
|
|
403
498
|
clearTimeout(task.timer);
|
|
404
499
|
this.pendingTasks.delete(taskId);
|
|
405
|
-
|
|
500
|
+
this.logger.info(`✅ [ShellX] Fallback match: completed task ${taskId} (type: ${task.type})`);
|
|
406
501
|
task.resolve(responseData);
|
|
407
|
-
break;
|
|
502
|
+
break;
|
|
408
503
|
}
|
|
409
504
|
}
|
|
410
505
|
}
|
|
411
|
-
sendMessage(message, taskType
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
this.pendingTasks.delete(taskId);
|
|
418
|
-
// 超时时返回 undefined 而不是 reject
|
|
419
|
-
console.log(`⏰ [ShellX] 任务超时: ${taskType}, 返回 undefined`);
|
|
420
|
-
resolve(undefined);
|
|
421
|
-
}, timeout ? timeout : this.config.timeout);
|
|
422
|
-
this.pendingTasks.set(taskId, {
|
|
423
|
-
resolve,
|
|
424
|
-
reject,
|
|
425
|
-
timer,
|
|
426
|
-
type: taskType
|
|
427
|
-
});
|
|
428
|
-
console.log(`📋 [ShellX] 创建任务: ${taskId}, 类型: ${taskType}}`);
|
|
429
|
-
}
|
|
430
|
-
if (this.shellxConnected && this.ws) {
|
|
431
|
-
try {
|
|
432
|
-
console.log('📤 [ShellX] 发送消息:', JSON.stringify(message));
|
|
433
|
-
this.ws.send((0, cbor_1.encode)(message));
|
|
434
|
-
if (!taskType)
|
|
435
|
-
resolve(undefined); // For fire-and-forget messages
|
|
436
|
-
}
|
|
437
|
-
catch (error) {
|
|
438
|
-
if (taskType) {
|
|
439
|
-
this.pendingTasks.delete(taskId);
|
|
440
|
-
}
|
|
441
|
-
// 发送失败时返回 undefined 而不是 reject
|
|
442
|
-
console.log(`❌ [ShellX] 发送消息失败,返回 undefined:`, error);
|
|
443
|
-
resolve(undefined);
|
|
444
|
-
}
|
|
445
|
-
}
|
|
446
|
-
else {
|
|
447
|
-
console.log('⏳ [ShellX] 连接未就绪,消息已加入队列');
|
|
448
|
-
this.reconnect();
|
|
449
|
-
this.messageQueue.push(message);
|
|
450
|
-
if (!taskType)
|
|
451
|
-
resolve(undefined);
|
|
452
|
-
}
|
|
453
|
-
});
|
|
454
|
-
});
|
|
506
|
+
async sendMessage(message, taskType) {
|
|
507
|
+
if (!taskType) {
|
|
508
|
+
taskType = "oneway";
|
|
509
|
+
}
|
|
510
|
+
const { promise } = await this.sendMessageWithTaskId(message, taskType);
|
|
511
|
+
return await promise;
|
|
455
512
|
}
|
|
456
513
|
// ===== PROTOCOL-AWARE TASK METHODS =====
|
|
457
514
|
/**
|
|
458
515
|
* Find UI elements on the screen
|
|
459
516
|
*/
|
|
460
|
-
findElement(selector, options) {
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
return this.sendMessage({ findElement: findAction }, 'findElement');
|
|
468
|
-
});
|
|
517
|
+
async findElement(selector, options) {
|
|
518
|
+
const findAction = {
|
|
519
|
+
type: "find",
|
|
520
|
+
selector,
|
|
521
|
+
options,
|
|
522
|
+
};
|
|
523
|
+
return this.sendMessage({ findElement: findAction }, "findElement");
|
|
469
524
|
}
|
|
470
525
|
/**
|
|
471
526
|
* Wait for UI element to appear
|
|
472
527
|
*/
|
|
473
|
-
waitElement(selector, options) {
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
return this.sendMessage({ waitElement: waitAction }, 'waitElement');
|
|
481
|
-
});
|
|
528
|
+
async waitElement(selector, options) {
|
|
529
|
+
const waitAction = {
|
|
530
|
+
type: "wait",
|
|
531
|
+
selector,
|
|
532
|
+
options,
|
|
533
|
+
};
|
|
534
|
+
return this.sendMessage({ waitElement: waitAction }, "waitElement");
|
|
482
535
|
}
|
|
483
536
|
/**
|
|
484
537
|
* Take a screenshot
|
|
538
|
+
* @deprecated Use takeScreenshot() instead for consistency
|
|
485
539
|
*/
|
|
486
|
-
screenShot(options) {
|
|
487
|
-
return
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
540
|
+
async screenShot(options) {
|
|
541
|
+
return this.takeScreenshot(options);
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* Take a screenshot
|
|
545
|
+
*/
|
|
546
|
+
async takeScreenshot(options) {
|
|
547
|
+
return this.sendMessage({
|
|
548
|
+
screenShot: options || {
|
|
549
|
+
format: "png",
|
|
550
|
+
quality: 30,
|
|
551
|
+
scale: 1.0,
|
|
552
|
+
region: {
|
|
553
|
+
left: 0,
|
|
554
|
+
top: 0,
|
|
555
|
+
width: 1080,
|
|
556
|
+
height: 2340,
|
|
499
557
|
},
|
|
500
|
-
},
|
|
501
|
-
});
|
|
558
|
+
},
|
|
559
|
+
}, "screenShot");
|
|
502
560
|
}
|
|
503
561
|
/**
|
|
504
562
|
* Get screen information
|
|
505
563
|
*/
|
|
506
|
-
getScreenInfo() {
|
|
507
|
-
return
|
|
508
|
-
return this.sendMessage({ screenInfo: {} }, 'screenInfo');
|
|
509
|
-
});
|
|
564
|
+
async getScreenInfo() {
|
|
565
|
+
return this.sendMessage({ screenInfo: { keepScreenOn: false, wakeApp: true } }, "screenInfo");
|
|
510
566
|
}
|
|
511
567
|
/**
|
|
512
|
-
* Get
|
|
568
|
+
* Get screen information
|
|
569
|
+
* @deprecated Use wakeScreenAndGetInfo() instead for clarity
|
|
513
570
|
*/
|
|
514
|
-
|
|
515
|
-
return
|
|
516
|
-
return this.sendMessage({ appList: options || {} }, 'appList');
|
|
517
|
-
});
|
|
571
|
+
async getAndWakeScreen() {
|
|
572
|
+
return this.wakeScreenAndGetInfo();
|
|
518
573
|
}
|
|
519
574
|
/**
|
|
520
|
-
*
|
|
575
|
+
* Wake screen and get screen information
|
|
521
576
|
*/
|
|
522
|
-
|
|
523
|
-
return
|
|
524
|
-
return this.sendMessage({ appInfo: { packageName } }, 'appInfo');
|
|
525
|
-
});
|
|
577
|
+
async wakeScreenAndGetInfo() {
|
|
578
|
+
return this.sendMessage({ screenInfo: { keepScreenOn: true, wakeApp: true } }, "screenInfo");
|
|
526
579
|
}
|
|
527
580
|
/**
|
|
528
581
|
* Execute an action sequence
|
|
529
582
|
*/
|
|
530
|
-
executeAction(actionSequence, taskId,
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
});
|
|
583
|
+
async executeAction(actionSequence, taskId, timeout) {
|
|
584
|
+
const { promise } = await this.sendMessageWithTaskId({ actions: actionSequence }, "action", { taskId, timeout });
|
|
585
|
+
return await promise;
|
|
534
586
|
}
|
|
535
587
|
/**
|
|
536
588
|
* Execute promptflow actions
|
|
537
589
|
*/
|
|
538
|
-
executePromptFlow(actionSequence) {
|
|
539
|
-
return
|
|
540
|
-
return this.sendMessage({ promptflowx: actionSequence });
|
|
541
|
-
});
|
|
590
|
+
async executePromptFlow(actionSequence) {
|
|
591
|
+
return this.sendMessage({ promptflowx: actionSequence });
|
|
542
592
|
}
|
|
543
593
|
/**
|
|
544
594
|
* Listen for display changes
|
|
545
595
|
*/
|
|
546
|
-
screenChange(options) {
|
|
547
|
-
return
|
|
548
|
-
|
|
549
|
-
screenChange: options || { enable: true },
|
|
550
|
-
});
|
|
596
|
+
async screenChange(options) {
|
|
597
|
+
return this.sendMessage({
|
|
598
|
+
screenChange: options || { enable: true },
|
|
551
599
|
});
|
|
552
600
|
}
|
|
553
601
|
/**
|
|
554
602
|
* Switch to a different node
|
|
555
603
|
*/
|
|
556
|
-
switchNode(nodeId) {
|
|
557
|
-
return
|
|
558
|
-
return this.sendMessage({ switchNode: nodeId });
|
|
559
|
-
});
|
|
604
|
+
async switchNode(nodeId) {
|
|
605
|
+
return this.sendMessage({ switchNode: nodeId });
|
|
560
606
|
}
|
|
561
607
|
/**
|
|
562
608
|
* Send authentication data
|
|
563
609
|
*/
|
|
564
|
-
authenticate(authData) {
|
|
565
|
-
return
|
|
566
|
-
return this.sendMessage({ authenticate: authData });
|
|
567
|
-
});
|
|
610
|
+
async authenticate(authData) {
|
|
611
|
+
return this.sendMessage({ authenticate: authData });
|
|
568
612
|
}
|
|
569
613
|
/**
|
|
570
614
|
* Set user name
|
|
571
615
|
*/
|
|
572
|
-
setName(name) {
|
|
573
|
-
return
|
|
574
|
-
return this.sendMessage({ setName: name });
|
|
575
|
-
});
|
|
616
|
+
async setName(name) {
|
|
617
|
+
return this.sendMessage({ setName: name });
|
|
576
618
|
}
|
|
577
619
|
/**
|
|
578
620
|
* Send chat message
|
|
579
621
|
*/
|
|
580
|
-
sendChat(message) {
|
|
581
|
-
return
|
|
582
|
-
|
|
583
|
-
|
|
622
|
+
async sendChat(message) {
|
|
623
|
+
return this.sendMessage({ chat: message });
|
|
624
|
+
}
|
|
625
|
+
/**
|
|
626
|
+
* Start ASR (Automatic Speech Recognition)
|
|
627
|
+
*/
|
|
628
|
+
async startAsr() {
|
|
629
|
+
return this.sendMessage({ asrControl: { action: "start_asr" } }, "oneway");
|
|
630
|
+
}
|
|
631
|
+
/**
|
|
632
|
+
* Stop ASR (Automatic Speech Recognition)
|
|
633
|
+
*/
|
|
634
|
+
async stopAsr() {
|
|
635
|
+
return this.sendMessage({ asrControl: { action: "stop_asr" } }, "oneway");
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* Speak text using TTS (Text-to-Speech)
|
|
639
|
+
*/
|
|
640
|
+
async speak(text) {
|
|
641
|
+
return this.sendMessage({ ttsControl: { action: "tts_speak", text } }, "oneway");
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Stop TTS (Text-to-Speech)
|
|
645
|
+
*/
|
|
646
|
+
async stopTts() {
|
|
647
|
+
return this.sendMessage({ ttsControl: { action: "tts_stop" } }, "oneway");
|
|
584
648
|
}
|
|
585
649
|
/**
|
|
586
650
|
* Send raw message (for custom integrations)
|
|
587
651
|
*/
|
|
588
|
-
sendRawMessage(message) {
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
652
|
+
async sendRawMessage(message) {
|
|
653
|
+
// Auto-detect task type from message content
|
|
654
|
+
const taskType = this.detectTaskType(message);
|
|
655
|
+
return this.sendMessage(message, taskType);
|
|
592
656
|
}
|
|
593
657
|
/**
|
|
594
|
-
*
|
|
595
|
-
* @param message 要发送的消息
|
|
596
|
-
* @param taskType 任务类型
|
|
597
|
-
* @returns Promise<{taskId: string, promise: Promise<any>}>
|
|
658
|
+
* Detect task type from message content
|
|
598
659
|
*/
|
|
599
|
-
|
|
660
|
+
detectTaskType(message) {
|
|
661
|
+
if (message.findElement)
|
|
662
|
+
return "findElement";
|
|
663
|
+
if (message.waitElement)
|
|
664
|
+
return "waitElement";
|
|
665
|
+
if (message.screenShot)
|
|
666
|
+
return "screenShot";
|
|
667
|
+
if (message.screenInfo)
|
|
668
|
+
return "screenInfo";
|
|
669
|
+
if (message.appList)
|
|
670
|
+
return "appList";
|
|
671
|
+
if (message.action)
|
|
672
|
+
return "action";
|
|
673
|
+
if (message.actions)
|
|
674
|
+
return "action";
|
|
675
|
+
if (message.promptflowx)
|
|
676
|
+
return "promptflowx";
|
|
677
|
+
if (message.asrControl)
|
|
678
|
+
return "asrControl";
|
|
679
|
+
// Oneway messages (fire-and-forget, no response expected)
|
|
680
|
+
if (message.ttsControl)
|
|
681
|
+
return "oneway";
|
|
682
|
+
if (message.screenChange)
|
|
683
|
+
return "oneway";
|
|
684
|
+
if (message.switchNode)
|
|
685
|
+
return "oneway";
|
|
686
|
+
if (message.authenticate)
|
|
687
|
+
return "oneway";
|
|
688
|
+
if (message.setName)
|
|
689
|
+
return "oneway";
|
|
690
|
+
if (message.chat)
|
|
691
|
+
return "oneway";
|
|
692
|
+
if (message.ping)
|
|
693
|
+
return "oneway";
|
|
694
|
+
// Default to oneway for unknown message types
|
|
695
|
+
return "oneway";
|
|
696
|
+
}
|
|
697
|
+
/**
|
|
698
|
+
* Send message and return taskId
|
|
699
|
+
* @param message Message to send
|
|
700
|
+
* @param taskType Task type
|
|
701
|
+
* @param options Optional parameters { timeout, taskId }
|
|
702
|
+
* @returns Promise<{taskId: string, promise: Promise<T>>}
|
|
703
|
+
*/
|
|
704
|
+
async sendMessageWithTaskId(message, taskType, options) {
|
|
600
705
|
return new Promise((resolve) => {
|
|
706
|
+
const { timeout, taskId: definedTaskId } = options || {};
|
|
601
707
|
let taskId = definedTaskId;
|
|
602
708
|
if (taskId == null) {
|
|
603
|
-
taskId = (
|
|
709
|
+
taskId = uuidv4();
|
|
604
710
|
}
|
|
711
|
+
this.logger.info(`📋 [ShellX] Created task: ${taskId}, Task type: ${taskType}`);
|
|
712
|
+
this.logger.info(message);
|
|
713
|
+
const outgoingMessage = taskId ? { ...message, taskId } : message;
|
|
605
714
|
if (isPendingTask(taskType)) {
|
|
606
715
|
const timer = setTimeout(() => {
|
|
607
716
|
if (taskId != null) {
|
|
717
|
+
const task = this.pendingTasks.get(taskId);
|
|
718
|
+
if (task) {
|
|
719
|
+
// Create a timeout result based on task type
|
|
720
|
+
const timeoutResult = this.createTimeoutResult(taskType, timeout ?? this.config.timeout);
|
|
721
|
+
// Resolve with timeout result instead of rejecting
|
|
722
|
+
task.resolve(timeoutResult);
|
|
723
|
+
}
|
|
608
724
|
this.pendingTasks.delete(taskId);
|
|
609
725
|
}
|
|
610
|
-
|
|
611
|
-
}, timeout
|
|
726
|
+
this.logger.info(`⏰ [ShellX] Task timeout: ${taskId}, Task type: ${taskType}`);
|
|
727
|
+
}, timeout ?? this.config.timeout);
|
|
612
728
|
this.pendingTasks.set(taskId, {
|
|
613
729
|
resolve: () => { },
|
|
614
730
|
reject: () => { },
|
|
615
731
|
timer,
|
|
616
732
|
type: taskType,
|
|
617
733
|
});
|
|
618
|
-
|
|
734
|
+
this.logger.info(`📋 [ShellX] Created manually controllable task: ${taskId}, Task type: ${taskType}}`);
|
|
619
735
|
}
|
|
620
736
|
if (this.shellxConnected && this.ws) {
|
|
621
737
|
try {
|
|
622
|
-
|
|
623
|
-
this.ws.send((
|
|
738
|
+
this.logger.info(" 📤 [ShellX] Sending message:", JSON.stringify(outgoingMessage));
|
|
739
|
+
this.ws.send(cborEncode(outgoingMessage));
|
|
624
740
|
const promise = new Promise((promiseResolve, promiseReject) => {
|
|
625
741
|
if (isPendingTask(taskType)) {
|
|
626
742
|
if (taskId != null) {
|
|
627
743
|
const task = this.pendingTasks.get(taskId);
|
|
628
744
|
if (task) {
|
|
745
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any
|
|
629
746
|
task.resolve = promiseResolve;
|
|
747
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any
|
|
630
748
|
task.reject = promiseReject;
|
|
631
749
|
}
|
|
632
750
|
}
|
|
@@ -641,12 +759,14 @@ class ConnectionTaskClient {
|
|
|
641
759
|
if (isPendingTask(taskType)) {
|
|
642
760
|
this.pendingTasks.delete(taskId);
|
|
643
761
|
}
|
|
644
|
-
|
|
762
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
763
|
+
resolve({ taskId, promise: Promise.reject(err) });
|
|
645
764
|
}
|
|
646
765
|
}
|
|
647
766
|
else {
|
|
648
|
-
|
|
649
|
-
this.messageQueue.push(
|
|
767
|
+
this.logger.info("⏳ [ShellX] Connection not ready, message queued");
|
|
768
|
+
this.messageQueue.push(outgoingMessage);
|
|
769
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
|
|
650
770
|
resolve({ taskId, promise: Promise.resolve(undefined) });
|
|
651
771
|
}
|
|
652
772
|
});
|
|
@@ -654,10 +774,10 @@ class ConnectionTaskClient {
|
|
|
654
774
|
// ===== CONNECTION MANAGEMENT =====
|
|
655
775
|
startPing() {
|
|
656
776
|
this.stopPing();
|
|
657
|
-
|
|
777
|
+
this.logger.info("🏓 [ShellX] Starting heartbeat detection...");
|
|
658
778
|
this.pingIntervalId = setInterval(() => {
|
|
659
779
|
if (this.ws && this.shellxConnected) {
|
|
660
|
-
this.ws.send((
|
|
780
|
+
this.ws.send(cborEncode({ ping: Date.now() }));
|
|
661
781
|
}
|
|
662
782
|
}, this.config.pingInterval);
|
|
663
783
|
}
|
|
@@ -667,32 +787,37 @@ class ConnectionTaskClient {
|
|
|
667
787
|
this.pingIntervalId = null;
|
|
668
788
|
}
|
|
669
789
|
}
|
|
670
|
-
reconnect() {
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
790
|
+
async reconnect() {
|
|
791
|
+
// TODO: Local needs reconnection
|
|
792
|
+
this.logger.info(`🔄 [ShellX] Reconnecting... (${this.reconnectAttempts}/${this.config.reconnectMaxAttempts})` +
|
|
793
|
+
this.wsUrl);
|
|
794
|
+
if (this.reconnectAttempts >= this.config.reconnectMaxAttempts) {
|
|
795
|
+
this.logger.error("❌ [ShellX] Reconnection failed, max attempts reached");
|
|
796
|
+
this.config.onReconnectFailed?.();
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
this.reconnectAttempts++;
|
|
800
|
+
// Exponential backoff strategy: baseInterval * 2^(attempts-1), with maximum delay cap and random jitter
|
|
801
|
+
const baseInterval = this.config.reconnectInterval || 1000;
|
|
802
|
+
const maxDelay = 30000; // Maximum delay 30 seconds
|
|
803
|
+
const exponentialDelay = baseInterval * Math.pow(2, this.reconnectAttempts - 1);
|
|
804
|
+
const delay = Math.min(exponentialDelay, maxDelay);
|
|
805
|
+
// Add random jitter (±20%) to avoid multiple clients reconnecting simultaneously
|
|
806
|
+
const jitter = delay * 0.2 * (Math.random() * 2 - 1);
|
|
807
|
+
const finalDelay = Math.max(0, delay + jitter);
|
|
808
|
+
await wait(Math.floor(finalDelay));
|
|
809
|
+
void this.init();
|
|
685
810
|
}
|
|
686
811
|
flushQueue() {
|
|
687
812
|
while (this.messageQueue.length > 0 && this.ws && this.shellxConnected) {
|
|
688
813
|
const message = this.messageQueue.shift();
|
|
689
814
|
if (message) {
|
|
690
815
|
try {
|
|
691
|
-
|
|
692
|
-
this.ws.send((
|
|
816
|
+
this.logger.info("flushQueue Sending message:", message);
|
|
817
|
+
this.ws.send(cborEncode(message));
|
|
693
818
|
}
|
|
694
819
|
catch (error) {
|
|
695
|
-
|
|
820
|
+
this.logger.error("Failed to send queued message:", error);
|
|
696
821
|
// Re-queue the message if sending fails
|
|
697
822
|
this.messageQueue.unshift(message);
|
|
698
823
|
break;
|
|
@@ -701,11 +826,10 @@ class ConnectionTaskClient {
|
|
|
701
826
|
}
|
|
702
827
|
}
|
|
703
828
|
close() {
|
|
704
|
-
var _a;
|
|
705
829
|
this.shellxConnected = false;
|
|
706
830
|
this.wsConnected = false;
|
|
707
831
|
this.authenticated = false;
|
|
708
|
-
|
|
832
|
+
this.ws?.close();
|
|
709
833
|
this.pendingTasks.clear();
|
|
710
834
|
this.messageQueue = [];
|
|
711
835
|
this.stopPing();
|
|
@@ -720,6 +844,12 @@ class ConnectionTaskClient {
|
|
|
720
844
|
get isAuthenticated() {
|
|
721
845
|
return this.authenticated;
|
|
722
846
|
}
|
|
847
|
+
/**
|
|
848
|
+
* Get the authenticated device ID (UUID)
|
|
849
|
+
*/
|
|
850
|
+
getDeviceId() {
|
|
851
|
+
return this.deviceId;
|
|
852
|
+
}
|
|
723
853
|
get pendingTaskCount() {
|
|
724
854
|
return this.pendingTasks.size;
|
|
725
855
|
}
|
|
@@ -727,36 +857,128 @@ class ConnectionTaskClient {
|
|
|
727
857
|
return this.messageQueue.length;
|
|
728
858
|
}
|
|
729
859
|
/**
|
|
730
|
-
*
|
|
860
|
+
* Get current execution log
|
|
861
|
+
*/
|
|
862
|
+
getExecutionLogs() {
|
|
863
|
+
return [...this.executionLogsRef];
|
|
864
|
+
}
|
|
865
|
+
/**
|
|
866
|
+
* Clear current execution log
|
|
867
|
+
*/
|
|
868
|
+
clearExecutionLogs() {
|
|
869
|
+
this.executionLogsRef.length = 0;
|
|
870
|
+
}
|
|
871
|
+
/**
|
|
872
|
+
* Manually append an execution log
|
|
873
|
+
*/
|
|
874
|
+
appendExecutionLog(log) {
|
|
875
|
+
this.logger.info("Append log:", log);
|
|
876
|
+
this.executionLogsRef.push(log);
|
|
877
|
+
}
|
|
878
|
+
/**
|
|
879
|
+
* Set associated ShellX instance
|
|
731
880
|
*/
|
|
732
881
|
setShellX(shellx) {
|
|
733
882
|
this.shellx = shellx;
|
|
734
|
-
|
|
883
|
+
this.logger.info("🔗 [ShellX] ShellX instance associated");
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
886
|
+
* Create a timeout result object based on task type
|
|
887
|
+
* @param taskType Type of the task that timed out
|
|
888
|
+
* @param timeoutMs Timeout duration in milliseconds
|
|
889
|
+
* @returns A result object with success: false
|
|
890
|
+
*/
|
|
891
|
+
createTimeoutResult(taskType, timeoutMs) {
|
|
892
|
+
const baseResult = {
|
|
893
|
+
success: false,
|
|
894
|
+
error: `Task timed out after ${timeoutMs}ms`,
|
|
895
|
+
duration: timeoutMs,
|
|
896
|
+
timestamp: Date.now(),
|
|
897
|
+
};
|
|
898
|
+
// Return type-specific result based on task type
|
|
899
|
+
switch (taskType) {
|
|
900
|
+
case "action":
|
|
901
|
+
// For action tasks, return a basic result
|
|
902
|
+
return baseResult;
|
|
903
|
+
case "screenShot":
|
|
904
|
+
return {
|
|
905
|
+
...baseResult,
|
|
906
|
+
imageData: "",
|
|
907
|
+
imagePath: "",
|
|
908
|
+
format: "png",
|
|
909
|
+
};
|
|
910
|
+
case "screenInfo":
|
|
911
|
+
return {
|
|
912
|
+
...baseResult,
|
|
913
|
+
width: 0,
|
|
914
|
+
height: 0,
|
|
915
|
+
density: 0,
|
|
916
|
+
screenOn: false,
|
|
917
|
+
screenUnlocked: false,
|
|
918
|
+
};
|
|
919
|
+
case "findElement":
|
|
920
|
+
case "waitElement":
|
|
921
|
+
return {
|
|
922
|
+
...baseResult,
|
|
923
|
+
elements: [],
|
|
924
|
+
};
|
|
925
|
+
case "appList":
|
|
926
|
+
return {
|
|
927
|
+
...baseResult,
|
|
928
|
+
apps: [],
|
|
929
|
+
totalCount: 0,
|
|
930
|
+
systemAppCount: 0,
|
|
931
|
+
userAppCount: 0,
|
|
932
|
+
};
|
|
933
|
+
case "input":
|
|
934
|
+
return baseResult;
|
|
935
|
+
case "swipe":
|
|
936
|
+
return {
|
|
937
|
+
...baseResult,
|
|
938
|
+
fromX: 0,
|
|
939
|
+
fromY: 0,
|
|
940
|
+
toX: 0,
|
|
941
|
+
toY: 0,
|
|
942
|
+
};
|
|
943
|
+
case "press":
|
|
944
|
+
return {
|
|
945
|
+
...baseResult,
|
|
946
|
+
key: "",
|
|
947
|
+
};
|
|
948
|
+
case "clipboard":
|
|
949
|
+
return {
|
|
950
|
+
...baseResult,
|
|
951
|
+
text: "",
|
|
952
|
+
};
|
|
953
|
+
default:
|
|
954
|
+
// Default fallback result
|
|
955
|
+
return baseResult;
|
|
956
|
+
}
|
|
735
957
|
}
|
|
736
958
|
/**
|
|
737
|
-
*
|
|
959
|
+
* Get associated ShellX instance
|
|
738
960
|
*/
|
|
739
961
|
getShellX() {
|
|
740
962
|
return this.shellx;
|
|
741
963
|
}
|
|
742
964
|
/**
|
|
743
|
-
*
|
|
744
|
-
* @param taskId
|
|
745
|
-
* @param result
|
|
746
|
-
* @param success
|
|
965
|
+
* Manually complete task by taskId
|
|
966
|
+
* @param taskId Task ID
|
|
967
|
+
* @param result Task result
|
|
968
|
+
* @param success Success status
|
|
747
969
|
*/
|
|
748
970
|
completeTask(taskId, result, success = true) {
|
|
749
971
|
const task = this.pendingTasks.get(taskId);
|
|
750
972
|
if (!task) {
|
|
751
|
-
|
|
973
|
+
this.logger.info(`⚠️ [ShellX] Task not found: ${taskId}`);
|
|
752
974
|
return false;
|
|
753
975
|
}
|
|
754
|
-
|
|
755
|
-
//
|
|
976
|
+
this.logger.info(`✅ [ShellX] Manually completed task: ${taskId}, Task type: ${task.type}, Success: ${success}`);
|
|
977
|
+
// Clear timeout timer
|
|
756
978
|
clearTimeout(task.timer);
|
|
757
|
-
//
|
|
979
|
+
// Remove from pending tasks
|
|
758
980
|
this.pendingTasks.delete(taskId);
|
|
759
|
-
//
|
|
981
|
+
// Call appropriate callback based on success status
|
|
760
982
|
if (success) {
|
|
761
983
|
task.resolve(result || { success: true, taskId });
|
|
762
984
|
}
|
|
@@ -766,13 +988,13 @@ class ConnectionTaskClient {
|
|
|
766
988
|
return true;
|
|
767
989
|
}
|
|
768
990
|
/**
|
|
769
|
-
*
|
|
991
|
+
* Get all pending task IDs
|
|
770
992
|
*/
|
|
771
993
|
getPendingTaskIds() {
|
|
772
994
|
return Array.from(this.pendingTasks.keys());
|
|
773
995
|
}
|
|
774
996
|
/**
|
|
775
|
-
*
|
|
997
|
+
* Get task info by taskId
|
|
776
998
|
*/
|
|
777
999
|
getTaskInfo(taskId) {
|
|
778
1000
|
const task = this.pendingTasks.get(taskId);
|
|
@@ -780,14 +1002,14 @@ class ConnectionTaskClient {
|
|
|
780
1002
|
return null;
|
|
781
1003
|
}
|
|
782
1004
|
return {
|
|
783
|
-
type: task.type
|
|
1005
|
+
type: task.type,
|
|
784
1006
|
};
|
|
785
1007
|
}
|
|
786
1008
|
/**
|
|
787
|
-
*
|
|
788
|
-
* @param taskType
|
|
789
|
-
* @param result
|
|
790
|
-
* @param success
|
|
1009
|
+
* Batch complete tasks by type
|
|
1010
|
+
* @param taskType Task type
|
|
1011
|
+
* @param result Task result
|
|
1012
|
+
* @param success Success status
|
|
791
1013
|
*/
|
|
792
1014
|
completeTasksByType(taskType, result, success = true) {
|
|
793
1015
|
const taskIds = this.getPendingTaskIds();
|
|
@@ -799,12 +1021,16 @@ class ConnectionTaskClient {
|
|
|
799
1021
|
completedCount++;
|
|
800
1022
|
}
|
|
801
1023
|
}
|
|
802
|
-
|
|
1024
|
+
this.logger.info(`📦 [ShellX] Batch completed ${completedCount} tasks of type ${taskType}`);
|
|
803
1025
|
return completedCount;
|
|
804
1026
|
}
|
|
805
1027
|
}
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
1028
|
+
export default ConnectionClient;
|
|
1029
|
+
export { ShellX } from "./shellx.js";
|
|
1030
|
+
// Error handling exports
|
|
1031
|
+
export { createErrorResponse, createOperationResult, ErrorCode, ShellXError, ConnectionError, OperationError, ElementError, ValidationError, } from "./errors.js";
|
|
1032
|
+
// Error handler utilities
|
|
1033
|
+
export { extractErrorMessage, createErrorResult, createSuccessResult, handleOperation, handleOperationWithRetry, validateRequired, validateOneOfRequired, wrapErrorWithContext, } from "./error-handler.js";
|
|
1034
|
+
// Domain management exports
|
|
1035
|
+
export { buildDeviceApiUrl, buildDeviceConfigUrl, buildDeviceRegisterUrl, buildTokenVerifyUrl, buildTokenRefreshUrl, getApiBaseUrl, getWsServerUrl, getCurrentDomainInfo, createDomainConfig, isChineseUser, } from "./domain-manager.js";
|
|
1036
|
+
//# sourceMappingURL=index.js.map
|