shellx-ai 1.0.11 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +586 -0
- package/dist/automation/element-finder.d.ts +189 -0
- package/dist/automation/element-finder.js +322 -0
- package/dist/automation/element-finder.js.map +1 -0
- package/dist/automation/ui-action-handler.d.ts +330 -0
- package/dist/automation/ui-action-handler.js +873 -0
- package/dist/automation/ui-action-handler.js.map +1 -0
- package/dist/cbor-compat.d.ts +27 -0
- package/dist/cbor-compat.js +108 -0
- package/dist/cbor-compat.js.map +1 -0
- package/dist/domain-manager.d.ts +80 -0
- package/dist/domain-manager.js +158 -0
- package/dist/domain-manager.js.map +1 -0
- package/dist/error-handler.d.ts +87 -0
- package/dist/error-handler.js +148 -0
- package/dist/error-handler.js.map +1 -0
- package/dist/errors.d.ts +114 -0
- package/dist/errors.js +139 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +163 -54
- package/dist/index.js +712 -472
- package/dist/index.js.map +1 -0
- package/dist/logger.d.ts +81 -0
- package/dist/logger.js +128 -0
- package/dist/logger.js.map +1 -0
- package/dist/protocol.d.ts +147 -31
- package/dist/protocol.js +2 -2
- package/dist/protocol.js.map +1 -0
- package/dist/shell/output-buffer.d.ts +152 -0
- package/dist/shell/output-buffer.js +163 -0
- package/dist/shell/output-buffer.js.map +1 -0
- package/dist/shell/shell-command-executor.d.ts +182 -0
- package/dist/shell/shell-command-executor.js +348 -0
- package/dist/shell/shell-command-executor.js.map +1 -0
- package/dist/shellx.d.ts +681 -176
- package/dist/shellx.js +763 -1047
- package/dist/shellx.js.map +1 -0
- package/dist/types.d.ts +132 -57
- package/dist/types.js +4 -4
- package/dist/types.js.map +1 -0
- package/dist/utils/retry-helper.d.ts +73 -0
- package/dist/utils/retry-helper.js +92 -0
- package/dist/utils/retry-helper.js.map +1 -0
- package/dist/utils.d.ts +3 -3
- package/dist/utils.js +17 -23
- package/dist/utils.js.map +1 -0
- package/package.json +95 -59
package/dist/index.js
CHANGED
|
@@ -1,52 +1,41 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
-
});
|
|
1
|
+
import { v4 as uuidv4 } from "uuid";
|
|
2
|
+
import { wait } from "./utils.js";
|
|
3
|
+
import { buildDeviceApiUrl } from "./domain-manager.js";
|
|
4
|
+
import { createLogger } from "./logger.js";
|
|
5
|
+
// Add readable timestamp to all logs
|
|
6
|
+
const formatTs = () => {
|
|
7
|
+
const now = new Date();
|
|
8
|
+
return now.toISOString();
|
|
9
|
+
};
|
|
10
|
+
const executionLogStore = [];
|
|
11
|
+
const formatLogArgs = (args) => args
|
|
12
|
+
.map((arg) => {
|
|
13
|
+
if (typeof arg === "string")
|
|
14
|
+
return arg;
|
|
15
|
+
try {
|
|
16
|
+
return JSON.stringify(arg);
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return String(arg);
|
|
20
|
+
}
|
|
21
|
+
})
|
|
22
|
+
.join(" ");
|
|
23
|
+
const pushExecutionLog = (ts, args) => {
|
|
24
|
+
executionLogStore.push(`[${ts}] ${formatLogArgs(args)}`);
|
|
25
|
+
};
|
|
26
|
+
const originalConsoleLog = console.log.bind(console);
|
|
27
|
+
const originalConsoleWarn = console.warn.bind(console);
|
|
28
|
+
const originalConsoleError = console.error.bind(console);
|
|
29
|
+
const wrapConsole = (originalFn) => (...args) => {
|
|
30
|
+
const ts = formatTs();
|
|
31
|
+
pushExecutionLog(ts, args);
|
|
32
|
+
originalFn(`[${ts}]`, ...args);
|
|
43
33
|
};
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
exports.DEFAULT_CONFIG = {
|
|
34
|
+
console.log = wrapConsole(originalConsoleLog);
|
|
35
|
+
console.warn = wrapConsole(originalConsoleWarn);
|
|
36
|
+
console.error = wrapConsole(originalConsoleError);
|
|
37
|
+
// Define default configuration
|
|
38
|
+
export const DEFAULT_CONFIG = {
|
|
50
39
|
timeout: 5000,
|
|
51
40
|
reconnect: true,
|
|
52
41
|
reconnectMaxAttempts: 5,
|
|
@@ -57,219 +46,316 @@ exports.DEFAULT_CONFIG = {
|
|
|
57
46
|
onError: () => { },
|
|
58
47
|
onReconnectFailed: () => { },
|
|
59
48
|
};
|
|
60
|
-
|
|
61
|
-
// 安全地获取环境变量,兼容浏览器和Node.js环境
|
|
62
|
-
let authKey = (0, utils_1.getEnvVar)('SHELLX_AUTH_KEY');
|
|
49
|
+
import { initCbor, cborEncode, cborDecode } from "./cbor-compat.js";
|
|
63
50
|
/**
|
|
64
|
-
*
|
|
51
|
+
* Get WebSocket constructor for current environment
|
|
65
52
|
*/
|
|
66
|
-
function getWebSocket() {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}
|
|
82
|
-
});
|
|
53
|
+
async function getWebSocket() {
|
|
54
|
+
// Check if global WebSocket exists (browser environment or Node.js 20+)
|
|
55
|
+
if (typeof globalThis.WebSocket !== "undefined") {
|
|
56
|
+
return globalThis.WebSocket;
|
|
57
|
+
}
|
|
58
|
+
// Dynamically import ws module in Node.js environment
|
|
59
|
+
try {
|
|
60
|
+
const wsModule = await import("ws");
|
|
61
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
|
|
62
|
+
return (wsModule.default || wsModule);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
console.error("❌ [WebSocket] ws module is not available, please install: npm install ws");
|
|
66
|
+
throw new Error("WebSocket not available. Please install ws module: npm install ws");
|
|
67
|
+
}
|
|
83
68
|
}
|
|
84
69
|
function isPendingTask(taskType) {
|
|
85
|
-
return taskType && taskType !== "command";
|
|
70
|
+
return taskType && taskType !== "command" && taskType !== "oneway";
|
|
86
71
|
}
|
|
87
72
|
/**
|
|
88
|
-
*
|
|
73
|
+
* Low-level WebSocket Client for direct protocol communication
|
|
74
|
+
*
|
|
75
|
+
* @warning This is a low-level API intended for advanced use cases only.
|
|
76
|
+
* Most users should use the ShellX class instead, which provides a simpler,
|
|
77
|
+
* more intuitive interface with automatic error handling and retry logic.
|
|
78
|
+
*
|
|
79
|
+
* @example
|
|
80
|
+
* ```typescript
|
|
81
|
+
* // For most users, use ShellX instead:
|
|
82
|
+
* import { ShellX } from './shellx';
|
|
83
|
+
* const shellx = new ShellX({ deviceId: 'your-device-id' });
|
|
84
|
+
* await shellx.connect();
|
|
85
|
+
* await shellx.click({ text: 'Submit' });
|
|
86
|
+
* ```
|
|
87
|
+
*
|
|
88
|
+
* @see Use ShellX for a high-level API unless you need low-level WebSocket access
|
|
89
89
|
*/
|
|
90
|
-
class
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
90
|
+
export class ConnectionClient {
|
|
91
|
+
wsUrl;
|
|
92
|
+
deviceId;
|
|
93
|
+
config;
|
|
94
|
+
ws = null;
|
|
95
|
+
shellxConnected = false; // ShellX.ai connection status
|
|
96
|
+
wsConnected = false; // WebSocket connection status
|
|
97
|
+
authenticated = false; // Authentication status
|
|
98
|
+
pendingTasks = new Map();
|
|
99
|
+
messageQueue = [];
|
|
100
|
+
reconnectAttempts = 0;
|
|
101
|
+
pingIntervalId = null;
|
|
102
|
+
initializationPromise = null;
|
|
103
|
+
shellx = null; // Associated ShellX instance
|
|
104
|
+
executionLogsRef = executionLogStore; // Current execution log
|
|
105
|
+
logger;
|
|
106
|
+
constructor(deviceId, config = {}, logger) {
|
|
102
107
|
this.deviceId = deviceId;
|
|
103
|
-
this.config =
|
|
108
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
109
|
+
// Use provided logger or create default one
|
|
110
|
+
this.logger = logger || createLogger("ConnectionClient");
|
|
104
111
|
this.initializationPromise = this.init();
|
|
105
112
|
}
|
|
106
|
-
init() {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
if (this.ws.binaryType !== undefined) {
|
|
117
|
-
this.ws.binaryType = 'arraybuffer';
|
|
113
|
+
async init() {
|
|
114
|
+
await initCbor(); // load the right CBOR codec for this environment
|
|
115
|
+
return new Promise((resolve, reject) => {
|
|
116
|
+
this.authenticateDevice()
|
|
117
|
+
.then((wsUrl) => {
|
|
118
|
+
if (!wsUrl) {
|
|
119
|
+
this.config.onError(new Event("connection"));
|
|
120
|
+
void this.reconnect();
|
|
121
|
+
reject(new Error("Failed to authenticate device"));
|
|
122
|
+
return;
|
|
118
123
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
this.
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
124
|
+
// ✅ Store the WebSocket URL for later use
|
|
125
|
+
this.wsUrl = wsUrl;
|
|
126
|
+
const separator = wsUrl.includes("?") ? "&" : "?";
|
|
127
|
+
const wsUrlWithSessionId = `${wsUrl}${separator}session_id=${this.deviceId}`;
|
|
128
|
+
this.logger.info("Initializing ShellX client wsUrl:" + wsUrl + " deviceId: " + this.deviceId);
|
|
129
|
+
this.logger.info("WebSocket URL with session_id: " + wsUrlWithSessionId);
|
|
130
|
+
// Get WebSocket constructor for current environment
|
|
131
|
+
getWebSocket()
|
|
132
|
+
.then((WebSocketConstructor) => {
|
|
133
|
+
try {
|
|
134
|
+
this.ws = new WebSocketConstructor(wsUrlWithSessionId);
|
|
135
|
+
// Set binary type (if supported)
|
|
136
|
+
if (this.ws.binaryType !== undefined) {
|
|
137
|
+
this.ws.binaryType = "arraybuffer";
|
|
138
|
+
}
|
|
139
|
+
this.ws.onopen = () => {
|
|
140
|
+
this.logger.info("🔗 [ShellX] WebSocket connection established:" + this.wsUrl);
|
|
141
|
+
this.wsConnected = true;
|
|
142
|
+
this.reconnectAttempts = 0;
|
|
143
|
+
// Mark as connected after WebSocket open
|
|
144
|
+
this.shellxConnected = true;
|
|
145
|
+
this.config.onOpen?.(this.deviceId);
|
|
146
|
+
void this.flushQueue();
|
|
147
|
+
void this.startPing();
|
|
148
|
+
// Resolve initialization promise after WebSocket is connected
|
|
149
|
+
resolve();
|
|
150
|
+
// Check if we're in Android sandbox environment
|
|
151
|
+
const isAndroidSandbox = process.env["SANDBOX"] === "android";
|
|
152
|
+
if (isAndroidSandbox) {
|
|
153
|
+
void this.getScreenInfo();
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
void this.getAndWakeScreen();
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
this.ws.onmessage = (event) => {
|
|
160
|
+
void this.handleMessage(event);
|
|
161
|
+
};
|
|
162
|
+
this.ws.onclose = (event) => {
|
|
163
|
+
this.logger.info("❌ [ShellX] WebSocket connection closed, attempting to reconnect..." +
|
|
164
|
+
event.code +
|
|
165
|
+
" " +
|
|
166
|
+
this.wsUrl);
|
|
167
|
+
this.wsConnected = false;
|
|
168
|
+
this.shellxConnected = false;
|
|
169
|
+
this.authenticated = false;
|
|
170
|
+
this.config.onClose?.(event);
|
|
171
|
+
this.stopPing();
|
|
172
|
+
void this.reconnect();
|
|
173
|
+
};
|
|
174
|
+
this.ws.onerror = (error) => {
|
|
175
|
+
this.logger.error("❌ [ShellX] WebSocket error:", error);
|
|
176
|
+
this.config.onError?.(error);
|
|
177
|
+
reject(new Error("WebSocket connection failed"));
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
catch (error) {
|
|
181
|
+
this.logger.error("❌ [WebSocket] Initialization failed:", error);
|
|
182
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
183
|
+
this.config.onError?.(err);
|
|
184
|
+
reject(err);
|
|
185
|
+
}
|
|
186
|
+
})
|
|
187
|
+
.catch((error) => {
|
|
188
|
+
this.logger.error("❌ [WebSocket] Failed to get constructor:", error);
|
|
189
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
190
|
+
reject(err);
|
|
191
|
+
});
|
|
192
|
+
})
|
|
193
|
+
.catch((error) => {
|
|
194
|
+
this.logger.error("❌ [Auth] Device authentication failed:", error);
|
|
195
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
196
|
+
reject(err);
|
|
197
|
+
});
|
|
152
198
|
});
|
|
153
199
|
}
|
|
154
|
-
authenticateDevice(
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
200
|
+
async authenticateDevice() {
|
|
201
|
+
let fallbackUrl = undefined;
|
|
202
|
+
// 1. Prioritize local service detection
|
|
203
|
+
try {
|
|
204
|
+
// fetch timeout implementation
|
|
205
|
+
const fetchWithTimeout = (url, options, timeout = 1000) => Promise.race([
|
|
206
|
+
fetch(url, options),
|
|
207
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), timeout)),
|
|
208
|
+
]);
|
|
209
|
+
const localResp = await fetchWithTimeout("http://127.0.0.1:9091/info", { method: "GET", headers: { Accept: "application/json" } }, 1000);
|
|
210
|
+
if (localResp.ok) {
|
|
211
|
+
const info = (await localResp.json());
|
|
212
|
+
if (info && (info.status === "ok" || info.status === 1) && info.uuid) {
|
|
213
|
+
if (this.deviceId === undefined || this.deviceId === info.uuid) {
|
|
214
|
+
const localUuid = info.uuid;
|
|
215
|
+
this.deviceId = localUuid;
|
|
216
|
+
fallbackUrl = `ws://127.0.0.1:9091/api/s/${localUuid}`;
|
|
217
|
+
this.logger.info("✅ [Auth] Local ShellX service available, using local /info returned uuid:", localUuid, ", Service address:", fallbackUrl);
|
|
218
|
+
return fallbackUrl;
|
|
173
219
|
}
|
|
174
220
|
}
|
|
175
221
|
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
// Local unavailable, proceeding with remote
|
|
225
|
+
}
|
|
226
|
+
if (this.deviceId === undefined) {
|
|
227
|
+
this.logger.warn("❌ [Auth] Device ID not set, local USB not connected, please set environment variable SHELLX_DEVICE_ID or connect USB device");
|
|
228
|
+
return undefined;
|
|
229
|
+
}
|
|
230
|
+
// 2. Remote authentication logic
|
|
231
|
+
try {
|
|
232
|
+
this.logger.info("🔑 [Auth] Authenticating device...");
|
|
233
|
+
const featGlobal = this.getFetch();
|
|
234
|
+
const deviceApiUrl = buildDeviceApiUrl(this.deviceId);
|
|
235
|
+
const jsonData = (await featGlobal(deviceApiUrl, {
|
|
236
|
+
method: "GET",
|
|
237
|
+
headers: {
|
|
238
|
+
"Content-Type": "application/json",
|
|
239
|
+
},
|
|
240
|
+
}));
|
|
241
|
+
this.logger.info("ShellX.ai device authentication response:", jsonData);
|
|
242
|
+
// Check if device is not registered
|
|
243
|
+
if (jsonData?.status === "not_registered") {
|
|
244
|
+
this.logger.error("❌ [Auth] Device not registered");
|
|
245
|
+
this.logger.error(`📝 [Auth] Status: ${jsonData.message || "Device not registered"}`);
|
|
246
|
+
this.logger.error("💡 [Auth] Please register this device on ShellX.ai platform first");
|
|
247
|
+
throw new Error(`Device "${this.deviceId}" not registered on ShellX.ai platform. Please visit https://shellx.ai to register the device first.`);
|
|
202
248
|
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
249
|
+
// Verify authentication response is valid
|
|
250
|
+
// If interface returns null or missing required fields, device is not registered
|
|
251
|
+
if (!jsonData || jsonData === null || !jsonData.machine || !jsonData.authenticate) {
|
|
252
|
+
this.logger.error("❌ [Auth] ShellX.ai device not registered or invalid authentication information");
|
|
253
|
+
throw new Error(`Device "${this.deviceId}" has invalid authentication information. Please check device ID or re-register on ShellX.ai platform.`);
|
|
207
254
|
}
|
|
208
|
-
|
|
255
|
+
// const jsonData = JSON.parse(data);
|
|
256
|
+
this.logger.info("✅ [Auth] ShellX.ai device authentication successful");
|
|
257
|
+
this.logger.info(`📡 [Auth] Device ID: ${jsonData.authenticate}`);
|
|
258
|
+
this.logger.info(`📡 [Auth] ShellX.ai service address: ${jsonData.machine}`);
|
|
259
|
+
this.logger.info(`📡 [Auth] Registration time: ${jsonData.registered_at}`);
|
|
260
|
+
this.logger.info(`📡 [Auth] Last update: ${jsonData.last_updated}`);
|
|
261
|
+
this.logger.info(`📡 [Auth] Device status: ${jsonData.status || "registered"}`);
|
|
262
|
+
// UUID is the deviceId, no need to extract from URL
|
|
263
|
+
const wsUrl = `${jsonData.machine}/api/s/${this.deviceId}`;
|
|
264
|
+
this.logger.info(`🔗 [Auth] WebSocket URL: ${wsUrl}`);
|
|
265
|
+
return wsUrl;
|
|
266
|
+
}
|
|
267
|
+
catch (error) {
|
|
268
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
269
|
+
this.logger.warn("⚠️ [Auth] Online authentication failed, using local service:", JSON.stringify(errorMessage));
|
|
270
|
+
return fallbackUrl;
|
|
271
|
+
}
|
|
209
272
|
}
|
|
210
273
|
getFetch() {
|
|
211
|
-
return (url, options) =>
|
|
212
|
-
//
|
|
213
|
-
if (typeof globalThis.fetch !==
|
|
214
|
-
|
|
215
|
-
const response =
|
|
216
|
-
|
|
274
|
+
return async (url, options) => {
|
|
275
|
+
// Try using global fetch
|
|
276
|
+
if (typeof globalThis.fetch !== "undefined") {
|
|
277
|
+
this.logger.info(`🌐 [Fetch] Request: ${options?.method || "GET"} ${url}`);
|
|
278
|
+
const response = await globalThis.fetch(url, options);
|
|
279
|
+
this.logger.info(`📡 [Fetch] Response: ${response.status} ${response.statusText}`);
|
|
217
280
|
if (!response.ok) {
|
|
218
281
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
219
282
|
}
|
|
220
283
|
return response.json();
|
|
221
284
|
}
|
|
222
285
|
try {
|
|
223
|
-
const { default: fetch } =
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
286
|
+
const { default: fetch } = await import("node-fetch");
|
|
287
|
+
this.logger.info(`🌐 [Fetch] Request: ${options?.method || "GET"} ${url}`);
|
|
288
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
289
|
+
const response = await fetch(url, options);
|
|
290
|
+
this.logger.info(`📡 [Fetch] Response: ${response.status} ${response.statusText}`);
|
|
227
291
|
if (!response.ok) {
|
|
228
292
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
229
293
|
}
|
|
230
294
|
return response.json();
|
|
231
295
|
}
|
|
232
296
|
catch (fetchError) {
|
|
233
|
-
|
|
297
|
+
const errorMessage = fetchError instanceof Error ? fetchError.message : String(fetchError);
|
|
298
|
+
throw new Error(`Fetch not available: ${errorMessage}`);
|
|
234
299
|
}
|
|
235
|
-
}
|
|
300
|
+
};
|
|
236
301
|
}
|
|
237
302
|
/**
|
|
238
|
-
*
|
|
303
|
+
* Wait for WebSocket initialization complete
|
|
239
304
|
*/
|
|
240
|
-
waitForInitialization() {
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
}
|
|
245
|
-
});
|
|
305
|
+
async waitForInitialization() {
|
|
306
|
+
if (this.initializationPromise) {
|
|
307
|
+
await this.initializationPromise;
|
|
308
|
+
}
|
|
246
309
|
}
|
|
247
310
|
/**
|
|
248
|
-
*
|
|
311
|
+
* Ensure connection is ready before executing device commands
|
|
312
|
+
* If not connected, wait for connection with timeout
|
|
313
|
+
*
|
|
314
|
+
* @param timeout - Timeout in milliseconds (default: 10000ms)
|
|
315
|
+
* @throws Error if connection timeout or connection failed
|
|
316
|
+
*
|
|
317
|
+
* @example
|
|
318
|
+
* ```typescript
|
|
319
|
+
* await client.ensureConnected(10000);
|
|
320
|
+
* console.log('Connection ready');
|
|
321
|
+
* ```
|
|
249
322
|
*/
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
323
|
+
async ensureConnected(timeout = 10000) {
|
|
324
|
+
// If already connected, return immediately
|
|
325
|
+
if (this.shellxConnected && this.wsConnected) {
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
this.logger.info(`⏳ [ConnectionClient] Waiting for connection to be ready (timeout: ${timeout}ms)...`);
|
|
329
|
+
const startTime = Date.now();
|
|
330
|
+
const checkInterval = 100; // Check every 100ms
|
|
331
|
+
return new Promise((resolve, reject) => {
|
|
332
|
+
const intervalId = setInterval(() => {
|
|
333
|
+
const elapsed = Date.now() - startTime;
|
|
334
|
+
// Check if connected
|
|
335
|
+
if (this.shellxConnected && this.wsConnected) {
|
|
336
|
+
clearInterval(intervalId);
|
|
337
|
+
this.logger.info(`✅ [ConnectionClient] Connection ready (took ${elapsed}ms)`);
|
|
338
|
+
resolve();
|
|
256
339
|
return;
|
|
257
340
|
}
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
341
|
+
// Check timeout
|
|
342
|
+
if (elapsed >= timeout) {
|
|
343
|
+
clearInterval(intervalId);
|
|
344
|
+
const errorMsg = `Connection timeout after ${timeout}ms. ` +
|
|
345
|
+
`Device "${this.deviceId}" is not connected. ` +
|
|
346
|
+
`Please check: ` +
|
|
347
|
+
`1. Device is online and registered on ShellX.ai ` +
|
|
348
|
+
`2. Network connection is stable ` +
|
|
349
|
+
`3. Device ID is correct`;
|
|
350
|
+
this.logger.error(`❌ [ConnectionClient] ${errorMsg}`);
|
|
351
|
+
reject(new Error(errorMsg));
|
|
352
|
+
return;
|
|
265
353
|
}
|
|
266
|
-
|
|
267
|
-
|
|
354
|
+
// Log progress every 2 seconds
|
|
355
|
+
if (elapsed > 0 && elapsed % 2000 < checkInterval) {
|
|
356
|
+
this.logger.info(`⏳ [ConnectionClient] Still waiting for connection... (${elapsed}/${timeout}ms)`);
|
|
268
357
|
}
|
|
269
|
-
}
|
|
270
|
-
catch (error) {
|
|
271
|
-
console.error('❌ [Auth] 发送认证消息失败:', error);
|
|
272
|
-
}
|
|
358
|
+
}, checkInterval);
|
|
273
359
|
});
|
|
274
360
|
}
|
|
275
361
|
handleMessage(event) {
|
|
@@ -289,11 +375,11 @@ class ConnectionTaskClient {
|
|
|
289
375
|
this.processServerMessage(serverMessage);
|
|
290
376
|
return;
|
|
291
377
|
}
|
|
292
|
-
serverMessage = (
|
|
378
|
+
serverMessage = cborDecode(binaryData);
|
|
293
379
|
this.processServerMessage(serverMessage);
|
|
294
380
|
}
|
|
295
381
|
catch (cborError) {
|
|
296
|
-
|
|
382
|
+
this.logger.info("CBOR decode failed, trying JSON fallback:", cborError);
|
|
297
383
|
try {
|
|
298
384
|
let textData;
|
|
299
385
|
if (event.data instanceof ArrayBuffer) {
|
|
@@ -306,316 +392,359 @@ class ConnectionTaskClient {
|
|
|
306
392
|
this.processServerMessage(serverMessage);
|
|
307
393
|
}
|
|
308
394
|
catch (jsonError) {
|
|
309
|
-
|
|
395
|
+
this.logger.error("Failed to parse message:", { cborError, jsonError });
|
|
310
396
|
}
|
|
311
397
|
}
|
|
312
398
|
}
|
|
313
399
|
processServerMessage(message) {
|
|
314
|
-
|
|
315
|
-
//console.log('📨 [ShellX] 收到服务器消息:', message);
|
|
316
|
-
this.authenticated = true;
|
|
317
|
-
// 只有在认证成功后才处理其他消息并认为连接成功
|
|
318
|
-
if (this.authenticated && !this.shellxConnected) {
|
|
319
|
-
this.shellxConnected = true;
|
|
320
|
-
console.log('✅ [ShellX] ShellX.ai服务连接成功!');
|
|
321
|
-
(_b = (_a = this.config).onOpen) === null || _b === void 0 ? void 0 : _b.call(_a);
|
|
322
|
-
this.flushQueue();
|
|
323
|
-
this.startPing();
|
|
324
|
-
}
|
|
400
|
+
//console.log('📨 [ShellX] Received server message:', message);
|
|
325
401
|
// Call user-defined message handler
|
|
326
|
-
|
|
402
|
+
this.config.onMessage?.(message);
|
|
327
403
|
// Handle specific message types and resolve pending tasks
|
|
328
404
|
if (message.jsonData) {
|
|
329
405
|
this.handleJsonDataResponse(message.jsonData);
|
|
330
406
|
}
|
|
331
407
|
// Handle other server message types (hello, users, shells, etc.)
|
|
332
408
|
if (message.hello !== undefined) {
|
|
333
|
-
|
|
409
|
+
this.logger.info("👋 [ShellX] Server greeting, user ID:", message.hello);
|
|
334
410
|
}
|
|
335
411
|
if (message.pong !== undefined) {
|
|
336
|
-
//console.log('🏓 [ShellX]
|
|
412
|
+
//console.log('🏓 [ShellX] Heartbeat response:', message.pong);
|
|
337
413
|
}
|
|
338
414
|
if (message.error) {
|
|
339
|
-
|
|
415
|
+
this.logger.error("❌ [ShellX] Server error:", message.error);
|
|
340
416
|
}
|
|
341
417
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
switch (task.type) {
|
|
348
|
-
case 'findElement':
|
|
349
|
-
if (jsonData.findElement) {
|
|
350
|
-
responseData = jsonData.findElement;
|
|
351
|
-
shouldResolve = true;
|
|
352
|
-
}
|
|
353
|
-
break;
|
|
354
|
-
case 'waitElement':
|
|
355
|
-
if (jsonData.waitElement) {
|
|
356
|
-
responseData = jsonData.waitElement;
|
|
357
|
-
shouldResolve = true;
|
|
358
|
-
}
|
|
359
|
-
break;
|
|
360
|
-
case 'screenShot':
|
|
361
|
-
if (jsonData.screenShot) {
|
|
362
|
-
responseData = jsonData.screenShot;
|
|
363
|
-
shouldResolve = true;
|
|
364
|
-
}
|
|
365
|
-
break;
|
|
366
|
-
case 'screenInfo':
|
|
367
|
-
if (jsonData.screenInfo) {
|
|
368
|
-
responseData = jsonData.screenInfo;
|
|
369
|
-
shouldResolve = true;
|
|
370
|
-
}
|
|
371
|
-
break;
|
|
372
|
-
case 'appList':
|
|
373
|
-
if (jsonData.appList) {
|
|
374
|
-
responseData = jsonData.appList;
|
|
375
|
-
shouldResolve = true;
|
|
376
|
-
}
|
|
377
|
-
break;
|
|
378
|
-
case 'action':
|
|
379
|
-
if (jsonData.action_event) {
|
|
380
|
-
responseData = jsonData.action_event;
|
|
381
|
-
shouldResolve = true;
|
|
382
|
-
}
|
|
383
|
-
break;
|
|
384
|
-
case 'actions':
|
|
385
|
-
if (jsonData.actions) {
|
|
386
|
-
responseData = jsonData.actions;
|
|
387
|
-
shouldResolve = true;
|
|
388
|
-
}
|
|
389
|
-
break;
|
|
390
|
-
}
|
|
391
|
-
if (shouldResolve) {
|
|
392
|
-
clearTimeout(task.timer);
|
|
393
|
-
this.pendingTasks.delete(taskId);
|
|
394
|
-
if ((responseData === null || responseData === void 0 ? void 0 : responseData.success) === false) {
|
|
395
|
-
task.reject(new Error(responseData.errorMessage || 'Task failed'));
|
|
418
|
+
matchResponseByType(jsonData, taskType) {
|
|
419
|
+
switch (taskType) {
|
|
420
|
+
case "findElement":
|
|
421
|
+
if (jsonData.findElement) {
|
|
422
|
+
return jsonData.findElement;
|
|
396
423
|
}
|
|
397
|
-
|
|
398
|
-
|
|
424
|
+
break;
|
|
425
|
+
case "waitElement":
|
|
426
|
+
if (jsonData.waitElement) {
|
|
427
|
+
return jsonData.waitElement;
|
|
399
428
|
}
|
|
400
|
-
break;
|
|
401
|
-
|
|
429
|
+
break;
|
|
430
|
+
case "screenShot":
|
|
431
|
+
if (jsonData.screenShot) {
|
|
432
|
+
return jsonData.screenShot;
|
|
433
|
+
}
|
|
434
|
+
break;
|
|
435
|
+
case "screenInfo":
|
|
436
|
+
if (jsonData.screenInfo) {
|
|
437
|
+
return jsonData.screenInfo;
|
|
438
|
+
}
|
|
439
|
+
break;
|
|
440
|
+
case "appList":
|
|
441
|
+
console.log("appList");
|
|
442
|
+
console.log(jsonData);
|
|
443
|
+
if (jsonData.appList) {
|
|
444
|
+
return jsonData.appList;
|
|
445
|
+
}
|
|
446
|
+
break;
|
|
447
|
+
case "action":
|
|
448
|
+
if (jsonData.action_event) {
|
|
449
|
+
return jsonData.action_event;
|
|
450
|
+
}
|
|
451
|
+
break;
|
|
452
|
+
case "promptflowx":
|
|
453
|
+
if (jsonData.promptflowx) {
|
|
454
|
+
return jsonData.promptflowx;
|
|
455
|
+
}
|
|
456
|
+
break;
|
|
457
|
+
case "clipboard":
|
|
458
|
+
if (jsonData.clipboard) {
|
|
459
|
+
return jsonData.clipboard;
|
|
460
|
+
}
|
|
461
|
+
break;
|
|
462
|
+
case "asrControl":
|
|
463
|
+
// ASR control doesn't expect a specific response
|
|
464
|
+
return { success: true };
|
|
402
465
|
}
|
|
466
|
+
return undefined;
|
|
403
467
|
}
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
reject,
|
|
416
|
-
timer,
|
|
417
|
-
type: taskType
|
|
418
|
-
});
|
|
419
|
-
console.log(`📋 [ShellX] 创建任务: ${taskId}, 类型: ${taskType}}`);
|
|
420
|
-
}
|
|
421
|
-
if (this.shellxConnected && this.ws) {
|
|
422
|
-
try {
|
|
423
|
-
console.log('📤 [ShellX] 发送消息:', JSON.stringify(message));
|
|
424
|
-
this.ws.send((0, cbor_1.encode)(message));
|
|
425
|
-
if (!taskType)
|
|
426
|
-
resolve(undefined); // For fire-and-forget messages
|
|
427
|
-
}
|
|
428
|
-
catch (error) {
|
|
429
|
-
if (taskType) {
|
|
430
|
-
this.pendingTasks.delete(taskId);
|
|
431
|
-
}
|
|
432
|
-
reject(error);
|
|
468
|
+
handleJsonDataResponse(jsonData) {
|
|
469
|
+
const responseTaskId = jsonData?.taskId;
|
|
470
|
+
if (responseTaskId) {
|
|
471
|
+
const task = this.pendingTasks.get(responseTaskId);
|
|
472
|
+
if (task) {
|
|
473
|
+
const responseData = this.matchResponseByType(jsonData, task.type);
|
|
474
|
+
if (responseData !== undefined) {
|
|
475
|
+
clearTimeout(task.timer);
|
|
476
|
+
this.pendingTasks.delete(responseTaskId);
|
|
477
|
+
if (task.type != "findElement") {
|
|
478
|
+
this.logger.info("✅ [ShellX] Response processing complete:", responseTaskId, jsonData);
|
|
433
479
|
}
|
|
480
|
+
task.resolve(responseData);
|
|
481
|
+
return;
|
|
434
482
|
}
|
|
435
483
|
else {
|
|
436
|
-
|
|
437
|
-
this.reconnect();
|
|
438
|
-
this.messageQueue.push(message);
|
|
439
|
-
if (!taskType)
|
|
440
|
-
resolve(undefined);
|
|
484
|
+
this.logger.warn(`⚠️ [ShellX] Task ${responseTaskId} (type: ${task.type}) found but no matching response data in:`, jsonData);
|
|
441
485
|
}
|
|
442
|
-
}
|
|
443
|
-
|
|
486
|
+
}
|
|
487
|
+
else {
|
|
488
|
+
this.logger.warn(`⚠️ [ShellX] Received taskId ${responseTaskId} but no pending task found`);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
else {
|
|
492
|
+
this.logger.debug(`📨 [ShellX] Received response without taskId, trying fallback match:`, jsonData);
|
|
493
|
+
}
|
|
494
|
+
// Fallback to legacy matching when taskId is missing or not tracked
|
|
495
|
+
for (const [taskId, task] of this.pendingTasks.entries()) {
|
|
496
|
+
const responseData = this.matchResponseByType(jsonData, task.type);
|
|
497
|
+
if (responseData !== undefined) {
|
|
498
|
+
clearTimeout(task.timer);
|
|
499
|
+
this.pendingTasks.delete(taskId);
|
|
500
|
+
this.logger.info(`✅ [ShellX] Fallback match: completed task ${taskId} (type: ${task.type})`);
|
|
501
|
+
task.resolve(responseData);
|
|
502
|
+
break;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
async sendMessage(message, taskType) {
|
|
507
|
+
if (!taskType) {
|
|
508
|
+
taskType = "oneway";
|
|
509
|
+
}
|
|
510
|
+
const { promise } = await this.sendMessageWithTaskId(message, taskType);
|
|
511
|
+
return await promise;
|
|
444
512
|
}
|
|
445
513
|
// ===== PROTOCOL-AWARE TASK METHODS =====
|
|
446
514
|
/**
|
|
447
515
|
* Find UI elements on the screen
|
|
448
516
|
*/
|
|
449
|
-
findElement(selector, options) {
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
return this.sendMessage({ findElement: findAction }, 'findElement');
|
|
457
|
-
});
|
|
517
|
+
async findElement(selector, options) {
|
|
518
|
+
const findAction = {
|
|
519
|
+
type: "find",
|
|
520
|
+
selector,
|
|
521
|
+
options,
|
|
522
|
+
};
|
|
523
|
+
return this.sendMessage({ findElement: findAction }, "findElement");
|
|
458
524
|
}
|
|
459
525
|
/**
|
|
460
526
|
* Wait for UI element to appear
|
|
461
527
|
*/
|
|
462
|
-
waitElement(selector, options) {
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
return this.sendMessage({ waitElement: waitAction }, 'waitElement');
|
|
470
|
-
});
|
|
528
|
+
async waitElement(selector, options) {
|
|
529
|
+
const waitAction = {
|
|
530
|
+
type: "wait",
|
|
531
|
+
selector,
|
|
532
|
+
options,
|
|
533
|
+
};
|
|
534
|
+
return this.sendMessage({ waitElement: waitAction }, "waitElement");
|
|
471
535
|
}
|
|
472
536
|
/**
|
|
473
537
|
* Take a screenshot
|
|
538
|
+
* @deprecated Use takeScreenshot() instead for consistency
|
|
474
539
|
*/
|
|
475
|
-
screenShot(options) {
|
|
476
|
-
return
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
540
|
+
async screenShot(options) {
|
|
541
|
+
return this.takeScreenshot(options);
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* Take a screenshot
|
|
545
|
+
*/
|
|
546
|
+
async takeScreenshot(options) {
|
|
547
|
+
return this.sendMessage({
|
|
548
|
+
screenShot: options || {
|
|
549
|
+
format: "png",
|
|
550
|
+
quality: 30,
|
|
551
|
+
scale: 1.0,
|
|
552
|
+
region: {
|
|
553
|
+
left: 0,
|
|
554
|
+
top: 0,
|
|
555
|
+
width: 1080,
|
|
556
|
+
height: 2340,
|
|
488
557
|
},
|
|
489
|
-
},
|
|
490
|
-
});
|
|
558
|
+
},
|
|
559
|
+
}, "screenShot");
|
|
491
560
|
}
|
|
492
561
|
/**
|
|
493
562
|
* Get screen information
|
|
494
563
|
*/
|
|
495
|
-
getScreenInfo() {
|
|
496
|
-
return
|
|
497
|
-
return this.sendMessage({ screenInfo: {} }, 'screenInfo');
|
|
498
|
-
});
|
|
564
|
+
async getScreenInfo() {
|
|
565
|
+
return this.sendMessage({ screenInfo: { keepScreenOn: false, wakeApp: true } }, "screenInfo");
|
|
499
566
|
}
|
|
500
567
|
/**
|
|
501
|
-
* Get
|
|
568
|
+
* Get screen information
|
|
569
|
+
* @deprecated Use wakeScreenAndGetInfo() instead for clarity
|
|
502
570
|
*/
|
|
503
|
-
|
|
504
|
-
return
|
|
505
|
-
return this.sendMessage({ appList: options || {} }, 'appList');
|
|
506
|
-
});
|
|
571
|
+
async getAndWakeScreen() {
|
|
572
|
+
return this.wakeScreenAndGetInfo();
|
|
507
573
|
}
|
|
508
574
|
/**
|
|
509
|
-
*
|
|
575
|
+
* Wake screen and get screen information
|
|
510
576
|
*/
|
|
511
|
-
|
|
512
|
-
return
|
|
513
|
-
return this.sendMessage({ appInfo: { packageName } }, 'appInfo');
|
|
514
|
-
});
|
|
577
|
+
async wakeScreenAndGetInfo() {
|
|
578
|
+
return this.sendMessage({ screenInfo: { keepScreenOn: true, wakeApp: true } }, "screenInfo");
|
|
515
579
|
}
|
|
516
580
|
/**
|
|
517
581
|
* Execute an action sequence
|
|
518
582
|
*/
|
|
519
|
-
executeAction(actionSequence, taskId,
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
});
|
|
583
|
+
async executeAction(actionSequence, taskId, timeout) {
|
|
584
|
+
const { promise } = await this.sendMessageWithTaskId({ actions: actionSequence }, "action", { taskId, timeout });
|
|
585
|
+
return await promise;
|
|
523
586
|
}
|
|
524
587
|
/**
|
|
525
588
|
* Execute promptflow actions
|
|
526
589
|
*/
|
|
527
|
-
executePromptFlow(actionSequence) {
|
|
528
|
-
return
|
|
529
|
-
return this.sendMessage({ promptflowx: actionSequence });
|
|
530
|
-
});
|
|
590
|
+
async executePromptFlow(actionSequence) {
|
|
591
|
+
return this.sendMessage({ promptflowx: actionSequence });
|
|
531
592
|
}
|
|
532
593
|
/**
|
|
533
594
|
* Listen for display changes
|
|
534
595
|
*/
|
|
535
|
-
screenChange(options) {
|
|
536
|
-
return
|
|
537
|
-
|
|
538
|
-
screenChange: options || { enable: true },
|
|
539
|
-
});
|
|
596
|
+
async screenChange(options) {
|
|
597
|
+
return this.sendMessage({
|
|
598
|
+
screenChange: options || { enable: true },
|
|
540
599
|
});
|
|
541
600
|
}
|
|
542
601
|
/**
|
|
543
602
|
* Switch to a different node
|
|
544
603
|
*/
|
|
545
|
-
switchNode(nodeId) {
|
|
546
|
-
return
|
|
547
|
-
return this.sendMessage({ switchNode: nodeId });
|
|
548
|
-
});
|
|
604
|
+
async switchNode(nodeId) {
|
|
605
|
+
return this.sendMessage({ switchNode: nodeId });
|
|
549
606
|
}
|
|
550
607
|
/**
|
|
551
608
|
* Send authentication data
|
|
552
609
|
*/
|
|
553
|
-
authenticate(authData) {
|
|
554
|
-
return
|
|
555
|
-
return this.sendMessage({ authenticate: authData });
|
|
556
|
-
});
|
|
610
|
+
async authenticate(authData) {
|
|
611
|
+
return this.sendMessage({ authenticate: authData });
|
|
557
612
|
}
|
|
558
613
|
/**
|
|
559
614
|
* Set user name
|
|
560
615
|
*/
|
|
561
|
-
setName(name) {
|
|
562
|
-
return
|
|
563
|
-
return this.sendMessage({ setName: name });
|
|
564
|
-
});
|
|
616
|
+
async setName(name) {
|
|
617
|
+
return this.sendMessage({ setName: name });
|
|
565
618
|
}
|
|
566
619
|
/**
|
|
567
620
|
* Send chat message
|
|
568
621
|
*/
|
|
569
|
-
sendChat(message) {
|
|
570
|
-
return
|
|
571
|
-
|
|
572
|
-
|
|
622
|
+
async sendChat(message) {
|
|
623
|
+
return this.sendMessage({ chat: message });
|
|
624
|
+
}
|
|
625
|
+
/**
|
|
626
|
+
* Start ASR (Automatic Speech Recognition)
|
|
627
|
+
*/
|
|
628
|
+
async startAsr() {
|
|
629
|
+
return this.sendMessage({ asrControl: { action: "start_asr" } }, "oneway");
|
|
630
|
+
}
|
|
631
|
+
/**
|
|
632
|
+
* Stop ASR (Automatic Speech Recognition)
|
|
633
|
+
*/
|
|
634
|
+
async stopAsr() {
|
|
635
|
+
return this.sendMessage({ asrControl: { action: "stop_asr" } }, "oneway");
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* Speak text using TTS (Text-to-Speech)
|
|
639
|
+
*/
|
|
640
|
+
async speak(text) {
|
|
641
|
+
return this.sendMessage({ ttsControl: { action: "tts_speak", text } }, "oneway");
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Stop TTS (Text-to-Speech)
|
|
645
|
+
*/
|
|
646
|
+
async stopTts() {
|
|
647
|
+
return this.sendMessage({ ttsControl: { action: "tts_stop" } }, "oneway");
|
|
573
648
|
}
|
|
574
649
|
/**
|
|
575
650
|
* Send raw message (for custom integrations)
|
|
576
651
|
*/
|
|
577
|
-
sendRawMessage(message) {
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
652
|
+
async sendRawMessage(message) {
|
|
653
|
+
// Auto-detect task type from message content
|
|
654
|
+
const taskType = this.detectTaskType(message);
|
|
655
|
+
return this.sendMessage(message, taskType);
|
|
581
656
|
}
|
|
582
657
|
/**
|
|
583
|
-
*
|
|
584
|
-
* @param message 要发送的消息
|
|
585
|
-
* @param taskType 任务类型
|
|
586
|
-
* @returns Promise<{taskId: string, promise: Promise<any>}>
|
|
658
|
+
* Detect task type from message content
|
|
587
659
|
*/
|
|
588
|
-
|
|
660
|
+
detectTaskType(message) {
|
|
661
|
+
if (message.findElement)
|
|
662
|
+
return "findElement";
|
|
663
|
+
if (message.waitElement)
|
|
664
|
+
return "waitElement";
|
|
665
|
+
if (message.screenShot)
|
|
666
|
+
return "screenShot";
|
|
667
|
+
if (message.screenInfo)
|
|
668
|
+
return "screenInfo";
|
|
669
|
+
if (message.appList)
|
|
670
|
+
return "appList";
|
|
671
|
+
if (message.action)
|
|
672
|
+
return "action";
|
|
673
|
+
if (message.actions)
|
|
674
|
+
return "action";
|
|
675
|
+
if (message.promptflowx)
|
|
676
|
+
return "promptflowx";
|
|
677
|
+
if (message.asrControl)
|
|
678
|
+
return "asrControl";
|
|
679
|
+
// Oneway messages (fire-and-forget, no response expected)
|
|
680
|
+
if (message.ttsControl)
|
|
681
|
+
return "oneway";
|
|
682
|
+
if (message.screenChange)
|
|
683
|
+
return "oneway";
|
|
684
|
+
if (message.switchNode)
|
|
685
|
+
return "oneway";
|
|
686
|
+
if (message.authenticate)
|
|
687
|
+
return "oneway";
|
|
688
|
+
if (message.setName)
|
|
689
|
+
return "oneway";
|
|
690
|
+
if (message.chat)
|
|
691
|
+
return "oneway";
|
|
692
|
+
if (message.ping)
|
|
693
|
+
return "oneway";
|
|
694
|
+
// Default to oneway for unknown message types
|
|
695
|
+
return "oneway";
|
|
696
|
+
}
|
|
697
|
+
/**
|
|
698
|
+
* Send message and return taskId
|
|
699
|
+
* @param message Message to send
|
|
700
|
+
* @param taskType Task type
|
|
701
|
+
* @param options Optional parameters { timeout, taskId }
|
|
702
|
+
* @returns Promise<{taskId: string, promise: Promise<T>>}
|
|
703
|
+
*/
|
|
704
|
+
async sendMessageWithTaskId(message, taskType, options) {
|
|
589
705
|
return new Promise((resolve) => {
|
|
706
|
+
const { timeout, taskId: definedTaskId } = options || {};
|
|
590
707
|
let taskId = definedTaskId;
|
|
591
708
|
if (taskId == null) {
|
|
592
|
-
taskId = (
|
|
709
|
+
taskId = uuidv4();
|
|
593
710
|
}
|
|
711
|
+
this.logger.info(`📋 [ShellX] Created task: ${taskId}, Task type: ${taskType}`);
|
|
712
|
+
this.logger.info(message);
|
|
713
|
+
const outgoingMessage = taskId ? { ...message, taskId } : message;
|
|
594
714
|
if (isPendingTask(taskType)) {
|
|
595
715
|
const timer = setTimeout(() => {
|
|
596
716
|
if (taskId != null) {
|
|
717
|
+
const task = this.pendingTasks.get(taskId);
|
|
718
|
+
if (task) {
|
|
719
|
+
// Create a timeout result based on task type
|
|
720
|
+
const timeoutResult = this.createTimeoutResult(taskType, timeout ?? this.config.timeout);
|
|
721
|
+
// Resolve with timeout result instead of rejecting
|
|
722
|
+
task.resolve(timeoutResult);
|
|
723
|
+
}
|
|
597
724
|
this.pendingTasks.delete(taskId);
|
|
598
725
|
}
|
|
599
|
-
|
|
600
|
-
}, timeout
|
|
726
|
+
this.logger.info(`⏰ [ShellX] Task timeout: ${taskId}, Task type: ${taskType}`);
|
|
727
|
+
}, timeout ?? this.config.timeout);
|
|
601
728
|
this.pendingTasks.set(taskId, {
|
|
602
729
|
resolve: () => { },
|
|
603
730
|
reject: () => { },
|
|
604
731
|
timer,
|
|
605
732
|
type: taskType,
|
|
606
733
|
});
|
|
607
|
-
|
|
734
|
+
this.logger.info(`📋 [ShellX] Created manually controllable task: ${taskId}, Task type: ${taskType}}`);
|
|
608
735
|
}
|
|
609
736
|
if (this.shellxConnected && this.ws) {
|
|
610
737
|
try {
|
|
611
|
-
|
|
612
|
-
this.ws.send((
|
|
738
|
+
this.logger.info(" 📤 [ShellX] Sending message:", JSON.stringify(outgoingMessage));
|
|
739
|
+
this.ws.send(cborEncode(outgoingMessage));
|
|
613
740
|
const promise = new Promise((promiseResolve, promiseReject) => {
|
|
614
741
|
if (isPendingTask(taskType)) {
|
|
615
742
|
if (taskId != null) {
|
|
616
743
|
const task = this.pendingTasks.get(taskId);
|
|
617
744
|
if (task) {
|
|
745
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any
|
|
618
746
|
task.resolve = promiseResolve;
|
|
747
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any
|
|
619
748
|
task.reject = promiseReject;
|
|
620
749
|
}
|
|
621
750
|
}
|
|
@@ -630,12 +759,14 @@ class ConnectionTaskClient {
|
|
|
630
759
|
if (isPendingTask(taskType)) {
|
|
631
760
|
this.pendingTasks.delete(taskId);
|
|
632
761
|
}
|
|
633
|
-
|
|
762
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
763
|
+
resolve({ taskId, promise: Promise.reject(err) });
|
|
634
764
|
}
|
|
635
765
|
}
|
|
636
766
|
else {
|
|
637
|
-
|
|
638
|
-
this.messageQueue.push(
|
|
767
|
+
this.logger.info("⏳ [ShellX] Connection not ready, message queued");
|
|
768
|
+
this.messageQueue.push(outgoingMessage);
|
|
769
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
|
|
639
770
|
resolve({ taskId, promise: Promise.resolve(undefined) });
|
|
640
771
|
}
|
|
641
772
|
});
|
|
@@ -643,10 +774,10 @@ class ConnectionTaskClient {
|
|
|
643
774
|
// ===== CONNECTION MANAGEMENT =====
|
|
644
775
|
startPing() {
|
|
645
776
|
this.stopPing();
|
|
646
|
-
|
|
777
|
+
this.logger.info("🏓 [ShellX] Starting heartbeat detection...");
|
|
647
778
|
this.pingIntervalId = setInterval(() => {
|
|
648
779
|
if (this.ws && this.shellxConnected) {
|
|
649
|
-
this.ws.send((
|
|
780
|
+
this.ws.send(cborEncode({ ping: Date.now() }));
|
|
650
781
|
}
|
|
651
782
|
}, this.config.pingInterval);
|
|
652
783
|
}
|
|
@@ -656,42 +787,49 @@ class ConnectionTaskClient {
|
|
|
656
787
|
this.pingIntervalId = null;
|
|
657
788
|
}
|
|
658
789
|
}
|
|
659
|
-
reconnect() {
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
790
|
+
async reconnect() {
|
|
791
|
+
// TODO: Local needs reconnection
|
|
792
|
+
this.logger.info(`🔄 [ShellX] Reconnecting... (${this.reconnectAttempts}/${this.config.reconnectMaxAttempts})` +
|
|
793
|
+
this.wsUrl);
|
|
794
|
+
if (this.reconnectAttempts >= this.config.reconnectMaxAttempts) {
|
|
795
|
+
this.logger.error("❌ [ShellX] Reconnection failed, max attempts reached");
|
|
796
|
+
this.config.onReconnectFailed?.();
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
this.reconnectAttempts++;
|
|
800
|
+
// Exponential backoff strategy: baseInterval * 2^(attempts-1), with maximum delay cap and random jitter
|
|
801
|
+
const baseInterval = this.config.reconnectInterval || 1000;
|
|
802
|
+
const maxDelay = 30000; // Maximum delay 30 seconds
|
|
803
|
+
const exponentialDelay = baseInterval * Math.pow(2, this.reconnectAttempts - 1);
|
|
804
|
+
const delay = Math.min(exponentialDelay, maxDelay);
|
|
805
|
+
// Add random jitter (±20%) to avoid multiple clients reconnecting simultaneously
|
|
806
|
+
const jitter = delay * 0.2 * (Math.random() * 2 - 1);
|
|
807
|
+
const finalDelay = Math.max(0, delay + jitter);
|
|
808
|
+
await wait(Math.floor(finalDelay));
|
|
809
|
+
void this.init();
|
|
674
810
|
}
|
|
675
811
|
flushQueue() {
|
|
676
|
-
while (this.messageQueue.length > 0 && this.ws) {
|
|
812
|
+
while (this.messageQueue.length > 0 && this.ws && this.shellxConnected) {
|
|
677
813
|
const message = this.messageQueue.shift();
|
|
678
814
|
if (message) {
|
|
679
815
|
try {
|
|
680
|
-
|
|
681
|
-
this.ws.send((
|
|
816
|
+
this.logger.info("flushQueue Sending message:", message);
|
|
817
|
+
this.ws.send(cborEncode(message));
|
|
682
818
|
}
|
|
683
819
|
catch (error) {
|
|
684
|
-
|
|
820
|
+
this.logger.error("Failed to send queued message:", error);
|
|
821
|
+
// Re-queue the message if sending fails
|
|
822
|
+
this.messageQueue.unshift(message);
|
|
823
|
+
break;
|
|
685
824
|
}
|
|
686
825
|
}
|
|
687
826
|
}
|
|
688
827
|
}
|
|
689
828
|
close() {
|
|
690
|
-
var _a;
|
|
691
829
|
this.shellxConnected = false;
|
|
692
830
|
this.wsConnected = false;
|
|
693
831
|
this.authenticated = false;
|
|
694
|
-
|
|
832
|
+
this.ws?.close();
|
|
695
833
|
this.pendingTasks.clear();
|
|
696
834
|
this.messageQueue = [];
|
|
697
835
|
this.stopPing();
|
|
@@ -706,6 +844,12 @@ class ConnectionTaskClient {
|
|
|
706
844
|
get isAuthenticated() {
|
|
707
845
|
return this.authenticated;
|
|
708
846
|
}
|
|
847
|
+
/**
|
|
848
|
+
* Get the authenticated device ID (UUID)
|
|
849
|
+
*/
|
|
850
|
+
getDeviceId() {
|
|
851
|
+
return this.deviceId;
|
|
852
|
+
}
|
|
709
853
|
get pendingTaskCount() {
|
|
710
854
|
return this.pendingTasks.size;
|
|
711
855
|
}
|
|
@@ -713,36 +857,128 @@ class ConnectionTaskClient {
|
|
|
713
857
|
return this.messageQueue.length;
|
|
714
858
|
}
|
|
715
859
|
/**
|
|
716
|
-
*
|
|
860
|
+
* Get current execution log
|
|
861
|
+
*/
|
|
862
|
+
getExecutionLogs() {
|
|
863
|
+
return [...this.executionLogsRef];
|
|
864
|
+
}
|
|
865
|
+
/**
|
|
866
|
+
* Clear current execution log
|
|
867
|
+
*/
|
|
868
|
+
clearExecutionLogs() {
|
|
869
|
+
this.executionLogsRef.length = 0;
|
|
870
|
+
}
|
|
871
|
+
/**
|
|
872
|
+
* Manually append an execution log
|
|
873
|
+
*/
|
|
874
|
+
appendExecutionLog(log) {
|
|
875
|
+
this.logger.info("Append log:", log);
|
|
876
|
+
this.executionLogsRef.push(log);
|
|
877
|
+
}
|
|
878
|
+
/**
|
|
879
|
+
* Set associated ShellX instance
|
|
717
880
|
*/
|
|
718
881
|
setShellX(shellx) {
|
|
719
882
|
this.shellx = shellx;
|
|
720
|
-
|
|
883
|
+
this.logger.info("🔗 [ShellX] ShellX instance associated");
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
886
|
+
* Create a timeout result object based on task type
|
|
887
|
+
* @param taskType Type of the task that timed out
|
|
888
|
+
* @param timeoutMs Timeout duration in milliseconds
|
|
889
|
+
* @returns A result object with success: false
|
|
890
|
+
*/
|
|
891
|
+
createTimeoutResult(taskType, timeoutMs) {
|
|
892
|
+
const baseResult = {
|
|
893
|
+
success: false,
|
|
894
|
+
error: `Task timed out after ${timeoutMs}ms`,
|
|
895
|
+
duration: timeoutMs,
|
|
896
|
+
timestamp: Date.now(),
|
|
897
|
+
};
|
|
898
|
+
// Return type-specific result based on task type
|
|
899
|
+
switch (taskType) {
|
|
900
|
+
case "action":
|
|
901
|
+
// For action tasks, return a basic result
|
|
902
|
+
return baseResult;
|
|
903
|
+
case "screenShot":
|
|
904
|
+
return {
|
|
905
|
+
...baseResult,
|
|
906
|
+
imageData: "",
|
|
907
|
+
imagePath: "",
|
|
908
|
+
format: "png",
|
|
909
|
+
};
|
|
910
|
+
case "screenInfo":
|
|
911
|
+
return {
|
|
912
|
+
...baseResult,
|
|
913
|
+
width: 0,
|
|
914
|
+
height: 0,
|
|
915
|
+
density: 0,
|
|
916
|
+
screenOn: false,
|
|
917
|
+
screenUnlocked: false,
|
|
918
|
+
};
|
|
919
|
+
case "findElement":
|
|
920
|
+
case "waitElement":
|
|
921
|
+
return {
|
|
922
|
+
...baseResult,
|
|
923
|
+
elements: [],
|
|
924
|
+
};
|
|
925
|
+
case "appList":
|
|
926
|
+
return {
|
|
927
|
+
...baseResult,
|
|
928
|
+
apps: [],
|
|
929
|
+
totalCount: 0,
|
|
930
|
+
systemAppCount: 0,
|
|
931
|
+
userAppCount: 0,
|
|
932
|
+
};
|
|
933
|
+
case "input":
|
|
934
|
+
return baseResult;
|
|
935
|
+
case "swipe":
|
|
936
|
+
return {
|
|
937
|
+
...baseResult,
|
|
938
|
+
fromX: 0,
|
|
939
|
+
fromY: 0,
|
|
940
|
+
toX: 0,
|
|
941
|
+
toY: 0,
|
|
942
|
+
};
|
|
943
|
+
case "press":
|
|
944
|
+
return {
|
|
945
|
+
...baseResult,
|
|
946
|
+
key: "",
|
|
947
|
+
};
|
|
948
|
+
case "clipboard":
|
|
949
|
+
return {
|
|
950
|
+
...baseResult,
|
|
951
|
+
text: "",
|
|
952
|
+
};
|
|
953
|
+
default:
|
|
954
|
+
// Default fallback result
|
|
955
|
+
return baseResult;
|
|
956
|
+
}
|
|
721
957
|
}
|
|
722
958
|
/**
|
|
723
|
-
*
|
|
959
|
+
* Get associated ShellX instance
|
|
724
960
|
*/
|
|
725
961
|
getShellX() {
|
|
726
962
|
return this.shellx;
|
|
727
963
|
}
|
|
728
964
|
/**
|
|
729
|
-
*
|
|
730
|
-
* @param taskId
|
|
731
|
-
* @param result
|
|
732
|
-
* @param success
|
|
965
|
+
* Manually complete task by taskId
|
|
966
|
+
* @param taskId Task ID
|
|
967
|
+
* @param result Task result
|
|
968
|
+
* @param success Success status
|
|
733
969
|
*/
|
|
734
970
|
completeTask(taskId, result, success = true) {
|
|
735
971
|
const task = this.pendingTasks.get(taskId);
|
|
736
972
|
if (!task) {
|
|
737
|
-
|
|
973
|
+
this.logger.info(`⚠️ [ShellX] Task not found: ${taskId}`);
|
|
738
974
|
return false;
|
|
739
975
|
}
|
|
740
|
-
|
|
741
|
-
//
|
|
976
|
+
this.logger.info(`✅ [ShellX] Manually completed task: ${taskId}, Task type: ${task.type}, Success: ${success}`);
|
|
977
|
+
// Clear timeout timer
|
|
742
978
|
clearTimeout(task.timer);
|
|
743
|
-
//
|
|
979
|
+
// Remove from pending tasks
|
|
744
980
|
this.pendingTasks.delete(taskId);
|
|
745
|
-
//
|
|
981
|
+
// Call appropriate callback based on success status
|
|
746
982
|
if (success) {
|
|
747
983
|
task.resolve(result || { success: true, taskId });
|
|
748
984
|
}
|
|
@@ -752,13 +988,13 @@ class ConnectionTaskClient {
|
|
|
752
988
|
return true;
|
|
753
989
|
}
|
|
754
990
|
/**
|
|
755
|
-
*
|
|
991
|
+
* Get all pending task IDs
|
|
756
992
|
*/
|
|
757
993
|
getPendingTaskIds() {
|
|
758
994
|
return Array.from(this.pendingTasks.keys());
|
|
759
995
|
}
|
|
760
996
|
/**
|
|
761
|
-
*
|
|
997
|
+
* Get task info by taskId
|
|
762
998
|
*/
|
|
763
999
|
getTaskInfo(taskId) {
|
|
764
1000
|
const task = this.pendingTasks.get(taskId);
|
|
@@ -766,14 +1002,14 @@ class ConnectionTaskClient {
|
|
|
766
1002
|
return null;
|
|
767
1003
|
}
|
|
768
1004
|
return {
|
|
769
|
-
type: task.type
|
|
1005
|
+
type: task.type,
|
|
770
1006
|
};
|
|
771
1007
|
}
|
|
772
1008
|
/**
|
|
773
|
-
*
|
|
774
|
-
* @param taskType
|
|
775
|
-
* @param result
|
|
776
|
-
* @param success
|
|
1009
|
+
* Batch complete tasks by type
|
|
1010
|
+
* @param taskType Task type
|
|
1011
|
+
* @param result Task result
|
|
1012
|
+
* @param success Success status
|
|
777
1013
|
*/
|
|
778
1014
|
completeTasksByType(taskType, result, success = true) {
|
|
779
1015
|
const taskIds = this.getPendingTaskIds();
|
|
@@ -785,12 +1021,16 @@ class ConnectionTaskClient {
|
|
|
785
1021
|
completedCount++;
|
|
786
1022
|
}
|
|
787
1023
|
}
|
|
788
|
-
|
|
1024
|
+
this.logger.info(`📦 [ShellX] Batch completed ${completedCount} tasks of type ${taskType}`);
|
|
789
1025
|
return completedCount;
|
|
790
1026
|
}
|
|
791
1027
|
}
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
1028
|
+
export default ConnectionClient;
|
|
1029
|
+
export { ShellX } from "./shellx.js";
|
|
1030
|
+
// Error handling exports
|
|
1031
|
+
export { createErrorResponse, createOperationResult, ErrorCode, ShellXError, ConnectionError, OperationError, ElementError, ValidationError, } from "./errors.js";
|
|
1032
|
+
// Error handler utilities
|
|
1033
|
+
export { extractErrorMessage, createErrorResult, createSuccessResult, handleOperation, handleOperationWithRetry, validateRequired, validateOneOfRequired, wrapErrorWithContext, } from "./error-handler.js";
|
|
1034
|
+
// Domain management exports
|
|
1035
|
+
export { buildDeviceApiUrl, buildDeviceConfigUrl, buildDeviceRegisterUrl, buildTokenVerifyUrl, buildTokenRefreshUrl, getApiBaseUrl, getWsServerUrl, getCurrentDomainInfo, createDomainConfig, isChineseUser, } from "./domain-manager.js";
|
|
1036
|
+
//# sourceMappingURL=index.js.map
|