recker 1.0.26 → 1.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/browser/cache.d.ts +40 -0
- package/dist/browser/browser/cache.js +199 -0
- package/dist/browser/browser/crypto.d.ts +24 -0
- package/dist/browser/browser/crypto.js +80 -0
- package/dist/browser/browser/index.d.ts +31 -0
- package/dist/browser/browser/index.js +31 -0
- package/dist/browser/browser/recker.d.ts +26 -0
- package/dist/browser/browser/recker.js +61 -0
- package/dist/browser/cache/basic-file-storage.d.ts +12 -0
- package/dist/browser/cache/basic-file-storage.js +50 -0
- package/dist/browser/cache/memory-limits.d.ts +20 -0
- package/dist/browser/cache/memory-limits.js +96 -0
- package/dist/browser/cache/memory-storage.d.ts +132 -0
- package/dist/browser/cache/memory-storage.js +454 -0
- package/dist/browser/cache.d.ts +40 -0
- package/dist/browser/cache.js +199 -0
- package/dist/browser/constants/http-status.d.ts +73 -0
- package/dist/browser/constants/http-status.js +156 -0
- package/dist/browser/cookies/memory-cookie-jar.d.ts +30 -0
- package/dist/browser/cookies/memory-cookie-jar.js +210 -0
- package/dist/browser/core/client.d.ts +118 -0
- package/dist/browser/core/client.js +667 -0
- package/dist/browser/core/errors.d.ts +142 -0
- package/dist/browser/core/errors.js +308 -0
- package/dist/browser/core/index.d.ts +5 -0
- package/dist/browser/core/index.js +5 -0
- package/dist/browser/core/request-promise.d.ts +23 -0
- package/dist/browser/core/request-promise.js +82 -0
- package/dist/browser/core/request.d.ts +20 -0
- package/dist/browser/core/request.js +76 -0
- package/dist/browser/core/response.d.ts +34 -0
- package/dist/browser/core/response.js +178 -0
- package/dist/browser/crypto.d.ts +24 -0
- package/dist/browser/crypto.js +80 -0
- package/dist/browser/index.d.ts +31 -0
- package/dist/browser/index.js +31 -0
- package/dist/browser/plugins/auth/api-key.d.ts +8 -0
- package/dist/browser/plugins/auth/api-key.js +27 -0
- package/dist/browser/plugins/auth/auth0.d.ts +33 -0
- package/dist/browser/plugins/auth/auth0.js +94 -0
- package/dist/browser/plugins/auth/aws-sigv4.d.ts +10 -0
- package/dist/browser/plugins/auth/aws-sigv4.js +88 -0
- package/dist/browser/plugins/auth/azure-ad.d.ts +48 -0
- package/dist/browser/plugins/auth/azure-ad.js +152 -0
- package/dist/browser/plugins/auth/basic.d.ts +7 -0
- package/dist/browser/plugins/auth/basic.js +13 -0
- package/dist/browser/plugins/auth/bearer.d.ts +8 -0
- package/dist/browser/plugins/auth/bearer.js +17 -0
- package/dist/browser/plugins/auth/cognito.d.ts +45 -0
- package/dist/browser/plugins/auth/cognito.js +208 -0
- package/dist/browser/plugins/auth/digest.d.ts +8 -0
- package/dist/browser/plugins/auth/digest.js +100 -0
- package/dist/browser/plugins/auth/firebase.d.ts +32 -0
- package/dist/browser/plugins/auth/firebase.js +195 -0
- package/dist/browser/plugins/auth/github-app.d.ts +36 -0
- package/dist/browser/plugins/auth/github-app.js +170 -0
- package/dist/browser/plugins/auth/google-service-account.d.ts +49 -0
- package/dist/browser/plugins/auth/google-service-account.js +172 -0
- package/dist/browser/plugins/auth/index.d.ts +15 -0
- package/dist/browser/plugins/auth/index.js +15 -0
- package/dist/browser/plugins/auth/mtls.d.ts +37 -0
- package/dist/browser/plugins/auth/mtls.js +140 -0
- package/dist/browser/plugins/auth/oauth2.d.ts +8 -0
- package/dist/browser/plugins/auth/oauth2.js +26 -0
- package/dist/browser/plugins/auth/oidc.d.ts +55 -0
- package/dist/browser/plugins/auth/oidc.js +222 -0
- package/dist/browser/plugins/auth/okta.d.ts +47 -0
- package/dist/browser/plugins/auth/okta.js +157 -0
- package/dist/browser/plugins/auth.d.ts +1 -0
- package/dist/browser/plugins/auth.js +1 -0
- package/dist/browser/plugins/cache.d.ts +15 -0
- package/dist/browser/plugins/cache.js +486 -0
- package/dist/browser/plugins/circuit-breaker.d.ts +13 -0
- package/dist/browser/plugins/circuit-breaker.js +100 -0
- package/dist/browser/plugins/compression.d.ts +4 -0
- package/dist/browser/plugins/compression.js +130 -0
- package/dist/browser/plugins/cookie-jar.d.ts +5 -0
- package/dist/browser/plugins/cookie-jar.js +72 -0
- package/dist/browser/plugins/dedup.d.ts +5 -0
- package/dist/browser/plugins/dedup.js +35 -0
- package/dist/browser/plugins/graphql.d.ts +13 -0
- package/dist/browser/plugins/graphql.js +58 -0
- package/dist/browser/plugins/grpc-web.d.ts +79 -0
- package/dist/browser/plugins/grpc-web.js +261 -0
- package/dist/browser/plugins/hls.d.ts +105 -0
- package/dist/browser/plugins/hls.js +395 -0
- package/dist/browser/plugins/jsonrpc.d.ts +75 -0
- package/dist/browser/plugins/jsonrpc.js +143 -0
- package/dist/browser/plugins/logger.d.ts +13 -0
- package/dist/browser/plugins/logger.js +108 -0
- package/dist/browser/plugins/odata.d.ts +181 -0
- package/dist/browser/plugins/odata.js +564 -0
- package/dist/browser/plugins/pagination.d.ts +16 -0
- package/dist/browser/plugins/pagination.js +105 -0
- package/dist/browser/plugins/rate-limit.d.ts +15 -0
- package/dist/browser/plugins/rate-limit.js +162 -0
- package/dist/browser/plugins/retry.d.ts +14 -0
- package/dist/browser/plugins/retry.js +116 -0
- package/dist/browser/plugins/scrape.d.ts +21 -0
- package/dist/browser/plugins/scrape.js +82 -0
- package/dist/browser/plugins/server-timing.d.ts +7 -0
- package/dist/browser/plugins/server-timing.js +24 -0
- package/dist/browser/plugins/soap.d.ts +72 -0
- package/dist/browser/plugins/soap.js +347 -0
- package/dist/browser/plugins/xml.d.ts +9 -0
- package/dist/browser/plugins/xml.js +194 -0
- package/dist/browser/plugins/xsrf.d.ts +9 -0
- package/dist/browser/plugins/xsrf.js +48 -0
- package/dist/browser/recker.d.ts +26 -0
- package/dist/browser/recker.js +61 -0
- package/dist/browser/runner/request-runner.d.ts +46 -0
- package/dist/browser/runner/request-runner.js +89 -0
- package/dist/browser/scrape/document.d.ts +44 -0
- package/dist/browser/scrape/document.js +210 -0
- package/dist/browser/scrape/element.d.ts +49 -0
- package/dist/browser/scrape/element.js +176 -0
- package/dist/browser/scrape/extractors.d.ts +16 -0
- package/dist/browser/scrape/extractors.js +356 -0
- package/dist/browser/scrape/types.d.ts +107 -0
- package/dist/browser/scrape/types.js +1 -0
- package/dist/browser/transport/fetch.d.ts +11 -0
- package/dist/browser/transport/fetch.js +143 -0
- package/dist/browser/transport/undici.d.ts +38 -0
- package/dist/browser/transport/undici.js +897 -0
- package/dist/browser/types/ai.d.ts +267 -0
- package/dist/browser/types/ai.js +1 -0
- package/dist/browser/types/index.d.ts +351 -0
- package/dist/browser/types/index.js +1 -0
- package/dist/browser/types/logger.d.ts +16 -0
- package/dist/browser/types/logger.js +66 -0
- package/dist/browser/types/udp.d.ts +138 -0
- package/dist/browser/types/udp.js +1 -0
- package/dist/browser/utils/agent-manager.d.ts +29 -0
- package/dist/browser/utils/agent-manager.js +160 -0
- package/dist/browser/utils/body.d.ts +10 -0
- package/dist/browser/utils/body.js +148 -0
- package/dist/browser/utils/charset.d.ts +15 -0
- package/dist/browser/utils/charset.js +169 -0
- package/dist/browser/utils/concurrency.d.ts +20 -0
- package/dist/browser/utils/concurrency.js +120 -0
- package/dist/browser/utils/dns.d.ts +6 -0
- package/dist/browser/utils/dns.js +26 -0
- package/dist/browser/utils/header-parser.d.ts +94 -0
- package/dist/browser/utils/header-parser.js +617 -0
- package/dist/browser/utils/html-cleaner.d.ts +1 -0
- package/dist/browser/utils/html-cleaner.js +21 -0
- package/dist/browser/utils/link-header.d.ts +69 -0
- package/dist/browser/utils/link-header.js +190 -0
- package/dist/browser/utils/optional-require.d.ts +19 -0
- package/dist/browser/utils/optional-require.js +105 -0
- package/dist/browser/utils/progress.d.ts +8 -0
- package/dist/browser/utils/progress.js +82 -0
- package/dist/browser/utils/request-pool.d.ts +22 -0
- package/dist/browser/utils/request-pool.js +101 -0
- package/dist/browser/utils/sse.d.ts +7 -0
- package/dist/browser/utils/sse.js +67 -0
- package/dist/browser/utils/streaming.d.ts +17 -0
- package/dist/browser/utils/streaming.js +84 -0
- package/dist/browser/utils/try-fn.d.ts +3 -0
- package/dist/browser/utils/try-fn.js +59 -0
- package/dist/browser/utils/user-agent.d.ts +44 -0
- package/dist/browser/utils/user-agent.js +100 -0
- package/dist/browser/utils/whois.d.ts +32 -0
- package/dist/browser/utils/whois.js +246 -0
- package/dist/browser/websocket/client.d.ts +65 -0
- package/dist/browser/websocket/client.js +313 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +1 -0
- package/dist/transport/fetch.d.ts +7 -1
- package/dist/transport/fetch.js +58 -76
- package/package.json +34 -2
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
import { WebSocket } from 'undici';
|
|
2
|
+
import { EventEmitter } from 'events';
|
|
3
|
+
import { pipeline } from 'node:stream/promises';
|
|
4
|
+
import { webToNodeStream } from '../utils/streaming.js';
|
|
5
|
+
import { StateError, StreamError, ConnectionError } from '../core/errors.js';
|
|
6
|
+
export class ReckerWebSocket extends EventEmitter {
|
|
7
|
+
ws = null;
|
|
8
|
+
url;
|
|
9
|
+
options;
|
|
10
|
+
reconnectAttempts = 0;
|
|
11
|
+
reconnectTimer;
|
|
12
|
+
heartbeatTimer;
|
|
13
|
+
isClosed = false;
|
|
14
|
+
isReconnecting = false;
|
|
15
|
+
pongWatchdog;
|
|
16
|
+
backoff;
|
|
17
|
+
closedByUser = false;
|
|
18
|
+
constructor(url, options = {}) {
|
|
19
|
+
super();
|
|
20
|
+
this.url = url;
|
|
21
|
+
this.options = {
|
|
22
|
+
protocols: options.protocols || [],
|
|
23
|
+
headers: options.headers || {},
|
|
24
|
+
reconnect: options.reconnect ?? false,
|
|
25
|
+
reconnectDelay: options.reconnectDelay ?? 1000,
|
|
26
|
+
maxReconnectAttempts: options.maxReconnectAttempts ?? 5,
|
|
27
|
+
heartbeatInterval: options.heartbeatInterval ?? 30000,
|
|
28
|
+
heartbeatTimeout: options.heartbeatTimeout ?? 10000,
|
|
29
|
+
dispatcher: options.dispatcher,
|
|
30
|
+
proxy: options.proxy,
|
|
31
|
+
tls: options.tls,
|
|
32
|
+
perMessageDeflate: options.perMessageDeflate ?? false
|
|
33
|
+
};
|
|
34
|
+
this.backoff = {
|
|
35
|
+
base: this.options.reconnectDelay,
|
|
36
|
+
factor: 2,
|
|
37
|
+
jitter: true,
|
|
38
|
+
max: 30000
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
async connect() {
|
|
42
|
+
return new Promise((resolve, reject) => {
|
|
43
|
+
try {
|
|
44
|
+
const wsOptions = {
|
|
45
|
+
headers: this.options.headers,
|
|
46
|
+
dispatcher: this.options.dispatcher,
|
|
47
|
+
perMessageDeflate: this.options.perMessageDeflate,
|
|
48
|
+
};
|
|
49
|
+
if (this.options.proxy) {
|
|
50
|
+
const proxyConfig = typeof this.options.proxy === 'string'
|
|
51
|
+
? { url: this.options.proxy }
|
|
52
|
+
: this.options.proxy;
|
|
53
|
+
const { ProxyAgent } = require('undici');
|
|
54
|
+
wsOptions.dispatcher = new ProxyAgent(proxyConfig.url);
|
|
55
|
+
}
|
|
56
|
+
if (this.options.tls) {
|
|
57
|
+
wsOptions.tls = this.options.tls;
|
|
58
|
+
}
|
|
59
|
+
this.ws = new WebSocket(this.url, this.options.protocols, wsOptions);
|
|
60
|
+
this.ws.addEventListener('open', () => {
|
|
61
|
+
this.reconnectAttempts = 0;
|
|
62
|
+
this.isReconnecting = false;
|
|
63
|
+
this.startHeartbeat();
|
|
64
|
+
this.emit('open');
|
|
65
|
+
resolve();
|
|
66
|
+
});
|
|
67
|
+
this.ws.addEventListener('message', (event) => {
|
|
68
|
+
const message = {
|
|
69
|
+
data: event.data,
|
|
70
|
+
isBinary: event.data instanceof Buffer
|
|
71
|
+
};
|
|
72
|
+
this.emit('message', message);
|
|
73
|
+
this.stopPongWatchdog();
|
|
74
|
+
});
|
|
75
|
+
this.ws.addEventListener('close', (event) => {
|
|
76
|
+
this.stopHeartbeat();
|
|
77
|
+
this.stopPongWatchdog();
|
|
78
|
+
this.emit('close', event.code, event.reason);
|
|
79
|
+
if (!this.closedByUser && !this.isClosed && this.options.reconnect) {
|
|
80
|
+
this.attemptReconnect();
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
this.ws.addEventListener('error', (event) => {
|
|
84
|
+
const err = event.error instanceof Error
|
|
85
|
+
? event.error
|
|
86
|
+
: new ConnectionError('WebSocket connection error', {
|
|
87
|
+
host: this.url,
|
|
88
|
+
retriable: true,
|
|
89
|
+
});
|
|
90
|
+
this.emit('error', err);
|
|
91
|
+
reject(err);
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
reject(error);
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
async send(data, options) {
|
|
100
|
+
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
|
101
|
+
throw new StateError('WebSocket is not connected', {
|
|
102
|
+
expectedState: 'open',
|
|
103
|
+
actualState: this.ws ? 'closed' : 'not-created',
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
const awaitDrain = options?.awaitDrain ?? false;
|
|
107
|
+
const highWaterMark = options?.highWaterMark ?? 16 * 1024;
|
|
108
|
+
this.ws.send(data);
|
|
109
|
+
if (awaitDrain) {
|
|
110
|
+
await this.waitForDrain(highWaterMark);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
async sendStream(stream, options) {
|
|
114
|
+
for await (const chunk of stream) {
|
|
115
|
+
await this.send(chunk, options);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
sendJSON(data) {
|
|
119
|
+
void this.send(JSON.stringify(data));
|
|
120
|
+
}
|
|
121
|
+
close(code = 1000, reason = '') {
|
|
122
|
+
this.isClosed = true;
|
|
123
|
+
this.closedByUser = true;
|
|
124
|
+
this.stopHeartbeat();
|
|
125
|
+
this.clearReconnectTimer();
|
|
126
|
+
if (this.ws) {
|
|
127
|
+
this.ws.close(code, reason);
|
|
128
|
+
this.ws = null;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
ping() {
|
|
132
|
+
if (!this.ws || this.ws.readyState !== WebSocket.OPEN)
|
|
133
|
+
return;
|
|
134
|
+
const anyWs = this.ws;
|
|
135
|
+
if (typeof anyWs.ping === 'function') {
|
|
136
|
+
try {
|
|
137
|
+
anyWs.ping();
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
try {
|
|
144
|
+
this.ws.send('__heartbeat__');
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
get readyState() {
|
|
150
|
+
return this.ws?.readyState ?? WebSocket.CLOSED;
|
|
151
|
+
}
|
|
152
|
+
get isConnected() {
|
|
153
|
+
return this.ws?.readyState === WebSocket.OPEN;
|
|
154
|
+
}
|
|
155
|
+
toReadable() {
|
|
156
|
+
if (!this.ws)
|
|
157
|
+
return null;
|
|
158
|
+
const wsAny = this.ws;
|
|
159
|
+
if (wsAny.readable) {
|
|
160
|
+
return webToNodeStream(wsAny.readable);
|
|
161
|
+
}
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
async pipeFrom(source, options) {
|
|
165
|
+
await this.sendStream(source, options);
|
|
166
|
+
}
|
|
167
|
+
async pipeTo(destination) {
|
|
168
|
+
const readable = this.toReadable();
|
|
169
|
+
if (!readable) {
|
|
170
|
+
throw new StreamError('WebSocket has no readable stream', {
|
|
171
|
+
streamType: 'websocket',
|
|
172
|
+
retriable: false,
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
await pipeline(readable, destination);
|
|
176
|
+
}
|
|
177
|
+
async *[Symbol.asyncIterator]() {
|
|
178
|
+
const queue = [];
|
|
179
|
+
let resolveNext = null;
|
|
180
|
+
let closed = false;
|
|
181
|
+
const messageHandler = (msg) => {
|
|
182
|
+
if (resolveNext) {
|
|
183
|
+
resolveNext(msg);
|
|
184
|
+
resolveNext = null;
|
|
185
|
+
}
|
|
186
|
+
else {
|
|
187
|
+
queue.push(msg);
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
const closeHandler = () => {
|
|
191
|
+
closed = true;
|
|
192
|
+
if (resolveNext) {
|
|
193
|
+
resolveNext(null);
|
|
194
|
+
resolveNext = null;
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
this.on('message', messageHandler);
|
|
198
|
+
this.on('close', closeHandler);
|
|
199
|
+
try {
|
|
200
|
+
while (true) {
|
|
201
|
+
if (queue.length > 0) {
|
|
202
|
+
yield queue.shift();
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
if (closed)
|
|
206
|
+
break;
|
|
207
|
+
const msg = await new Promise((resolve) => {
|
|
208
|
+
resolveNext = resolve;
|
|
209
|
+
});
|
|
210
|
+
if (msg) {
|
|
211
|
+
yield msg;
|
|
212
|
+
}
|
|
213
|
+
else {
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
finally {
|
|
220
|
+
this.off('message', messageHandler);
|
|
221
|
+
this.off('close', closeHandler);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
attemptReconnect() {
|
|
225
|
+
if (this.isReconnecting)
|
|
226
|
+
return;
|
|
227
|
+
if (this.options.maxReconnectAttempts > 0 &&
|
|
228
|
+
this.reconnectAttempts >= this.options.maxReconnectAttempts) {
|
|
229
|
+
this.emit('max-reconnect-attempts');
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
this.isReconnecting = true;
|
|
233
|
+
this.reconnectAttempts++;
|
|
234
|
+
const baseDelay = this.backoff.base * Math.pow(this.backoff.factor, this.reconnectAttempts - 1);
|
|
235
|
+
const capped = this.backoff.max ? Math.min(baseDelay, this.backoff.max) : baseDelay;
|
|
236
|
+
const jittered = this.backoff.jitter ? randomJitter(capped) : capped;
|
|
237
|
+
this.emit('reconnecting', this.reconnectAttempts, jittered);
|
|
238
|
+
this.reconnectTimer = setTimeout(() => {
|
|
239
|
+
this.connect().catch((error) => {
|
|
240
|
+
this.emit('reconnect-error', error);
|
|
241
|
+
});
|
|
242
|
+
}, jittered);
|
|
243
|
+
}
|
|
244
|
+
clearReconnectTimer() {
|
|
245
|
+
if (this.reconnectTimer) {
|
|
246
|
+
clearTimeout(this.reconnectTimer);
|
|
247
|
+
this.reconnectTimer = undefined;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
startHeartbeat() {
|
|
251
|
+
if (this.options.heartbeatInterval <= 0)
|
|
252
|
+
return;
|
|
253
|
+
this.heartbeatTimer = setInterval(() => {
|
|
254
|
+
if (this.isConnected) {
|
|
255
|
+
this.ping();
|
|
256
|
+
this.startPongWatchdog();
|
|
257
|
+
}
|
|
258
|
+
}, this.options.heartbeatInterval);
|
|
259
|
+
}
|
|
260
|
+
stopHeartbeat() {
|
|
261
|
+
if (this.heartbeatTimer) {
|
|
262
|
+
clearInterval(this.heartbeatTimer);
|
|
263
|
+
this.heartbeatTimer = undefined;
|
|
264
|
+
}
|
|
265
|
+
this.stopPongWatchdog();
|
|
266
|
+
}
|
|
267
|
+
startPongWatchdog() {
|
|
268
|
+
this.stopPongWatchdog();
|
|
269
|
+
if (this.options.heartbeatTimeout <= 0)
|
|
270
|
+
return;
|
|
271
|
+
this.pongWatchdog = setTimeout(() => {
|
|
272
|
+
this.emit('heartbeat-timeout');
|
|
273
|
+
if (!this.closedByUser && this.options.reconnect) {
|
|
274
|
+
this.ws?.close(4000, 'heartbeat timeout');
|
|
275
|
+
}
|
|
276
|
+
}, this.options.heartbeatTimeout);
|
|
277
|
+
}
|
|
278
|
+
stopPongWatchdog() {
|
|
279
|
+
if (this.pongWatchdog) {
|
|
280
|
+
clearTimeout(this.pongWatchdog);
|
|
281
|
+
this.pongWatchdog = undefined;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
getBufferedAmount() {
|
|
285
|
+
return this.ws?.bufferedAmount ?? 0;
|
|
286
|
+
}
|
|
287
|
+
async waitForDrain(highWaterMark) {
|
|
288
|
+
const buffered = this.getBufferedAmount();
|
|
289
|
+
if (buffered <= highWaterMark)
|
|
290
|
+
return;
|
|
291
|
+
await new Promise((resolve) => {
|
|
292
|
+
const check = () => {
|
|
293
|
+
if (this.getBufferedAmount() <= highWaterMark || !this.isConnected) {
|
|
294
|
+
resolve();
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
setTimeout(check, 10);
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
setTimeout(check, 10);
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
export function createWebSocket(url, options) {
|
|
305
|
+
const ws = new ReckerWebSocket(url, options);
|
|
306
|
+
ws.connect().catch(() => {
|
|
307
|
+
});
|
|
308
|
+
return ws;
|
|
309
|
+
}
|
|
310
|
+
function randomJitter(value) {
|
|
311
|
+
const jitter = 0.2 * value;
|
|
312
|
+
return value - jitter + Math.random() * (2 * jitter);
|
|
313
|
+
}
|
package/dist/cli/index.d.ts
CHANGED
package/dist/cli/index.js
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { ReckerRequest, ReckerResponse, Transport } from '../types/index.js';
|
|
2
|
+
export interface FetchTransportOptions {
|
|
3
|
+
credentials?: RequestCredentials;
|
|
4
|
+
cache?: RequestCache;
|
|
5
|
+
keepalive?: boolean;
|
|
6
|
+
}
|
|
2
7
|
export declare class FetchTransport implements Transport {
|
|
3
|
-
|
|
8
|
+
private options;
|
|
9
|
+
constructor(options?: FetchTransportOptions);
|
|
4
10
|
dispatch(req: ReckerRequest): Promise<ReckerResponse>;
|
|
5
11
|
}
|
package/dist/transport/fetch.js
CHANGED
|
@@ -1,12 +1,29 @@
|
|
|
1
1
|
export class FetchTransport {
|
|
2
|
-
|
|
2
|
+
options;
|
|
3
|
+
constructor(options = {}) {
|
|
4
|
+
this.options = options;
|
|
5
|
+
}
|
|
3
6
|
async dispatch(req) {
|
|
4
7
|
const start = performance.now();
|
|
8
|
+
let timeoutId;
|
|
9
|
+
let abortController;
|
|
10
|
+
const timeoutMs = typeof req.timeout === 'number'
|
|
11
|
+
? req.timeout
|
|
12
|
+
: req.timeout?.request;
|
|
13
|
+
let signal = req.signal;
|
|
14
|
+
if (timeoutMs && !signal) {
|
|
15
|
+
abortController = new AbortController();
|
|
16
|
+
signal = abortController.signal;
|
|
17
|
+
timeoutId = setTimeout(() => abortController.abort(), timeoutMs);
|
|
18
|
+
}
|
|
5
19
|
const requestInit = {
|
|
6
20
|
method: req.method,
|
|
7
21
|
headers: req.headers,
|
|
8
22
|
body: req.body,
|
|
9
|
-
signal
|
|
23
|
+
signal,
|
|
24
|
+
credentials: this.options.credentials,
|
|
25
|
+
cache: this.options.cache,
|
|
26
|
+
keepalive: this.options.keepalive ?? true,
|
|
10
27
|
duplex: req.body ? 'half' : undefined
|
|
11
28
|
};
|
|
12
29
|
try {
|
|
@@ -16,84 +33,22 @@ export class FetchTransport {
|
|
|
16
33
|
total: totalTime,
|
|
17
34
|
firstByte: totalTime,
|
|
18
35
|
};
|
|
19
|
-
|
|
20
|
-
status: response.status,
|
|
21
|
-
statusText: response.statusText,
|
|
22
|
-
headers: response.headers,
|
|
23
|
-
ok: response.ok,
|
|
24
|
-
url: response.url,
|
|
25
|
-
timings,
|
|
26
|
-
raw: response,
|
|
27
|
-
json: () => response.json(),
|
|
28
|
-
text: () => response.text(),
|
|
29
|
-
blob: () => response.blob(),
|
|
30
|
-
cleanText: async () => {
|
|
31
|
-
const text = await response.text();
|
|
32
|
-
return text.replace(/<[^>]*>?/gm, '');
|
|
33
|
-
},
|
|
34
|
-
read: () => response.body,
|
|
35
|
-
clone: () => {
|
|
36
|
-
const cloned = response.clone();
|
|
37
|
-
return createReckerResponseWrapper(cloned, timings);
|
|
38
|
-
},
|
|
39
|
-
async *sse() {
|
|
40
|
-
if (!response.body)
|
|
41
|
-
return;
|
|
42
|
-
const reader = response.body.pipeThrough(new TextDecoderStream()).getReader();
|
|
43
|
-
while (true) {
|
|
44
|
-
const { done, value } = await reader.read();
|
|
45
|
-
if (done)
|
|
46
|
-
break;
|
|
47
|
-
const lines = value.split('\n');
|
|
48
|
-
for (const line of lines) {
|
|
49
|
-
if (line.startsWith('data: ')) {
|
|
50
|
-
yield { data: line.slice(6) };
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
},
|
|
55
|
-
async *download() {
|
|
56
|
-
if (!response.body)
|
|
57
|
-
return;
|
|
58
|
-
const reader = response.body.getReader();
|
|
59
|
-
let loaded = 0;
|
|
60
|
-
const total = Number(response.headers.get('content-length')) || undefined;
|
|
61
|
-
while (true) {
|
|
62
|
-
const { done, value } = await reader.read();
|
|
63
|
-
if (done)
|
|
64
|
-
break;
|
|
65
|
-
loaded += value.length;
|
|
66
|
-
yield {
|
|
67
|
-
loaded,
|
|
68
|
-
transferred: loaded,
|
|
69
|
-
total,
|
|
70
|
-
percent: total ? (loaded / total) * 100 : undefined,
|
|
71
|
-
direction: 'download'
|
|
72
|
-
};
|
|
73
|
-
}
|
|
74
|
-
},
|
|
75
|
-
async *[Symbol.asyncIterator]() {
|
|
76
|
-
if (!response.body)
|
|
77
|
-
return;
|
|
78
|
-
const reader = response.body.getReader();
|
|
79
|
-
while (true) {
|
|
80
|
-
const { done, value } = await reader.read();
|
|
81
|
-
if (done)
|
|
82
|
-
break;
|
|
83
|
-
yield value;
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
};
|
|
87
|
-
return reckerResponse;
|
|
36
|
+
return new FetchResponseWrapper(response, timings);
|
|
88
37
|
}
|
|
89
38
|
catch (error) {
|
|
39
|
+
if (error.name === 'AbortError' && abortController) {
|
|
40
|
+
const timeoutError = new Error(`Request timeout after ${timeoutMs}ms`);
|
|
41
|
+
timeoutError.name = 'TimeoutError';
|
|
42
|
+
throw timeoutError;
|
|
43
|
+
}
|
|
90
44
|
throw error;
|
|
91
45
|
}
|
|
46
|
+
finally {
|
|
47
|
+
if (timeoutId)
|
|
48
|
+
clearTimeout(timeoutId);
|
|
49
|
+
}
|
|
92
50
|
}
|
|
93
51
|
}
|
|
94
|
-
function createReckerResponseWrapper(response, timings) {
|
|
95
|
-
return new FetchResponseWrapper(response, timings);
|
|
96
|
-
}
|
|
97
52
|
class FetchResponseWrapper {
|
|
98
53
|
raw;
|
|
99
54
|
timings;
|
|
@@ -118,12 +73,39 @@ class FetchResponseWrapper {
|
|
|
118
73
|
return;
|
|
119
74
|
const stream = this.raw.body.pipeThrough(new TextDecoderStream());
|
|
120
75
|
const reader = stream.getReader();
|
|
76
|
+
let buffer = '';
|
|
121
77
|
while (true) {
|
|
122
78
|
const { done, value } = await reader.read();
|
|
123
79
|
if (done)
|
|
124
80
|
break;
|
|
125
|
-
|
|
126
|
-
|
|
81
|
+
buffer += value;
|
|
82
|
+
const events = buffer.split('\n\n');
|
|
83
|
+
buffer = events.pop() || '';
|
|
84
|
+
for (const event of events) {
|
|
85
|
+
if (!event.trim())
|
|
86
|
+
continue;
|
|
87
|
+
let data = '';
|
|
88
|
+
let eventType;
|
|
89
|
+
let id;
|
|
90
|
+
let retry;
|
|
91
|
+
const lines = event.split('\n');
|
|
92
|
+
for (const line of lines) {
|
|
93
|
+
if (line.startsWith('data: ')) {
|
|
94
|
+
data = data ? data + '\n' + line.slice(6) : line.slice(6);
|
|
95
|
+
}
|
|
96
|
+
else if (line.startsWith('event: ')) {
|
|
97
|
+
eventType = line.slice(7);
|
|
98
|
+
}
|
|
99
|
+
else if (line.startsWith('id: ')) {
|
|
100
|
+
id = line.slice(4);
|
|
101
|
+
}
|
|
102
|
+
else if (line.startsWith('retry: ')) {
|
|
103
|
+
retry = parseInt(line.slice(7), 10);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
if (data) {
|
|
107
|
+
yield { data, event: eventType, id, retry };
|
|
108
|
+
}
|
|
127
109
|
}
|
|
128
110
|
}
|
|
129
111
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "recker",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.27",
|
|
4
4
|
"description": "AI & DevX focused HTTP client for Node.js 18+",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -58,6 +58,24 @@
|
|
|
58
58
|
],
|
|
59
59
|
"exports": {
|
|
60
60
|
".": {
|
|
61
|
+
"types": "./dist/index.d.ts",
|
|
62
|
+
"node": {
|
|
63
|
+
"import": "./dist/index.js",
|
|
64
|
+
"require": "./dist/index.cjs"
|
|
65
|
+
},
|
|
66
|
+
"browser": {
|
|
67
|
+
"import": "./dist/browser/browser/index.js",
|
|
68
|
+
"default": "./dist/browser/browser/index.js"
|
|
69
|
+
},
|
|
70
|
+
"import": "./dist/index.js",
|
|
71
|
+
"default": "./dist/index.js"
|
|
72
|
+
},
|
|
73
|
+
"./browser": {
|
|
74
|
+
"types": "./dist/browser/browser/index.d.ts",
|
|
75
|
+
"import": "./dist/browser/browser/index.js",
|
|
76
|
+
"default": "./dist/browser/browser/index.js"
|
|
77
|
+
},
|
|
78
|
+
"./node": {
|
|
61
79
|
"types": "./dist/index.d.ts",
|
|
62
80
|
"import": "./dist/index.js",
|
|
63
81
|
"default": "./dist/index.js"
|
|
@@ -114,8 +132,15 @@
|
|
|
114
132
|
"types": "./dist/mini.d.ts",
|
|
115
133
|
"import": "./dist/mini.js"
|
|
116
134
|
},
|
|
135
|
+
"./presets/*": {
|
|
136
|
+
"types": "./dist/presets/*.d.ts",
|
|
137
|
+
"import": "./dist/presets/*.js"
|
|
138
|
+
},
|
|
117
139
|
"./package.json": "./package.json"
|
|
118
140
|
},
|
|
141
|
+
"browser": "./dist/browser/browser/index.js",
|
|
142
|
+
"unpkg": "./dist/browser/index.umd.min.js",
|
|
143
|
+
"jsdelivr": "./dist/browser/index.umd.min.js",
|
|
119
144
|
"bin": {
|
|
120
145
|
"rek": "./dist/cli/index.js",
|
|
121
146
|
"recker": "./dist/cli/index.js"
|
|
@@ -164,8 +189,10 @@
|
|
|
164
189
|
"commander": "^14.0.0",
|
|
165
190
|
"cross-fetch": "^4.1.0",
|
|
166
191
|
"domhandler": "^5.0.3",
|
|
192
|
+
"esbuild": "^0.24.2",
|
|
167
193
|
"fastembed": "^2.0.0",
|
|
168
194
|
"got": "^14.6.5",
|
|
195
|
+
"happy-dom": "^20.0.11",
|
|
169
196
|
"husky": "^9.1.7",
|
|
170
197
|
"ky": "^1.14.0",
|
|
171
198
|
"make-fetch-happen": "^15.0.3",
|
|
@@ -186,11 +213,16 @@
|
|
|
186
213
|
"zod": "^4.1.13"
|
|
187
214
|
},
|
|
188
215
|
"scripts": {
|
|
189
|
-
"build": "
|
|
216
|
+
"build": "pnpm build:node && pnpm build:browser",
|
|
217
|
+
"build:node": "tsc",
|
|
218
|
+
"build:browser": "tsc -p tsconfig.browser.json",
|
|
219
|
+
"build:browser:bundle": "pnpm build:browser && node scripts/bundle-browser.js",
|
|
190
220
|
"build:embeddings": "tsx scripts/build-embeddings.ts",
|
|
191
221
|
"test": "NODE_OPTIONS='--max-old-space-size=512' vitest run",
|
|
192
222
|
"test:coverage": "NODE_OPTIONS='--max-old-space-size=512' vitest run --coverage",
|
|
193
223
|
"test:light": "NODE_OPTIONS='--max-old-space-size=256' vitest run --poolOptions.forks.maxForks=1",
|
|
224
|
+
"test:browser": "vitest run --config vitest.config.browser.ts",
|
|
225
|
+
"test:browser:coverage": "vitest run --config vitest.config.browser.ts --coverage",
|
|
194
226
|
"bench": "tsx benchmark/index.ts",
|
|
195
227
|
"bench:compare": "tsx benchmark/http-clients-comparison.ts",
|
|
196
228
|
"bench:all": "tsx benchmark/run-all.ts",
|