rezo 1.0.74 → 1.0.76

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/adapters/entries/curl.d.ts +4 -1
  2. package/dist/adapters/entries/fetch.d.ts +4 -1
  3. package/dist/adapters/entries/http.d.ts +4 -1
  4. package/dist/adapters/entries/http2.d.ts +4 -1
  5. package/dist/adapters/entries/react-native.d.ts +4 -1
  6. package/dist/adapters/entries/xhr.d.ts +4 -1
  7. package/dist/adapters/http.cjs +2 -1
  8. package/dist/adapters/http.js +2 -1
  9. package/dist/adapters/index.cjs +6 -6
  10. package/dist/cache/index.cjs +9 -9
  11. package/dist/crawler/crawler-options.cjs +1 -1
  12. package/dist/crawler/crawler-options.js +1 -1
  13. package/dist/crawler/crawler.cjs +72 -1
  14. package/dist/crawler/crawler.js +72 -1
  15. package/dist/crawler/index.cjs +40 -40
  16. package/dist/crawler/plugin/index.cjs +1 -1
  17. package/dist/crawler.d.ts +101 -0
  18. package/dist/entries/crawler.cjs +4 -4
  19. package/dist/index.cjs +30 -30
  20. package/dist/index.d.ts +4 -1
  21. package/dist/internal/agents/bun-socks-http.cjs +573 -0
  22. package/dist/internal/agents/bun-socks-http.js +570 -0
  23. package/dist/internal/agents/index.cjs +14 -10
  24. package/dist/internal/agents/index.js +1 -0
  25. package/dist/platform/browser.d.ts +4 -1
  26. package/dist/platform/bun.d.ts +4 -1
  27. package/dist/platform/deno.d.ts +4 -1
  28. package/dist/platform/node.d.ts +4 -1
  29. package/dist/platform/react-native.d.ts +4 -1
  30. package/dist/platform/worker.d.ts +4 -1
  31. package/dist/proxy/index.cjs +4 -4
  32. package/dist/queue/index.cjs +8 -8
  33. package/dist/queue/queue.cjs +3 -0
  34. package/dist/queue/queue.js +3 -0
  35. package/dist/responses/universal/index.cjs +11 -11
  36. package/dist/version.cjs +1 -1
  37. package/dist/version.js +1 -1
  38. package/dist/wget/index.cjs +49 -49
  39. package/dist/wget/index.d.ts +3 -0
  40. package/package.json +1 -1
@@ -0,0 +1,573 @@
1
+ const { EventEmitter } = require("node:events");
2
+ const { Readable } = require("node:stream");
3
+ const net = require("node:net");
4
+ const tls = require("node:tls");
5
+ const { SocksClient } = require('./socks-client.cjs');
6
+ function isBunRuntime() {
7
+ return typeof globalThis.Bun !== "undefined";
8
+ }
9
+ function isBunSocksRequest(proxy) {
10
+ if (!isBunRuntime() || !proxy)
11
+ return false;
12
+ if (typeof proxy === "string") {
13
+ return proxy.startsWith("socks");
14
+ }
15
+ return proxy.protocol?.startsWith("socks") ?? false;
16
+ }
17
+
18
+ class BunSocksIncomingMessage extends Readable {
19
+ httpVersion = "1.1";
20
+ httpVersionMajor = 1;
21
+ httpVersionMinor = 1;
22
+ complete = false;
23
+ headers = {};
24
+ rawHeaders = [];
25
+ trailers = {};
26
+ rawTrailers = [];
27
+ statusCode = 0;
28
+ statusMessage = "";
29
+ socket;
30
+ url = "";
31
+ method = "";
32
+ aborted = false;
33
+ get connection() {
34
+ return this.socket;
35
+ }
36
+ _finished = false;
37
+ constructor(socket) {
38
+ super();
39
+ this.socket = socket;
40
+ }
41
+ _setHeaders(headers, rawHeaders) {
42
+ this.headers = headers;
43
+ this.rawHeaders = rawHeaders;
44
+ }
45
+ _setStatus(code, message) {
46
+ this.statusCode = code;
47
+ this.statusMessage = message;
48
+ }
49
+ _pushChunk(chunk) {
50
+ if (!this._finished) {
51
+ this.push(chunk);
52
+ }
53
+ }
54
+ _finish() {
55
+ this._finished = true;
56
+ this.complete = true;
57
+ this.push(null);
58
+ }
59
+ _read() {}
60
+ setTimeout(msecs, callback) {
61
+ this.socket.setTimeout(msecs, callback);
62
+ return this;
63
+ }
64
+ destroy(error) {
65
+ this.aborted = true;
66
+ this.socket.destroy(error);
67
+ return super.destroy(error);
68
+ }
69
+ }
70
+
71
+ class BunSocksClientRequest extends EventEmitter {
72
+ method;
73
+ path;
74
+ host;
75
+ protocol;
76
+ maxHeadersCount = 2000;
77
+ reusedSocket = false;
78
+ writableEnded = false;
79
+ writableFinished = false;
80
+ destroyed = false;
81
+ upgrading = false;
82
+ chunkedEncoding = false;
83
+ shouldKeepAlive = true;
84
+ useChunkedEncodingByDefault = true;
85
+ sendDate = true;
86
+ get finished() {
87
+ return this.writableEnded;
88
+ }
89
+ _proxy;
90
+ _options;
91
+ _headers;
92
+ _body = [];
93
+ _socket = null;
94
+ _finished = false;
95
+ _aborted = false;
96
+ _timeout = 0;
97
+ _timeoutTimer = null;
98
+ _headersSent = false;
99
+ get socket() {
100
+ return this._socket;
101
+ }
102
+ get connection() {
103
+ return this._socket;
104
+ }
105
+ constructor(proxy, options) {
106
+ super();
107
+ this._proxy = proxy;
108
+ this._options = options;
109
+ this._headers = options.headers && typeof options.headers === "object" && !Array.isArray(options.headers) ? { ...options.headers } : {};
110
+ this.method = options.method || "GET";
111
+ this.path = options.path || "/";
112
+ this.host = options.hostname || options.host || "localhost";
113
+ this.protocol = options.protocol || "http:";
114
+ }
115
+ setHeader(name, value) {
116
+ this._headers[name.toLowerCase()] = value;
117
+ return this;
118
+ }
119
+ getHeader(name) {
120
+ return this._headers[name.toLowerCase()];
121
+ }
122
+ getHeaders() {
123
+ return { ...this._headers };
124
+ }
125
+ getHeaderNames() {
126
+ return Object.keys(this._headers);
127
+ }
128
+ getRawHeaderNames() {
129
+ return Object.keys(this._headers);
130
+ }
131
+ hasHeader(name) {
132
+ return name.toLowerCase() in this._headers;
133
+ }
134
+ removeHeader(name) {
135
+ delete this._headers[name.toLowerCase()];
136
+ }
137
+ flushHeaders() {
138
+ this._headersSent = true;
139
+ }
140
+ get headersSent() {
141
+ return this._headersSent;
142
+ }
143
+ setNoDelay(noDelay) {
144
+ if (this._socket && "setNoDelay" in this._socket) {
145
+ this._socket.setNoDelay(noDelay);
146
+ }
147
+ }
148
+ setSocketKeepAlive(enable, initialDelay) {
149
+ if (this._socket && "setKeepAlive" in this._socket) {
150
+ this._socket.setKeepAlive(enable, initialDelay);
151
+ }
152
+ }
153
+ cork() {}
154
+ uncork() {}
155
+ addTrailers(_headers) {}
156
+ setTimeout(timeout, callback) {
157
+ this._timeout = timeout;
158
+ if (callback) {
159
+ this.once("timeout", callback);
160
+ }
161
+ return this;
162
+ }
163
+ abort() {
164
+ this._aborted = true;
165
+ this.destroyed = true;
166
+ if (this._socket) {
167
+ this._socket.destroy();
168
+ }
169
+ this.emit("abort");
170
+ }
171
+ destroy(error) {
172
+ this._aborted = true;
173
+ this.destroyed = true;
174
+ if (this._socket) {
175
+ this._socket.destroy(error);
176
+ }
177
+ if (error) {
178
+ this.emit("error", error);
179
+ }
180
+ return this;
181
+ }
182
+ write(chunk, encoding, callback) {
183
+ if (this._finished || this._aborted)
184
+ return false;
185
+ const buf = typeof chunk === "string" ? Buffer.from(chunk, typeof encoding === "string" ? encoding : "utf8") : chunk;
186
+ this._body.push(buf);
187
+ if (typeof encoding === "function") {
188
+ encoding();
189
+ } else if (callback) {
190
+ callback();
191
+ }
192
+ return true;
193
+ }
194
+ end(data, encoding, callback) {
195
+ if (this._finished)
196
+ return this;
197
+ this._finished = true;
198
+ this.writableEnded = true;
199
+ this._headersSent = true;
200
+ if (typeof data === "function") {
201
+ callback = data;
202
+ data = undefined;
203
+ } else if (typeof encoding === "function") {
204
+ callback = encoding;
205
+ encoding = undefined;
206
+ }
207
+ if (data) {
208
+ this.write(data, encoding);
209
+ }
210
+ this._execute().then(() => {
211
+ this.writableFinished = true;
212
+ if (callback)
213
+ callback();
214
+ }).catch((err) => {
215
+ this.emit("error", err);
216
+ });
217
+ return this;
218
+ }
219
+ async _execute() {
220
+ if (this._aborted)
221
+ return;
222
+ const rawPort = this._options.port ? typeof this._options.port === "string" ? parseInt(this._options.port, 10) : this._options.port : null;
223
+ const isSecure = this.protocol === "https:" || rawPort === 443;
224
+ const port = rawPort ?? (isSecure ? 443 : 80);
225
+ if (this._timeout > 0) {
226
+ this._timeoutTimer = setTimeout(() => {
227
+ this.emit("timeout");
228
+ if (this._socket) {
229
+ this._socket.destroy(new Error("Socket timeout"));
230
+ }
231
+ }, this._timeout);
232
+ }
233
+ try {
234
+ const socksOpts = {
235
+ proxy: this._proxy,
236
+ destination: { host: this.host, port },
237
+ command: "connect"
238
+ };
239
+ const { socket } = await SocksClient.createConnection(socksOpts);
240
+ if (this._aborted) {
241
+ socket.destroy();
242
+ return;
243
+ }
244
+ if (isSecure) {
245
+ if (socket.destroyed) {
246
+ throw new Error("Socket was destroyed before TLS upgrade");
247
+ }
248
+ const rejectUnauthorized = this._options.rejectUnauthorized;
249
+ const tlsSocket = tls.connect({
250
+ socket,
251
+ servername: !net.isIP(this.host) ? this.host : undefined,
252
+ rejectUnauthorized: rejectUnauthorized !== false
253
+ });
254
+ await new Promise((resolve, reject) => {
255
+ const onError = (err) => {
256
+ tlsSocket.removeListener("secureConnect", onSecure);
257
+ socket.removeListener("error", onSocketError);
258
+ socket.removeListener("close", onSocketClose);
259
+ reject(err);
260
+ };
261
+ const onSecure = () => {
262
+ tlsSocket.removeListener("error", onError);
263
+ socket.removeListener("error", onSocketError);
264
+ socket.removeListener("close", onSocketClose);
265
+ resolve();
266
+ };
267
+ const onSocketError = (err) => {
268
+ tlsSocket.removeListener("secureConnect", onSecure);
269
+ tlsSocket.removeListener("error", onError);
270
+ socket.removeListener("close", onSocketClose);
271
+ reject(err);
272
+ };
273
+ const onSocketClose = () => {
274
+ tlsSocket.removeListener("secureConnect", onSecure);
275
+ tlsSocket.removeListener("error", onError);
276
+ socket.removeListener("error", onSocketError);
277
+ reject(new Error("Underlying socket closed during TLS handshake"));
278
+ };
279
+ tlsSocket.once("secureConnect", onSecure);
280
+ tlsSocket.once("error", onError);
281
+ socket.once("error", onSocketError);
282
+ socket.once("close", onSocketClose);
283
+ });
284
+ this._socket = tlsSocket;
285
+ } else {
286
+ this._socket = socket;
287
+ }
288
+ this.emit("socket", this._socket);
289
+ const body = this._body.length > 0 ? Buffer.concat(this._body) : null;
290
+ await this._sendRequest(this._socket, body);
291
+ } catch (err) {
292
+ this._clearTimeout();
293
+ throw err;
294
+ }
295
+ }
296
+ _clearTimeout() {
297
+ if (this._timeoutTimer) {
298
+ clearTimeout(this._timeoutTimer);
299
+ this._timeoutTimer = null;
300
+ }
301
+ }
302
+ async _sendRequest(socket, body) {
303
+ return new Promise((resolve, reject) => {
304
+ const lines = [];
305
+ lines.push(`${this.method} ${this.path} HTTP/1.1`);
306
+ const port = this._options.port ? typeof this._options.port === "string" ? parseInt(this._options.port, 10) : this._options.port : this.protocol === "https:" ? 443 : 80;
307
+ const defaultPort = this.protocol === "https:" ? 443 : 80;
308
+ const hostHeader = port === defaultPort ? this.host : `${this.host}:${port}`;
309
+ lines.push(`Host: ${hostHeader}`);
310
+ for (const [key, value] of Object.entries(this._headers)) {
311
+ if (key.toLowerCase() === "host")
312
+ continue;
313
+ if (value === undefined)
314
+ continue;
315
+ if (Array.isArray(value)) {
316
+ for (const v of value) {
317
+ lines.push(`${key}: ${v}`);
318
+ }
319
+ } else {
320
+ lines.push(`${key}: ${value}`);
321
+ }
322
+ }
323
+ if (body && !this._headers["content-length"]) {
324
+ lines.push(`Content-Length: ${body.length}`);
325
+ }
326
+ const requestData = lines.join(`\r
327
+ `) + `\r
328
+ \r
329
+ `;
330
+ let headerBuffer = Buffer.alloc(0);
331
+ let headersParsed = false;
332
+ let response = null;
333
+ let expectedBodyLength = null;
334
+ let receivedBodyLength = 0;
335
+ let isChunked = false;
336
+ let chunkedBuffer = Buffer.alloc(0);
337
+ const cleanup = () => {
338
+ socket.removeListener("data", onData);
339
+ socket.removeListener("error", onError);
340
+ socket.removeListener("close", onClose);
341
+ this._clearTimeout();
342
+ };
343
+ const onData = (chunk) => {
344
+ if (!headersParsed) {
345
+ headerBuffer = Buffer.concat([headerBuffer, chunk]);
346
+ const headerEnd = headerBuffer.indexOf(`\r
347
+ \r
348
+ `);
349
+ if (headerEnd === -1)
350
+ return;
351
+ const headerData = headerBuffer.subarray(0, headerEnd).toString("utf8");
352
+ const bodyStart = headerBuffer.subarray(headerEnd + 4);
353
+ const firstLine = headerData.split(`\r
354
+ `)[0];
355
+ const statusMatch = firstLine.match(/^HTTP\/([\d.]+)\s+(\d+)\s*(.*)?$/);
356
+ if (!statusMatch) {
357
+ cleanup();
358
+ reject(new Error(`Invalid HTTP response: ${firstLine}`));
359
+ return;
360
+ }
361
+ const httpVersion = statusMatch[1];
362
+ const statusCode = parseInt(statusMatch[2], 10);
363
+ const statusMessage = statusMatch[3] || "";
364
+ const headers = {};
365
+ const rawHeaders = [];
366
+ const headerLines = headerData.split(`\r
367
+ `).slice(1);
368
+ for (const line of headerLines) {
369
+ const colonIdx = line.indexOf(":");
370
+ if (colonIdx > 0) {
371
+ const name = line.substring(0, colonIdx).trim();
372
+ const value = line.substring(colonIdx + 1).trim();
373
+ const lowerName = name.toLowerCase();
374
+ rawHeaders.push(name, value);
375
+ if (headers[lowerName]) {
376
+ const existing = headers[lowerName];
377
+ if (Array.isArray(existing)) {
378
+ existing.push(value);
379
+ } else {
380
+ headers[lowerName] = [existing, value];
381
+ }
382
+ } else {
383
+ headers[lowerName] = value;
384
+ }
385
+ }
386
+ }
387
+ headersParsed = true;
388
+ headerBuffer = Buffer.alloc(0);
389
+ response = new BunSocksIncomingMessage(socket);
390
+ response.httpVersion = httpVersion;
391
+ response.httpVersionMajor = parseInt(httpVersion.split(".")[0], 10);
392
+ response.httpVersionMinor = parseInt(httpVersion.split(".")[1] || "1", 10);
393
+ response._setStatus(statusCode, statusMessage);
394
+ response._setHeaders(headers, rawHeaders);
395
+ this.emit("response", response);
396
+ const contentLength = headers["content-length"];
397
+ const transferEncoding = headers["transfer-encoding"];
398
+ if (transferEncoding?.toLowerCase().includes("chunked")) {
399
+ isChunked = true;
400
+ } else if (contentLength) {
401
+ expectedBodyLength = parseInt(contentLength, 10);
402
+ }
403
+ if (statusCode === 204 || statusCode === 304 || this.method === "HEAD" || expectedBodyLength === 0) {
404
+ cleanup();
405
+ response._finish();
406
+ resolve();
407
+ return;
408
+ }
409
+ if (bodyStart.length > 0) {
410
+ receivedBodyLength = bodyStart.length;
411
+ if (isChunked) {
412
+ chunkedBuffer = bodyStart;
413
+ const result = this._parseChunkedBody(chunkedBuffer);
414
+ for (const decodedChunk of result.newChunks) {
415
+ response._pushChunk(decodedChunk);
416
+ }
417
+ if (result.complete) {
418
+ cleanup();
419
+ response._finish();
420
+ resolve();
421
+ return;
422
+ }
423
+ chunkedBuffer = result.remaining;
424
+ } else if (expectedBodyLength !== null) {
425
+ response._pushChunk(bodyStart);
426
+ if (receivedBodyLength >= expectedBodyLength) {
427
+ cleanup();
428
+ response._finish();
429
+ resolve();
430
+ return;
431
+ }
432
+ } else {
433
+ response._pushChunk(bodyStart);
434
+ }
435
+ }
436
+ return;
437
+ }
438
+ if (!response)
439
+ return;
440
+ receivedBodyLength += chunk.length;
441
+ if (isChunked) {
442
+ chunkedBuffer = Buffer.concat([chunkedBuffer, chunk]);
443
+ const result = this._parseChunkedBody(chunkedBuffer);
444
+ for (const decodedChunk of result.newChunks) {
445
+ response._pushChunk(decodedChunk);
446
+ }
447
+ if (result.complete) {
448
+ cleanup();
449
+ response._finish();
450
+ resolve();
451
+ } else {
452
+ chunkedBuffer = result.remaining;
453
+ }
454
+ } else if (expectedBodyLength !== null) {
455
+ response._pushChunk(chunk);
456
+ if (receivedBodyLength >= expectedBodyLength) {
457
+ cleanup();
458
+ response._finish();
459
+ resolve();
460
+ }
461
+ } else {
462
+ response._pushChunk(chunk);
463
+ }
464
+ };
465
+ const onError = (err) => {
466
+ cleanup();
467
+ reject(err);
468
+ };
469
+ const onClose = () => {
470
+ cleanup();
471
+ if (response && !response.complete) {
472
+ response._finish();
473
+ }
474
+ resolve();
475
+ };
476
+ socket.on("data", onData);
477
+ socket.once("error", onError);
478
+ socket.once("close", onClose);
479
+ socket.write(requestData);
480
+ if (body) {
481
+ socket.write(body);
482
+ }
483
+ });
484
+ }
485
+ _parseChunkedBody(data) {
486
+ const newChunks = [];
487
+ let offset = 0;
488
+ while (offset < data.length) {
489
+ const lineEnd = data.indexOf(`\r
490
+ `, offset);
491
+ if (lineEnd === -1) {
492
+ return { complete: false, newChunks, remaining: data.subarray(offset) };
493
+ }
494
+ const sizeLine = data.subarray(offset, lineEnd).toString("utf8");
495
+ const chunkSize = parseInt(sizeLine.split(";")[0], 16);
496
+ if (isNaN(chunkSize)) {
497
+ return { complete: false, newChunks, remaining: data.subarray(offset) };
498
+ }
499
+ if (chunkSize === 0) {
500
+ return { complete: true, newChunks, remaining: Buffer.alloc(0) };
501
+ }
502
+ const chunkStart = lineEnd + 2;
503
+ const chunkEnd = chunkStart + chunkSize;
504
+ if (data.length < chunkEnd + 2) {
505
+ return { complete: false, newChunks, remaining: data.subarray(offset) };
506
+ }
507
+ newChunks.push(data.subarray(chunkStart, chunkEnd));
508
+ offset = chunkEnd + 2;
509
+ }
510
+ return { complete: false, newChunks, remaining: Buffer.alloc(0) };
511
+ }
512
+ }
513
+ function extractProxyFromAgent(agent) {
514
+ if (!agent || typeof agent !== "object")
515
+ return null;
516
+ const proxyAgent = agent;
517
+ if (proxyAgent.proxy && typeof proxyAgent.proxy === "object") {
518
+ return proxyAgent.proxy;
519
+ }
520
+ return null;
521
+ }
522
+ function parseRequestOptions(urlOrOptions, optionsOrCallback, callback) {
523
+ let options;
524
+ let cb = callback;
525
+ if (typeof urlOrOptions === "string" || urlOrOptions instanceof URL) {
526
+ const url = typeof urlOrOptions === "string" ? new URL(urlOrOptions) : urlOrOptions;
527
+ const baseOptions = {
528
+ protocol: url.protocol,
529
+ hostname: url.hostname,
530
+ port: url.port || (url.protocol === "https:" ? 443 : 80),
531
+ path: url.pathname + url.search,
532
+ method: "GET"
533
+ };
534
+ if (typeof optionsOrCallback === "function") {
535
+ options = baseOptions;
536
+ cb = optionsOrCallback;
537
+ } else if (optionsOrCallback && typeof optionsOrCallback === "object") {
538
+ options = { ...baseOptions, ...optionsOrCallback };
539
+ } else {
540
+ options = baseOptions;
541
+ }
542
+ } else {
543
+ options = urlOrOptions;
544
+ if (typeof optionsOrCallback === "function") {
545
+ cb = optionsOrCallback;
546
+ }
547
+ }
548
+ return { options, callback: cb };
549
+ }
550
+ const bunHttp = exports.bunHttp = {
551
+ request(urlOrOptions, optionsOrCallback, callback) {
552
+ const { options, callback: cb } = parseRequestOptions(urlOrOptions, optionsOrCallback, callback);
553
+ const proxyOpts = extractProxyFromAgent(options.agent);
554
+ if (!proxyOpts) {
555
+ throw new Error("bunHttp.request requires an agent with SOCKS proxy configuration (e.g., SocksProxyAgent)");
556
+ }
557
+ const req = new BunSocksClientRequest(proxyOpts, options);
558
+ if (cb) {
559
+ req.once("response", cb);
560
+ }
561
+ return req;
562
+ },
563
+ get(urlOrOptions, optionsOrCallback, callback) {
564
+ const { options, callback: cb } = parseRequestOptions(urlOrOptions, optionsOrCallback, callback);
565
+ options.method = "GET";
566
+ const req = bunHttp.request(options, cb);
567
+ req.end();
568
+ return req;
569
+ }
570
+ };
571
+
572
+ exports.isBunRuntime = isBunRuntime;
573
+ exports.isBunSocksRequest = isBunSocksRequest;