rezo 1.0.41 → 1.0.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/dist/adapters/curl.cjs +143 -32
  2. package/dist/adapters/curl.js +143 -32
  3. package/dist/adapters/entries/curl.d.ts +65 -0
  4. package/dist/adapters/entries/fetch.d.ts +65 -0
  5. package/dist/adapters/entries/http.d.ts +65 -0
  6. package/dist/adapters/entries/http2.d.ts +65 -0
  7. package/dist/adapters/entries/react-native.d.ts +65 -0
  8. package/dist/adapters/entries/xhr.d.ts +65 -0
  9. package/dist/adapters/fetch.cjs +98 -12
  10. package/dist/adapters/fetch.js +98 -12
  11. package/dist/adapters/http.cjs +26 -14
  12. package/dist/adapters/http.js +26 -14
  13. package/dist/adapters/http2.cjs +756 -227
  14. package/dist/adapters/http2.js +756 -227
  15. package/dist/adapters/index.cjs +6 -6
  16. package/dist/adapters/xhr.cjs +94 -2
  17. package/dist/adapters/xhr.js +94 -2
  18. package/dist/cache/dns-cache.cjs +5 -3
  19. package/dist/cache/dns-cache.js +5 -3
  20. package/dist/cache/file-cacher.cjs +7 -1
  21. package/dist/cache/file-cacher.js +7 -1
  22. package/dist/cache/index.cjs +15 -13
  23. package/dist/cache/index.js +1 -0
  24. package/dist/cache/navigation-history.cjs +298 -0
  25. package/dist/cache/navigation-history.js +296 -0
  26. package/dist/cache/url-store.cjs +7 -1
  27. package/dist/cache/url-store.js +7 -1
  28. package/dist/core/rezo.cjs +7 -0
  29. package/dist/core/rezo.js +7 -0
  30. package/dist/crawler.d.ts +196 -11
  31. package/dist/entries/crawler.cjs +5 -5
  32. package/dist/index.cjs +27 -24
  33. package/dist/index.d.ts +73 -0
  34. package/dist/index.js +1 -0
  35. package/dist/internal/agents/base.cjs +113 -0
  36. package/dist/internal/agents/base.js +110 -0
  37. package/dist/internal/agents/http-proxy.cjs +89 -0
  38. package/dist/internal/agents/http-proxy.js +86 -0
  39. package/dist/internal/agents/https-proxy.cjs +176 -0
  40. package/dist/internal/agents/https-proxy.js +173 -0
  41. package/dist/internal/agents/index.cjs +10 -0
  42. package/dist/internal/agents/index.js +5 -0
  43. package/dist/internal/agents/socks-client.cjs +571 -0
  44. package/dist/internal/agents/socks-client.js +567 -0
  45. package/dist/internal/agents/socks-proxy.cjs +75 -0
  46. package/dist/internal/agents/socks-proxy.js +72 -0
  47. package/dist/platform/browser.d.ts +65 -0
  48. package/dist/platform/bun.d.ts +65 -0
  49. package/dist/platform/deno.d.ts +65 -0
  50. package/dist/platform/node.d.ts +65 -0
  51. package/dist/platform/react-native.d.ts +65 -0
  52. package/dist/platform/worker.d.ts +65 -0
  53. package/dist/plugin/crawler-options.cjs +1 -1
  54. package/dist/plugin/crawler-options.js +1 -1
  55. package/dist/plugin/crawler.cjs +192 -1
  56. package/dist/plugin/crawler.js +192 -1
  57. package/dist/plugin/index.cjs +36 -36
  58. package/dist/proxy/index.cjs +18 -16
  59. package/dist/proxy/index.js +17 -12
  60. package/dist/queue/index.cjs +8 -8
  61. package/dist/responses/buildError.cjs +11 -2
  62. package/dist/responses/buildError.js +11 -2
  63. package/dist/responses/universal/index.cjs +11 -11
  64. package/dist/utils/agent-pool.cjs +1 -17
  65. package/dist/utils/agent-pool.js +1 -17
  66. package/dist/utils/curl.cjs +317 -0
  67. package/dist/utils/curl.js +314 -0
  68. package/package.json +1 -1
@@ -0,0 +1,176 @@
1
+ const net = require("node:net");
2
+ const tls = require("node:tls");
3
+ const { Agent } = require('./base.cjs');
4
+ function omit(obj, ...keys) {
5
+ const ret = {};
6
+ for (const key in obj) {
7
+ if (!keys.includes(key)) {
8
+ ret[key] = obj[key];
9
+ }
10
+ }
11
+ return ret;
12
+ }
13
+ function setServernameFromNonIpHost(options) {
14
+ if (options.servername === undefined && options.host && !net.isIP(options.host)) {
15
+ return { ...options, servername: options.host };
16
+ }
17
+ return options;
18
+ }
19
+ function parseProxyResponse(socket) {
20
+ return new Promise((resolve, reject) => {
21
+ let buffersLength = 0;
22
+ const buffers = [];
23
+ function read() {
24
+ const b = socket.read();
25
+ if (b)
26
+ ondata(b);
27
+ else
28
+ socket.once("readable", read);
29
+ }
30
+ function cleanup() {
31
+ socket.removeListener("end", onend);
32
+ socket.removeListener("error", onerror);
33
+ socket.removeListener("readable", read);
34
+ }
35
+ function onend() {
36
+ cleanup();
37
+ reject(new Error("Proxy connection ended before receiving CONNECT response"));
38
+ }
39
+ function onerror(err) {
40
+ cleanup();
41
+ reject(err);
42
+ }
43
+ function ondata(b) {
44
+ buffers.push(b);
45
+ buffersLength += b.length;
46
+ const buffered = Buffer.concat(buffers, buffersLength);
47
+ const endOfHeaders = buffered.indexOf(`\r
48
+ \r
49
+ `);
50
+ if (endOfHeaders === -1) {
51
+ read();
52
+ return;
53
+ }
54
+ const headerParts = buffered.slice(0, endOfHeaders).toString("ascii").split(`\r
55
+ `);
56
+ const firstLine = headerParts.shift();
57
+ if (!firstLine) {
58
+ socket.destroy();
59
+ return reject(new Error("No header received from proxy CONNECT response"));
60
+ }
61
+ const firstLineParts = firstLine.split(" ");
62
+ const statusCode = +firstLineParts[1];
63
+ const statusText = firstLineParts.slice(2).join(" ");
64
+ const headers = {};
65
+ for (const header of headerParts) {
66
+ if (!header)
67
+ continue;
68
+ const firstColon = header.indexOf(":");
69
+ if (firstColon === -1) {
70
+ socket.destroy();
71
+ return reject(new Error(`Invalid header from proxy CONNECT response: "${header}"`));
72
+ }
73
+ const key = header.slice(0, firstColon).toLowerCase();
74
+ const value = header.slice(firstColon + 1).trimStart();
75
+ const current = headers[key];
76
+ if (typeof current === "string") {
77
+ headers[key] = [current, value];
78
+ } else if (Array.isArray(current)) {
79
+ current.push(value);
80
+ } else {
81
+ headers[key] = value;
82
+ }
83
+ }
84
+ cleanup();
85
+ resolve({
86
+ connect: { statusCode, statusText, headers },
87
+ buffered
88
+ });
89
+ }
90
+ socket.on("error", onerror);
91
+ socket.on("end", onend);
92
+ read();
93
+ });
94
+ }
95
+ function resume(socket) {
96
+ socket.resume();
97
+ }
98
+
99
+ class HttpsProxyAgent extends Agent {
100
+ static protocols = ["http", "https"];
101
+ proxy;
102
+ proxyHeaders;
103
+ connectOpts;
104
+ constructor(proxy, opts) {
105
+ super(opts);
106
+ this.proxy = typeof proxy === "string" ? new URL(proxy) : proxy;
107
+ this.proxyHeaders = opts?.headers ?? {};
108
+ const host = (this.proxy.hostname || this.proxy.host).replace(/^\[|\]$/g, "");
109
+ const port = this.proxy.port ? parseInt(this.proxy.port, 10) : this.proxy.protocol === "https:" ? 443 : 80;
110
+ this.connectOpts = {
111
+ ALPNProtocols: ["http/1.1"],
112
+ ...opts ? omit(opts, "headers") : {},
113
+ host,
114
+ port
115
+ };
116
+ }
117
+ async connect(req, opts) {
118
+ const { proxy } = this;
119
+ if (!opts.host) {
120
+ throw new TypeError('No "host" provided');
121
+ }
122
+ let socket;
123
+ if (proxy.protocol === "https:") {
124
+ socket = tls.connect(setServernameFromNonIpHost(this.connectOpts));
125
+ } else {
126
+ socket = net.connect(this.connectOpts);
127
+ }
128
+ const headers = typeof this.proxyHeaders === "function" ? this.proxyHeaders() : { ...this.proxyHeaders };
129
+ const host = net.isIPv6(opts.host) ? `[${opts.host}]` : opts.host;
130
+ let payload = `CONNECT ${host}:${opts.port} HTTP/1.1\r
131
+ `;
132
+ if (proxy.username || proxy.password) {
133
+ const auth = `${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`;
134
+ headers["Proxy-Authorization"] = `Basic ${Buffer.from(auth).toString("base64")}`;
135
+ }
136
+ headers["Host"] = `${host}:${opts.port}`;
137
+ if (!headers["Proxy-Connection"]) {
138
+ headers["Proxy-Connection"] = this.keepAlive ? "Keep-Alive" : "close";
139
+ }
140
+ for (const name of Object.keys(headers)) {
141
+ payload += `${name}: ${headers[name]}\r
142
+ `;
143
+ }
144
+ const proxyResponsePromise = parseProxyResponse(socket);
145
+ socket.write(`${payload}\r
146
+ `);
147
+ const { connect, buffered } = await proxyResponsePromise;
148
+ req.emit("proxyConnect", connect);
149
+ this.emit("proxyConnect", connect, req);
150
+ if (connect.statusCode === 200) {
151
+ req.once("socket", resume);
152
+ if (opts.secureEndpoint) {
153
+ const tlsOpts = {
154
+ ...omit(setServernameFromNonIpHost(opts), "host", "path", "port"),
155
+ socket
156
+ };
157
+ return tls.connect(tlsOpts);
158
+ }
159
+ return socket;
160
+ }
161
+ socket.destroy();
162
+ const fakeSocket = new net.Socket({ writable: false });
163
+ fakeSocket.readable = true;
164
+ req.once("socket", (s) => {
165
+ if (s.listenerCount("data") > 0) {
166
+ s.push(buffered);
167
+ s.push(null);
168
+ }
169
+ });
170
+ return fakeSocket;
171
+ }
172
+ }
173
+
174
+ exports.HttpsProxyAgent = HttpsProxyAgent;
175
+ exports.default = HttpsProxyAgent;
176
+ module.exports = Object.assign(HttpsProxyAgent, exports);
@@ -0,0 +1,173 @@
1
+ import * as net from "node:net";
2
+ import * as tls from "node:tls";
3
+ import { Agent } from './base.js';
4
+ function omit(obj, ...keys) {
5
+ const ret = {};
6
+ for (const key in obj) {
7
+ if (!keys.includes(key)) {
8
+ ret[key] = obj[key];
9
+ }
10
+ }
11
+ return ret;
12
+ }
13
+ function setServernameFromNonIpHost(options) {
14
+ if (options.servername === undefined && options.host && !net.isIP(options.host)) {
15
+ return { ...options, servername: options.host };
16
+ }
17
+ return options;
18
+ }
19
+ function parseProxyResponse(socket) {
20
+ return new Promise((resolve, reject) => {
21
+ let buffersLength = 0;
22
+ const buffers = [];
23
+ function read() {
24
+ const b = socket.read();
25
+ if (b)
26
+ ondata(b);
27
+ else
28
+ socket.once("readable", read);
29
+ }
30
+ function cleanup() {
31
+ socket.removeListener("end", onend);
32
+ socket.removeListener("error", onerror);
33
+ socket.removeListener("readable", read);
34
+ }
35
+ function onend() {
36
+ cleanup();
37
+ reject(new Error("Proxy connection ended before receiving CONNECT response"));
38
+ }
39
+ function onerror(err) {
40
+ cleanup();
41
+ reject(err);
42
+ }
43
+ function ondata(b) {
44
+ buffers.push(b);
45
+ buffersLength += b.length;
46
+ const buffered = Buffer.concat(buffers, buffersLength);
47
+ const endOfHeaders = buffered.indexOf(`\r
48
+ \r
49
+ `);
50
+ if (endOfHeaders === -1) {
51
+ read();
52
+ return;
53
+ }
54
+ const headerParts = buffered.slice(0, endOfHeaders).toString("ascii").split(`\r
55
+ `);
56
+ const firstLine = headerParts.shift();
57
+ if (!firstLine) {
58
+ socket.destroy();
59
+ return reject(new Error("No header received from proxy CONNECT response"));
60
+ }
61
+ const firstLineParts = firstLine.split(" ");
62
+ const statusCode = +firstLineParts[1];
63
+ const statusText = firstLineParts.slice(2).join(" ");
64
+ const headers = {};
65
+ for (const header of headerParts) {
66
+ if (!header)
67
+ continue;
68
+ const firstColon = header.indexOf(":");
69
+ if (firstColon === -1) {
70
+ socket.destroy();
71
+ return reject(new Error(`Invalid header from proxy CONNECT response: "${header}"`));
72
+ }
73
+ const key = header.slice(0, firstColon).toLowerCase();
74
+ const value = header.slice(firstColon + 1).trimStart();
75
+ const current = headers[key];
76
+ if (typeof current === "string") {
77
+ headers[key] = [current, value];
78
+ } else if (Array.isArray(current)) {
79
+ current.push(value);
80
+ } else {
81
+ headers[key] = value;
82
+ }
83
+ }
84
+ cleanup();
85
+ resolve({
86
+ connect: { statusCode, statusText, headers },
87
+ buffered
88
+ });
89
+ }
90
+ socket.on("error", onerror);
91
+ socket.on("end", onend);
92
+ read();
93
+ });
94
+ }
95
+ function resume(socket) {
96
+ socket.resume();
97
+ }
98
+
99
+ export class HttpsProxyAgent extends Agent {
100
+ static protocols = ["http", "https"];
101
+ proxy;
102
+ proxyHeaders;
103
+ connectOpts;
104
+ constructor(proxy, opts) {
105
+ super(opts);
106
+ this.proxy = typeof proxy === "string" ? new URL(proxy) : proxy;
107
+ this.proxyHeaders = opts?.headers ?? {};
108
+ const host = (this.proxy.hostname || this.proxy.host).replace(/^\[|\]$/g, "");
109
+ const port = this.proxy.port ? parseInt(this.proxy.port, 10) : this.proxy.protocol === "https:" ? 443 : 80;
110
+ this.connectOpts = {
111
+ ALPNProtocols: ["http/1.1"],
112
+ ...opts ? omit(opts, "headers") : {},
113
+ host,
114
+ port
115
+ };
116
+ }
117
+ async connect(req, opts) {
118
+ const { proxy } = this;
119
+ if (!opts.host) {
120
+ throw new TypeError('No "host" provided');
121
+ }
122
+ let socket;
123
+ if (proxy.protocol === "https:") {
124
+ socket = tls.connect(setServernameFromNonIpHost(this.connectOpts));
125
+ } else {
126
+ socket = net.connect(this.connectOpts);
127
+ }
128
+ const headers = typeof this.proxyHeaders === "function" ? this.proxyHeaders() : { ...this.proxyHeaders };
129
+ const host = net.isIPv6(opts.host) ? `[${opts.host}]` : opts.host;
130
+ let payload = `CONNECT ${host}:${opts.port} HTTP/1.1\r
131
+ `;
132
+ if (proxy.username || proxy.password) {
133
+ const auth = `${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`;
134
+ headers["Proxy-Authorization"] = `Basic ${Buffer.from(auth).toString("base64")}`;
135
+ }
136
+ headers["Host"] = `${host}:${opts.port}`;
137
+ if (!headers["Proxy-Connection"]) {
138
+ headers["Proxy-Connection"] = this.keepAlive ? "Keep-Alive" : "close";
139
+ }
140
+ for (const name of Object.keys(headers)) {
141
+ payload += `${name}: ${headers[name]}\r
142
+ `;
143
+ }
144
+ const proxyResponsePromise = parseProxyResponse(socket);
145
+ socket.write(`${payload}\r
146
+ `);
147
+ const { connect, buffered } = await proxyResponsePromise;
148
+ req.emit("proxyConnect", connect);
149
+ this.emit("proxyConnect", connect, req);
150
+ if (connect.statusCode === 200) {
151
+ req.once("socket", resume);
152
+ if (opts.secureEndpoint) {
153
+ const tlsOpts = {
154
+ ...omit(setServernameFromNonIpHost(opts), "host", "path", "port"),
155
+ socket
156
+ };
157
+ return tls.connect(tlsOpts);
158
+ }
159
+ return socket;
160
+ }
161
+ socket.destroy();
162
+ const fakeSocket = new net.Socket({ writable: false });
163
+ fakeSocket.readable = true;
164
+ req.once("socket", (s) => {
165
+ if (s.listenerCount("data") > 0) {
166
+ s.push(buffered);
167
+ s.push(null);
168
+ }
169
+ });
170
+ return fakeSocket;
171
+ }
172
+ }
173
+ export default HttpsProxyAgent;
@@ -0,0 +1,10 @@
1
+ const _mod_7qp1aa = require('./base.cjs');
2
+ exports.Agent = _mod_7qp1aa.Agent;;
3
+ const _mod_p65m0p = require('./http-proxy.cjs');
4
+ exports.HttpProxyAgent = _mod_p65m0p.HttpProxyAgent;;
5
+ const _mod_vvhz9m = require('./https-proxy.cjs');
6
+ exports.HttpsProxyAgent = _mod_vvhz9m.HttpsProxyAgent;;
7
+ const _mod_jsl1v1 = require('./socks-proxy.cjs');
8
+ exports.SocksProxyAgent = _mod_jsl1v1.SocksProxyAgent;;
9
+ const _mod_xeq0c5 = require('./socks-client.cjs');
10
+ exports.SocksClient = _mod_xeq0c5.SocksClient;;
@@ -0,0 +1,5 @@
1
+ export { Agent } from './base.js';
2
+ export { HttpProxyAgent } from './http-proxy.js';
3
+ export { HttpsProxyAgent } from './https-proxy.js';
4
+ export { SocksProxyAgent } from './socks-proxy.js';
5
+ export { SocksClient } from './socks-client.js';