rezo 1.0.42 → 1.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/dist/adapters/curl.cjs +131 -29
  2. package/dist/adapters/curl.js +131 -29
  3. package/dist/adapters/entries/curl.d.ts +65 -0
  4. package/dist/adapters/entries/fetch.d.ts +65 -0
  5. package/dist/adapters/entries/http.d.ts +65 -0
  6. package/dist/adapters/entries/http2.d.ts +65 -0
  7. package/dist/adapters/entries/react-native.d.ts +65 -0
  8. package/dist/adapters/entries/xhr.d.ts +65 -0
  9. package/dist/adapters/http2.cjs +209 -22
  10. package/dist/adapters/http2.js +209 -22
  11. package/dist/adapters/index.cjs +6 -6
  12. package/dist/cache/index.cjs +9 -13
  13. package/dist/cache/index.js +0 -2
  14. package/dist/core/rezo.cjs +7 -0
  15. package/dist/core/rezo.js +7 -0
  16. package/dist/crawler/addon/decodo/index.cjs +1 -0
  17. package/dist/crawler/addon/decodo/index.js +1 -0
  18. package/dist/crawler/crawler-options.cjs +1 -0
  19. package/dist/crawler/crawler-options.js +1 -0
  20. package/dist/crawler/crawler.cjs +1070 -0
  21. package/dist/crawler/crawler.js +1068 -0
  22. package/dist/crawler/index.cjs +40 -0
  23. package/dist/{plugin → crawler}/index.js +4 -2
  24. package/dist/crawler/plugin/file-cacher.cjs +19 -0
  25. package/dist/crawler/plugin/file-cacher.js +19 -0
  26. package/dist/crawler/plugin/index.cjs +1 -0
  27. package/dist/crawler/plugin/index.js +1 -0
  28. package/dist/crawler/plugin/navigation-history.cjs +43 -0
  29. package/dist/crawler/plugin/navigation-history.js +43 -0
  30. package/dist/crawler/plugin/robots-txt.cjs +2 -0
  31. package/dist/crawler/plugin/robots-txt.js +2 -0
  32. package/dist/crawler/plugin/url-store.cjs +18 -0
  33. package/dist/crawler/plugin/url-store.js +18 -0
  34. package/dist/crawler.d.ts +511 -183
  35. package/dist/entries/crawler.cjs +5 -5
  36. package/dist/entries/crawler.js +2 -2
  37. package/dist/index.cjs +27 -24
  38. package/dist/index.d.ts +73 -0
  39. package/dist/index.js +1 -0
  40. package/dist/internal/agents/base.cjs +113 -0
  41. package/dist/internal/agents/base.js +110 -0
  42. package/dist/internal/agents/http-proxy.cjs +89 -0
  43. package/dist/internal/agents/http-proxy.js +86 -0
  44. package/dist/internal/agents/https-proxy.cjs +176 -0
  45. package/dist/internal/agents/https-proxy.js +173 -0
  46. package/dist/internal/agents/index.cjs +10 -0
  47. package/dist/internal/agents/index.js +5 -0
  48. package/dist/internal/agents/socks-client.cjs +571 -0
  49. package/dist/internal/agents/socks-client.js +567 -0
  50. package/dist/internal/agents/socks-proxy.cjs +75 -0
  51. package/dist/internal/agents/socks-proxy.js +72 -0
  52. package/dist/platform/browser.d.ts +65 -0
  53. package/dist/platform/bun.d.ts +65 -0
  54. package/dist/platform/deno.d.ts +65 -0
  55. package/dist/platform/node.d.ts +65 -0
  56. package/dist/platform/react-native.d.ts +65 -0
  57. package/dist/platform/worker.d.ts +65 -0
  58. package/dist/proxy/index.cjs +18 -16
  59. package/dist/proxy/index.js +17 -12
  60. package/dist/queue/index.cjs +8 -8
  61. package/dist/responses/buildError.cjs +11 -2
  62. package/dist/responses/buildError.js +11 -2
  63. package/dist/responses/universal/index.cjs +11 -11
  64. package/dist/utils/curl.cjs +317 -0
  65. package/dist/utils/curl.js +314 -0
  66. package/package.json +2 -6
  67. package/dist/cache/file-cacher.cjs +0 -264
  68. package/dist/cache/file-cacher.js +0 -261
  69. package/dist/cache/url-store.cjs +0 -288
  70. package/dist/cache/url-store.js +0 -285
  71. package/dist/plugin/addon/decodo/index.cjs +0 -1
  72. package/dist/plugin/addon/decodo/index.js +0 -1
  73. package/dist/plugin/crawler-options.cjs +0 -1
  74. package/dist/plugin/crawler-options.js +0 -1
  75. package/dist/plugin/crawler.cjs +0 -519
  76. package/dist/plugin/crawler.js +0 -517
  77. package/dist/plugin/index.cjs +0 -36
  78. /package/dist/{plugin → crawler}/addon/decodo/options.cjs +0 -0
  79. /package/dist/{plugin → crawler}/addon/decodo/options.js +0 -0
  80. /package/dist/{plugin → crawler}/addon/decodo/types.cjs +0 -0
  81. /package/dist/{plugin → crawler}/addon/decodo/types.js +0 -0
  82. /package/dist/{plugin → crawler}/addon/oxylabs/index.cjs +0 -0
  83. /package/dist/{plugin → crawler}/addon/oxylabs/index.js +0 -0
  84. /package/dist/{plugin → crawler}/addon/oxylabs/options.cjs +0 -0
  85. /package/dist/{plugin → crawler}/addon/oxylabs/options.js +0 -0
  86. /package/dist/{plugin → crawler}/addon/oxylabs/types.cjs +0 -0
  87. /package/dist/{plugin → crawler}/addon/oxylabs/types.js +0 -0
  88. /package/dist/{plugin → crawler}/scraper.cjs +0 -0
  89. /package/dist/{plugin → crawler}/scraper.js +0 -0
@@ -0,0 +1,176 @@
1
+ const net = require("node:net");
2
+ const tls = require("node:tls");
3
+ const { Agent } = require('./base.cjs');
4
+ function omit(obj, ...keys) {
5
+ const ret = {};
6
+ for (const key in obj) {
7
+ if (!keys.includes(key)) {
8
+ ret[key] = obj[key];
9
+ }
10
+ }
11
+ return ret;
12
+ }
13
+ function setServernameFromNonIpHost(options) {
14
+ if (options.servername === undefined && options.host && !net.isIP(options.host)) {
15
+ return { ...options, servername: options.host };
16
+ }
17
+ return options;
18
+ }
19
+ function parseProxyResponse(socket) {
20
+ return new Promise((resolve, reject) => {
21
+ let buffersLength = 0;
22
+ const buffers = [];
23
+ function read() {
24
+ const b = socket.read();
25
+ if (b)
26
+ ondata(b);
27
+ else
28
+ socket.once("readable", read);
29
+ }
30
+ function cleanup() {
31
+ socket.removeListener("end", onend);
32
+ socket.removeListener("error", onerror);
33
+ socket.removeListener("readable", read);
34
+ }
35
+ function onend() {
36
+ cleanup();
37
+ reject(new Error("Proxy connection ended before receiving CONNECT response"));
38
+ }
39
+ function onerror(err) {
40
+ cleanup();
41
+ reject(err);
42
+ }
43
+ function ondata(b) {
44
+ buffers.push(b);
45
+ buffersLength += b.length;
46
+ const buffered = Buffer.concat(buffers, buffersLength);
47
+ const endOfHeaders = buffered.indexOf(`\r
48
+ \r
49
+ `);
50
+ if (endOfHeaders === -1) {
51
+ read();
52
+ return;
53
+ }
54
+ const headerParts = buffered.slice(0, endOfHeaders).toString("ascii").split(`\r
55
+ `);
56
+ const firstLine = headerParts.shift();
57
+ if (!firstLine) {
58
+ socket.destroy();
59
+ return reject(new Error("No header received from proxy CONNECT response"));
60
+ }
61
+ const firstLineParts = firstLine.split(" ");
62
+ const statusCode = +firstLineParts[1];
63
+ const statusText = firstLineParts.slice(2).join(" ");
64
+ const headers = {};
65
+ for (const header of headerParts) {
66
+ if (!header)
67
+ continue;
68
+ const firstColon = header.indexOf(":");
69
+ if (firstColon === -1) {
70
+ socket.destroy();
71
+ return reject(new Error(`Invalid header from proxy CONNECT response: "${header}"`));
72
+ }
73
+ const key = header.slice(0, firstColon).toLowerCase();
74
+ const value = header.slice(firstColon + 1).trimStart();
75
+ const current = headers[key];
76
+ if (typeof current === "string") {
77
+ headers[key] = [current, value];
78
+ } else if (Array.isArray(current)) {
79
+ current.push(value);
80
+ } else {
81
+ headers[key] = value;
82
+ }
83
+ }
84
+ cleanup();
85
+ resolve({
86
+ connect: { statusCode, statusText, headers },
87
+ buffered
88
+ });
89
+ }
90
+ socket.on("error", onerror);
91
+ socket.on("end", onend);
92
+ read();
93
+ });
94
+ }
95
+ function resume(socket) {
96
+ socket.resume();
97
+ }
98
+
99
+ class HttpsProxyAgent extends Agent {
100
+ static protocols = ["http", "https"];
101
+ proxy;
102
+ proxyHeaders;
103
+ connectOpts;
104
+ constructor(proxy, opts) {
105
+ super(opts);
106
+ this.proxy = typeof proxy === "string" ? new URL(proxy) : proxy;
107
+ this.proxyHeaders = opts?.headers ?? {};
108
+ const host = (this.proxy.hostname || this.proxy.host).replace(/^\[|\]$/g, "");
109
+ const port = this.proxy.port ? parseInt(this.proxy.port, 10) : this.proxy.protocol === "https:" ? 443 : 80;
110
+ this.connectOpts = {
111
+ ALPNProtocols: ["http/1.1"],
112
+ ...opts ? omit(opts, "headers") : {},
113
+ host,
114
+ port
115
+ };
116
+ }
117
+ async connect(req, opts) {
118
+ const { proxy } = this;
119
+ if (!opts.host) {
120
+ throw new TypeError('No "host" provided');
121
+ }
122
+ let socket;
123
+ if (proxy.protocol === "https:") {
124
+ socket = tls.connect(setServernameFromNonIpHost(this.connectOpts));
125
+ } else {
126
+ socket = net.connect(this.connectOpts);
127
+ }
128
+ const headers = typeof this.proxyHeaders === "function" ? this.proxyHeaders() : { ...this.proxyHeaders };
129
+ const host = net.isIPv6(opts.host) ? `[${opts.host}]` : opts.host;
130
+ let payload = `CONNECT ${host}:${opts.port} HTTP/1.1\r
131
+ `;
132
+ if (proxy.username || proxy.password) {
133
+ const auth = `${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`;
134
+ headers["Proxy-Authorization"] = `Basic ${Buffer.from(auth).toString("base64")}`;
135
+ }
136
+ headers["Host"] = `${host}:${opts.port}`;
137
+ if (!headers["Proxy-Connection"]) {
138
+ headers["Proxy-Connection"] = this.keepAlive ? "Keep-Alive" : "close";
139
+ }
140
+ for (const name of Object.keys(headers)) {
141
+ payload += `${name}: ${headers[name]}\r
142
+ `;
143
+ }
144
+ const proxyResponsePromise = parseProxyResponse(socket);
145
+ socket.write(`${payload}\r
146
+ `);
147
+ const { connect, buffered } = await proxyResponsePromise;
148
+ req.emit("proxyConnect", connect);
149
+ this.emit("proxyConnect", connect, req);
150
+ if (connect.statusCode === 200) {
151
+ req.once("socket", resume);
152
+ if (opts.secureEndpoint) {
153
+ const tlsOpts = {
154
+ ...omit(setServernameFromNonIpHost(opts), "host", "path", "port"),
155
+ socket
156
+ };
157
+ return tls.connect(tlsOpts);
158
+ }
159
+ return socket;
160
+ }
161
+ socket.destroy();
162
+ const fakeSocket = new net.Socket({ writable: false });
163
+ fakeSocket.readable = true;
164
+ req.once("socket", (s) => {
165
+ if (s.listenerCount("data") > 0) {
166
+ s.push(buffered);
167
+ s.push(null);
168
+ }
169
+ });
170
+ return fakeSocket;
171
+ }
172
+ }
173
+
174
+ exports.HttpsProxyAgent = HttpsProxyAgent;
175
+ exports.default = HttpsProxyAgent;
176
+ module.exports = Object.assign(HttpsProxyAgent, exports);
@@ -0,0 +1,173 @@
1
+ import * as net from "node:net";
2
+ import * as tls from "node:tls";
3
+ import { Agent } from './base.js';
4
+ function omit(obj, ...keys) {
5
+ const ret = {};
6
+ for (const key in obj) {
7
+ if (!keys.includes(key)) {
8
+ ret[key] = obj[key];
9
+ }
10
+ }
11
+ return ret;
12
+ }
13
+ function setServernameFromNonIpHost(options) {
14
+ if (options.servername === undefined && options.host && !net.isIP(options.host)) {
15
+ return { ...options, servername: options.host };
16
+ }
17
+ return options;
18
+ }
19
+ function parseProxyResponse(socket) {
20
+ return new Promise((resolve, reject) => {
21
+ let buffersLength = 0;
22
+ const buffers = [];
23
+ function read() {
24
+ const b = socket.read();
25
+ if (b)
26
+ ondata(b);
27
+ else
28
+ socket.once("readable", read);
29
+ }
30
+ function cleanup() {
31
+ socket.removeListener("end", onend);
32
+ socket.removeListener("error", onerror);
33
+ socket.removeListener("readable", read);
34
+ }
35
+ function onend() {
36
+ cleanup();
37
+ reject(new Error("Proxy connection ended before receiving CONNECT response"));
38
+ }
39
+ function onerror(err) {
40
+ cleanup();
41
+ reject(err);
42
+ }
43
+ function ondata(b) {
44
+ buffers.push(b);
45
+ buffersLength += b.length;
46
+ const buffered = Buffer.concat(buffers, buffersLength);
47
+ const endOfHeaders = buffered.indexOf(`\r
48
+ \r
49
+ `);
50
+ if (endOfHeaders === -1) {
51
+ read();
52
+ return;
53
+ }
54
+ const headerParts = buffered.slice(0, endOfHeaders).toString("ascii").split(`\r
55
+ `);
56
+ const firstLine = headerParts.shift();
57
+ if (!firstLine) {
58
+ socket.destroy();
59
+ return reject(new Error("No header received from proxy CONNECT response"));
60
+ }
61
+ const firstLineParts = firstLine.split(" ");
62
+ const statusCode = +firstLineParts[1];
63
+ const statusText = firstLineParts.slice(2).join(" ");
64
+ const headers = {};
65
+ for (const header of headerParts) {
66
+ if (!header)
67
+ continue;
68
+ const firstColon = header.indexOf(":");
69
+ if (firstColon === -1) {
70
+ socket.destroy();
71
+ return reject(new Error(`Invalid header from proxy CONNECT response: "${header}"`));
72
+ }
73
+ const key = header.slice(0, firstColon).toLowerCase();
74
+ const value = header.slice(firstColon + 1).trimStart();
75
+ const current = headers[key];
76
+ if (typeof current === "string") {
77
+ headers[key] = [current, value];
78
+ } else if (Array.isArray(current)) {
79
+ current.push(value);
80
+ } else {
81
+ headers[key] = value;
82
+ }
83
+ }
84
+ cleanup();
85
+ resolve({
86
+ connect: { statusCode, statusText, headers },
87
+ buffered
88
+ });
89
+ }
90
+ socket.on("error", onerror);
91
+ socket.on("end", onend);
92
+ read();
93
+ });
94
+ }
95
+ function resume(socket) {
96
+ socket.resume();
97
+ }
98
+
99
+ export class HttpsProxyAgent extends Agent {
100
+ static protocols = ["http", "https"];
101
+ proxy;
102
+ proxyHeaders;
103
+ connectOpts;
104
+ constructor(proxy, opts) {
105
+ super(opts);
106
+ this.proxy = typeof proxy === "string" ? new URL(proxy) : proxy;
107
+ this.proxyHeaders = opts?.headers ?? {};
108
+ const host = (this.proxy.hostname || this.proxy.host).replace(/^\[|\]$/g, "");
109
+ const port = this.proxy.port ? parseInt(this.proxy.port, 10) : this.proxy.protocol === "https:" ? 443 : 80;
110
+ this.connectOpts = {
111
+ ALPNProtocols: ["http/1.1"],
112
+ ...opts ? omit(opts, "headers") : {},
113
+ host,
114
+ port
115
+ };
116
+ }
117
+ async connect(req, opts) {
118
+ const { proxy } = this;
119
+ if (!opts.host) {
120
+ throw new TypeError('No "host" provided');
121
+ }
122
+ let socket;
123
+ if (proxy.protocol === "https:") {
124
+ socket = tls.connect(setServernameFromNonIpHost(this.connectOpts));
125
+ } else {
126
+ socket = net.connect(this.connectOpts);
127
+ }
128
+ const headers = typeof this.proxyHeaders === "function" ? this.proxyHeaders() : { ...this.proxyHeaders };
129
+ const host = net.isIPv6(opts.host) ? `[${opts.host}]` : opts.host;
130
+ let payload = `CONNECT ${host}:${opts.port} HTTP/1.1\r
131
+ `;
132
+ if (proxy.username || proxy.password) {
133
+ const auth = `${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`;
134
+ headers["Proxy-Authorization"] = `Basic ${Buffer.from(auth).toString("base64")}`;
135
+ }
136
+ headers["Host"] = `${host}:${opts.port}`;
137
+ if (!headers["Proxy-Connection"]) {
138
+ headers["Proxy-Connection"] = this.keepAlive ? "Keep-Alive" : "close";
139
+ }
140
+ for (const name of Object.keys(headers)) {
141
+ payload += `${name}: ${headers[name]}\r
142
+ `;
143
+ }
144
+ const proxyResponsePromise = parseProxyResponse(socket);
145
+ socket.write(`${payload}\r
146
+ `);
147
+ const { connect, buffered } = await proxyResponsePromise;
148
+ req.emit("proxyConnect", connect);
149
+ this.emit("proxyConnect", connect, req);
150
+ if (connect.statusCode === 200) {
151
+ req.once("socket", resume);
152
+ if (opts.secureEndpoint) {
153
+ const tlsOpts = {
154
+ ...omit(setServernameFromNonIpHost(opts), "host", "path", "port"),
155
+ socket
156
+ };
157
+ return tls.connect(tlsOpts);
158
+ }
159
+ return socket;
160
+ }
161
+ socket.destroy();
162
+ const fakeSocket = new net.Socket({ writable: false });
163
+ fakeSocket.readable = true;
164
+ req.once("socket", (s) => {
165
+ if (s.listenerCount("data") > 0) {
166
+ s.push(buffered);
167
+ s.push(null);
168
+ }
169
+ });
170
+ return fakeSocket;
171
+ }
172
+ }
173
+ export default HttpsProxyAgent;
@@ -0,0 +1,10 @@
1
+ const _mod_ka4e3g = require('./base.cjs');
2
+ exports.Agent = _mod_ka4e3g.Agent;;
3
+ const _mod_nmerdf = require('./http-proxy.cjs');
4
+ exports.HttpProxyAgent = _mod_nmerdf.HttpProxyAgent;;
5
+ const _mod_do1bvj = require('./https-proxy.cjs');
6
+ exports.HttpsProxyAgent = _mod_do1bvj.HttpsProxyAgent;;
7
+ const _mod_1bn8q4 = require('./socks-proxy.cjs');
8
+ exports.SocksProxyAgent = _mod_1bn8q4.SocksProxyAgent;;
9
+ const _mod_wb9id7 = require('./socks-client.cjs');
10
+ exports.SocksClient = _mod_wb9id7.SocksClient;;
@@ -0,0 +1,5 @@
1
+ export { Agent } from './base.js';
2
+ export { HttpProxyAgent } from './http-proxy.js';
3
+ export { HttpsProxyAgent } from './https-proxy.js';
4
+ export { SocksProxyAgent } from './socks-proxy.js';
5
+ export { SocksClient } from './socks-client.js';