rezo 1.0.41 → 1.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +143 -32
- package/dist/adapters/curl.js +143 -32
- package/dist/adapters/entries/curl.d.ts +65 -0
- package/dist/adapters/entries/fetch.d.ts +65 -0
- package/dist/adapters/entries/http.d.ts +65 -0
- package/dist/adapters/entries/http2.d.ts +65 -0
- package/dist/adapters/entries/react-native.d.ts +65 -0
- package/dist/adapters/entries/xhr.d.ts +65 -0
- package/dist/adapters/fetch.cjs +98 -12
- package/dist/adapters/fetch.js +98 -12
- package/dist/adapters/http.cjs +26 -14
- package/dist/adapters/http.js +26 -14
- package/dist/adapters/http2.cjs +756 -227
- package/dist/adapters/http2.js +756 -227
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/xhr.cjs +94 -2
- package/dist/adapters/xhr.js +94 -2
- package/dist/cache/dns-cache.cjs +5 -3
- package/dist/cache/dns-cache.js +5 -3
- package/dist/cache/file-cacher.cjs +7 -1
- package/dist/cache/file-cacher.js +7 -1
- package/dist/cache/index.cjs +15 -13
- package/dist/cache/index.js +1 -0
- package/dist/cache/navigation-history.cjs +298 -0
- package/dist/cache/navigation-history.js +296 -0
- package/dist/cache/url-store.cjs +7 -1
- package/dist/cache/url-store.js +7 -1
- package/dist/core/rezo.cjs +7 -0
- package/dist/core/rezo.js +7 -0
- package/dist/crawler.d.ts +196 -11
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +27 -24
- package/dist/index.d.ts +73 -0
- package/dist/index.js +1 -0
- package/dist/internal/agents/base.cjs +113 -0
- package/dist/internal/agents/base.js +110 -0
- package/dist/internal/agents/http-proxy.cjs +89 -0
- package/dist/internal/agents/http-proxy.js +86 -0
- package/dist/internal/agents/https-proxy.cjs +176 -0
- package/dist/internal/agents/https-proxy.js +173 -0
- package/dist/internal/agents/index.cjs +10 -0
- package/dist/internal/agents/index.js +5 -0
- package/dist/internal/agents/socks-client.cjs +571 -0
- package/dist/internal/agents/socks-client.js +567 -0
- package/dist/internal/agents/socks-proxy.cjs +75 -0
- package/dist/internal/agents/socks-proxy.js +72 -0
- package/dist/platform/browser.d.ts +65 -0
- package/dist/platform/bun.d.ts +65 -0
- package/dist/platform/deno.d.ts +65 -0
- package/dist/platform/node.d.ts +65 -0
- package/dist/platform/react-native.d.ts +65 -0
- package/dist/platform/worker.d.ts +65 -0
- package/dist/plugin/crawler-options.cjs +1 -1
- package/dist/plugin/crawler-options.js +1 -1
- package/dist/plugin/crawler.cjs +192 -1
- package/dist/plugin/crawler.js +192 -1
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +18 -16
- package/dist/proxy/index.js +17 -12
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/buildError.cjs +11 -2
- package/dist/responses/buildError.js +11 -2
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/agent-pool.cjs +1 -17
- package/dist/utils/agent-pool.js +1 -17
- package/dist/utils/curl.cjs +317 -0
- package/dist/utils/curl.js +314 -0
- package/package.json +1 -1
package/dist/adapters/curl.cjs
CHANGED
|
@@ -195,7 +195,7 @@ function buildUrlTree(config, finalUrl) {
|
|
|
195
195
|
}
|
|
196
196
|
return urls.length > 0 ? urls : [finalUrl];
|
|
197
197
|
}
|
|
198
|
-
async function updateCookies(config, cookieStrings, url) {
|
|
198
|
+
async function updateCookies(config, cookieStrings, url, rootJar) {
|
|
199
199
|
if (!cookieStrings || cookieStrings.length === 0)
|
|
200
200
|
return;
|
|
201
201
|
const tempJar = new RezoCookieJar;
|
|
@@ -233,8 +233,9 @@ async function updateCookies(config, cookieStrings, url) {
|
|
|
233
233
|
acceptedCookies.push(...parsedCookies.array);
|
|
234
234
|
}
|
|
235
235
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
236
|
-
|
|
237
|
-
|
|
236
|
+
const jarToUpdate = rootJar || config.cookieJar;
|
|
237
|
+
if (!config.disableCookieJar && jarToUpdate) {
|
|
238
|
+
jarToUpdate.setCookiesSync(acceptedCookieStrings, url);
|
|
238
239
|
}
|
|
239
240
|
if (config.useCookies) {
|
|
240
241
|
const existingArray = config.responseCookies?.array || [];
|
|
@@ -1484,14 +1485,15 @@ class CurlCommandBuilder {
|
|
|
1484
1485
|
case "socks5h":
|
|
1485
1486
|
this.addArg("--socks5", hostPort);
|
|
1486
1487
|
if (auth) {
|
|
1487
|
-
this.addArg("--socks5-
|
|
1488
|
+
this.addArg("--socks5-basic");
|
|
1489
|
+
this.addArg("--proxy-user", auth);
|
|
1488
1490
|
}
|
|
1489
1491
|
break;
|
|
1490
1492
|
case "socks4":
|
|
1491
1493
|
case "socks4a":
|
|
1492
1494
|
this.addArg("--socks4", hostPort);
|
|
1493
1495
|
if (username) {
|
|
1494
|
-
this.addArg("--
|
|
1496
|
+
this.addArg("--proxy-user", username);
|
|
1495
1497
|
}
|
|
1496
1498
|
break;
|
|
1497
1499
|
case "http":
|
|
@@ -1634,6 +1636,7 @@ class CurlCommandBuilder {
|
|
|
1634
1636
|
"local_ip:%{local_ip}",
|
|
1635
1637
|
"local_port:%{local_port}",
|
|
1636
1638
|
"redirect_url:%{redirect_url}",
|
|
1639
|
+
"url_effective:%{url_effective}",
|
|
1637
1640
|
"ssl_verify_result:%{ssl_verify_result}",
|
|
1638
1641
|
"content_type:%{content_type}",
|
|
1639
1642
|
"---CURL_STATS_END---"
|
|
@@ -1655,27 +1658,68 @@ class CurlResponseParser {
|
|
|
1655
1658
|
body = stdout.slice(0, statsStart);
|
|
1656
1659
|
for (const line of statsSection.split(`
|
|
1657
1660
|
`)) {
|
|
1658
|
-
const
|
|
1659
|
-
if (
|
|
1660
|
-
|
|
1661
|
+
const colonIndex = line.indexOf(":");
|
|
1662
|
+
if (colonIndex !== -1) {
|
|
1663
|
+
const key = line.slice(0, colonIndex).trim();
|
|
1664
|
+
const value = line.slice(colonIndex + 1).trim();
|
|
1665
|
+
if (key && value !== undefined) {
|
|
1666
|
+
stats[key] = value;
|
|
1667
|
+
}
|
|
1661
1668
|
}
|
|
1662
1669
|
}
|
|
1663
1670
|
}
|
|
1664
|
-
const
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
let
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1671
|
+
const allResponses = this.parseAllHttpResponses(body);
|
|
1672
|
+
const finalResponse = allResponses[allResponses.length - 1] || { headers: "", body };
|
|
1673
|
+
let headerSection = finalResponse.headers;
|
|
1674
|
+
let responseBody = finalResponse.body;
|
|
1675
|
+
const allSetCookies = [];
|
|
1676
|
+
for (const resp of allResponses) {
|
|
1677
|
+
const respHeaders = this.parseHeaders(resp.headers);
|
|
1678
|
+
const setCookieHeader = respHeaders["set-cookie"];
|
|
1679
|
+
if (setCookieHeader) {
|
|
1680
|
+
if (Array.isArray(setCookieHeader)) {
|
|
1681
|
+
allSetCookies.push(...setCookieHeader);
|
|
1682
|
+
} else {
|
|
1683
|
+
allSetCookies.push(setCookieHeader);
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
if (allResponses.length > 1) {
|
|
1688
|
+
config.redirectHistory = config.redirectHistory || [];
|
|
1689
|
+
let currentUrl = config.url || "";
|
|
1690
|
+
for (let i = 0;i < allResponses.length - 1; i++) {
|
|
1691
|
+
const resp = allResponses[i];
|
|
1692
|
+
const respHeaders = this.parseHeaders(resp.headers);
|
|
1693
|
+
const statusMatch = resp.headers.match(/HTTP\/[\d.]+ (\d+)/);
|
|
1694
|
+
const statusCode = statusMatch ? parseInt(statusMatch[1]) : 0;
|
|
1695
|
+
const locationHeader = respHeaders["location"] || "";
|
|
1696
|
+
config.redirectHistory.push({
|
|
1697
|
+
url: currentUrl,
|
|
1698
|
+
statusCode,
|
|
1699
|
+
statusText: this.getStatusText(statusCode),
|
|
1700
|
+
headers: new RezoHeaders(respHeaders),
|
|
1701
|
+
method: config.method || "GET",
|
|
1702
|
+
cookies: [],
|
|
1703
|
+
duration: 0,
|
|
1704
|
+
request: originalRequest
|
|
1705
|
+
});
|
|
1706
|
+
if (locationHeader) {
|
|
1707
|
+
try {
|
|
1708
|
+
currentUrl = new URL(locationHeader, currentUrl).toString();
|
|
1709
|
+
} catch {
|
|
1710
|
+
currentUrl = locationHeader;
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
config.redirectCount = allResponses.length - 1;
|
|
1672
1715
|
}
|
|
1673
1716
|
const statusMatch = headerSection.match(/HTTP\/[\d.]+ (\d+)/);
|
|
1674
1717
|
const status = statusMatch ? parseInt(statusMatch[1]) : parseInt(stats["http_code"]) || 200;
|
|
1675
1718
|
const statusText = this.getStatusText(status);
|
|
1676
1719
|
const headers = this.parseHeaders(headerSection);
|
|
1677
1720
|
const rezoHeaders = new RezoHeaders(headers);
|
|
1678
|
-
|
|
1721
|
+
rezoHeaders.delete("set-cookie");
|
|
1722
|
+
const responseCookies = this.parseCookiesFromStrings(allSetCookies, config.url || "");
|
|
1679
1723
|
const mergedCookieArray = mergeRequestAndResponseCookies(config.requestCookies, responseCookies.array);
|
|
1680
1724
|
let cookies;
|
|
1681
1725
|
if (mergedCookieArray.length > 0) {
|
|
@@ -1714,7 +1758,7 @@ class CurlResponseParser {
|
|
|
1714
1758
|
const isSecure = config.url?.startsWith("https") || false;
|
|
1715
1759
|
config.adapterUsed = "curl";
|
|
1716
1760
|
config.isSecure = isSecure;
|
|
1717
|
-
config.finalUrl = stats["
|
|
1761
|
+
config.finalUrl = stats["url_effective"] || config.url || "";
|
|
1718
1762
|
if (!config.network) {
|
|
1719
1763
|
config.network = {};
|
|
1720
1764
|
}
|
|
@@ -1728,7 +1772,7 @@ class CurlResponseParser {
|
|
|
1728
1772
|
config.transfer.bodySize = responseBody.length;
|
|
1729
1773
|
config.transfer.headerSize = headerSection.length;
|
|
1730
1774
|
config.responseCookies = cookies;
|
|
1731
|
-
const finalUrl = stats["
|
|
1775
|
+
const finalUrl = stats["url_effective"] || config.url || "";
|
|
1732
1776
|
const urls = buildUrlTree(config, finalUrl);
|
|
1733
1777
|
const timingDurations = getTimingDurations(config);
|
|
1734
1778
|
debugLog.responseHeaders(config, headers);
|
|
@@ -1812,6 +1856,55 @@ class CurlResponseParser {
|
|
|
1812
1856
|
setCookiesString: cookieArray
|
|
1813
1857
|
};
|
|
1814
1858
|
}
|
|
1859
|
+
static parseAllHttpResponses(output) {
|
|
1860
|
+
const responses = [];
|
|
1861
|
+
const httpPattern = /HTTP\/[\d.]+ \d+/g;
|
|
1862
|
+
const matches = [];
|
|
1863
|
+
let match;
|
|
1864
|
+
while ((match = httpPattern.exec(output)) !== null) {
|
|
1865
|
+
matches.push(match.index);
|
|
1866
|
+
}
|
|
1867
|
+
if (matches.length === 0) {
|
|
1868
|
+
return [{ headers: "", body: output }];
|
|
1869
|
+
}
|
|
1870
|
+
for (let i = 0;i < matches.length; i++) {
|
|
1871
|
+
const start = matches[i];
|
|
1872
|
+
const end = i < matches.length - 1 ? matches[i + 1] : output.length;
|
|
1873
|
+
const segment = output.slice(start, end);
|
|
1874
|
+
const separator = segment.indexOf(`\r
|
|
1875
|
+
\r
|
|
1876
|
+
`);
|
|
1877
|
+
if (separator !== -1) {
|
|
1878
|
+
const headers = segment.slice(0, separator);
|
|
1879
|
+
const body = segment.slice(separator + 4);
|
|
1880
|
+
responses.push({ headers, body });
|
|
1881
|
+
} else {
|
|
1882
|
+
responses.push({ headers: segment.trim(), body: "" });
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
if (responses.length > 1) {
|
|
1886
|
+
const lastBody = responses[responses.length - 1].body;
|
|
1887
|
+
for (let i = 0;i < responses.length - 1; i++) {
|
|
1888
|
+
responses[i].body = "";
|
|
1889
|
+
}
|
|
1890
|
+
responses[responses.length - 1].body = lastBody;
|
|
1891
|
+
}
|
|
1892
|
+
return responses;
|
|
1893
|
+
}
|
|
1894
|
+
static parseCookiesFromStrings(setCookieStrings, url) {
|
|
1895
|
+
if (setCookieStrings.length === 0) {
|
|
1896
|
+
return {
|
|
1897
|
+
array: [],
|
|
1898
|
+
serialized: [],
|
|
1899
|
+
netscape: "",
|
|
1900
|
+
string: "",
|
|
1901
|
+
setCookiesString: []
|
|
1902
|
+
};
|
|
1903
|
+
}
|
|
1904
|
+
const jar = new RezoCookieJar;
|
|
1905
|
+
jar.setCookiesSync(setCookieStrings, url);
|
|
1906
|
+
return jar.cookies();
|
|
1907
|
+
}
|
|
1815
1908
|
}
|
|
1816
1909
|
|
|
1817
1910
|
class CurlExecutor {
|
|
@@ -2148,6 +2241,9 @@ async function executeRequest(options, defaultOptions, jar) {
|
|
|
2148
2241
|
}
|
|
2149
2242
|
}
|
|
2150
2243
|
}
|
|
2244
|
+
if (!config.requestId) {
|
|
2245
|
+
config.requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
|
|
2246
|
+
}
|
|
2151
2247
|
debugLog.requestStart(config, requestUrl, method);
|
|
2152
2248
|
let streamResponse;
|
|
2153
2249
|
let downloadResponse;
|
|
@@ -2201,6 +2297,11 @@ async function executeRequest(options, defaultOptions, jar) {
|
|
|
2201
2297
|
const duration = perform.now();
|
|
2202
2298
|
debugLog.response(config, response.status, response.statusText, duration);
|
|
2203
2299
|
debugLog.cookies(config, response.cookies?.array?.length || 0);
|
|
2300
|
+
if (response.cookies?.setCookiesString?.length > 0 && jar) {
|
|
2301
|
+
try {
|
|
2302
|
+
jar.setCookiesSync(response.cookies.setCookiesString, response.finalUrl || requestUrl);
|
|
2303
|
+
} catch (e) {}
|
|
2304
|
+
}
|
|
2204
2305
|
if (cache) {
|
|
2205
2306
|
if (response.status === 304 && cachedEntry) {
|
|
2206
2307
|
const responseHeaders = response.headers instanceof RezoHeaders ? Object.fromEntries(response.headers.entries()) : response.headers;
|
|
@@ -2220,21 +2321,31 @@ async function executeRequest(options, defaultOptions, jar) {
|
|
|
2220
2321
|
if (config.retry) {
|
|
2221
2322
|
const maxRetries = config.retry.maxRetries || 0;
|
|
2222
2323
|
const statusCodes = config.retry.statusCodes || [408, 429, 500, 502, 503, 504];
|
|
2223
|
-
if (
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2324
|
+
if (statusCodes.includes(response.status)) {
|
|
2325
|
+
if (config.retryAttempts < maxRetries) {
|
|
2326
|
+
config.retryAttempts++;
|
|
2327
|
+
if (!config.errors)
|
|
2328
|
+
config.errors = [];
|
|
2329
|
+
config.errors.push({
|
|
2330
|
+
attempt: config.retryAttempts,
|
|
2331
|
+
error: httpError,
|
|
2332
|
+
duration: perform.now()
|
|
2333
|
+
});
|
|
2334
|
+
perform.reset();
|
|
2335
|
+
const retryDelay = config.retry.retryDelay || 0;
|
|
2336
|
+
const incrementDelay = config.retry.incrementDelay || false;
|
|
2337
|
+
const delay = incrementDelay ? retryDelay * config.retryAttempts : retryDelay;
|
|
2338
|
+
debugLog.retry(config, config.retryAttempts, maxRetries, response.status, delay);
|
|
2339
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
2340
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
2341
|
+
await hook(config, httpError, config.retryAttempts);
|
|
2342
|
+
}
|
|
2231
2343
|
}
|
|
2344
|
+
if (delay > 0) {
|
|
2345
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
2346
|
+
}
|
|
2347
|
+
return executeRequest(options, defaultOptions, jar);
|
|
2232
2348
|
}
|
|
2233
|
-
if (delay > 0) {
|
|
2234
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
2235
|
-
}
|
|
2236
|
-
config.retryAttempts++;
|
|
2237
|
-
return executeRequest(options, defaultOptions, jar);
|
|
2238
2349
|
}
|
|
2239
2350
|
debugLog.maxRetries(config, maxRetries);
|
|
2240
2351
|
}
|
package/dist/adapters/curl.js
CHANGED
|
@@ -195,7 +195,7 @@ function buildUrlTree(config, finalUrl) {
|
|
|
195
195
|
}
|
|
196
196
|
return urls.length > 0 ? urls : [finalUrl];
|
|
197
197
|
}
|
|
198
|
-
async function updateCookies(config, cookieStrings, url) {
|
|
198
|
+
async function updateCookies(config, cookieStrings, url, rootJar) {
|
|
199
199
|
if (!cookieStrings || cookieStrings.length === 0)
|
|
200
200
|
return;
|
|
201
201
|
const tempJar = new RezoCookieJar;
|
|
@@ -233,8 +233,9 @@ async function updateCookies(config, cookieStrings, url) {
|
|
|
233
233
|
acceptedCookies.push(...parsedCookies.array);
|
|
234
234
|
}
|
|
235
235
|
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
236
|
-
|
|
237
|
-
|
|
236
|
+
const jarToUpdate = rootJar || config.cookieJar;
|
|
237
|
+
if (!config.disableCookieJar && jarToUpdate) {
|
|
238
|
+
jarToUpdate.setCookiesSync(acceptedCookieStrings, url);
|
|
238
239
|
}
|
|
239
240
|
if (config.useCookies) {
|
|
240
241
|
const existingArray = config.responseCookies?.array || [];
|
|
@@ -1484,14 +1485,15 @@ class CurlCommandBuilder {
|
|
|
1484
1485
|
case "socks5h":
|
|
1485
1486
|
this.addArg("--socks5", hostPort);
|
|
1486
1487
|
if (auth) {
|
|
1487
|
-
this.addArg("--socks5-
|
|
1488
|
+
this.addArg("--socks5-basic");
|
|
1489
|
+
this.addArg("--proxy-user", auth);
|
|
1488
1490
|
}
|
|
1489
1491
|
break;
|
|
1490
1492
|
case "socks4":
|
|
1491
1493
|
case "socks4a":
|
|
1492
1494
|
this.addArg("--socks4", hostPort);
|
|
1493
1495
|
if (username) {
|
|
1494
|
-
this.addArg("--
|
|
1496
|
+
this.addArg("--proxy-user", username);
|
|
1495
1497
|
}
|
|
1496
1498
|
break;
|
|
1497
1499
|
case "http":
|
|
@@ -1634,6 +1636,7 @@ class CurlCommandBuilder {
|
|
|
1634
1636
|
"local_ip:%{local_ip}",
|
|
1635
1637
|
"local_port:%{local_port}",
|
|
1636
1638
|
"redirect_url:%{redirect_url}",
|
|
1639
|
+
"url_effective:%{url_effective}",
|
|
1637
1640
|
"ssl_verify_result:%{ssl_verify_result}",
|
|
1638
1641
|
"content_type:%{content_type}",
|
|
1639
1642
|
"---CURL_STATS_END---"
|
|
@@ -1655,27 +1658,68 @@ class CurlResponseParser {
|
|
|
1655
1658
|
body = stdout.slice(0, statsStart);
|
|
1656
1659
|
for (const line of statsSection.split(`
|
|
1657
1660
|
`)) {
|
|
1658
|
-
const
|
|
1659
|
-
if (
|
|
1660
|
-
|
|
1661
|
+
const colonIndex = line.indexOf(":");
|
|
1662
|
+
if (colonIndex !== -1) {
|
|
1663
|
+
const key = line.slice(0, colonIndex).trim();
|
|
1664
|
+
const value = line.slice(colonIndex + 1).trim();
|
|
1665
|
+
if (key && value !== undefined) {
|
|
1666
|
+
stats[key] = value;
|
|
1667
|
+
}
|
|
1661
1668
|
}
|
|
1662
1669
|
}
|
|
1663
1670
|
}
|
|
1664
|
-
const
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
let
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1671
|
+
const allResponses = this.parseAllHttpResponses(body);
|
|
1672
|
+
const finalResponse = allResponses[allResponses.length - 1] || { headers: "", body };
|
|
1673
|
+
let headerSection = finalResponse.headers;
|
|
1674
|
+
let responseBody = finalResponse.body;
|
|
1675
|
+
const allSetCookies = [];
|
|
1676
|
+
for (const resp of allResponses) {
|
|
1677
|
+
const respHeaders = this.parseHeaders(resp.headers);
|
|
1678
|
+
const setCookieHeader = respHeaders["set-cookie"];
|
|
1679
|
+
if (setCookieHeader) {
|
|
1680
|
+
if (Array.isArray(setCookieHeader)) {
|
|
1681
|
+
allSetCookies.push(...setCookieHeader);
|
|
1682
|
+
} else {
|
|
1683
|
+
allSetCookies.push(setCookieHeader);
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
if (allResponses.length > 1) {
|
|
1688
|
+
config.redirectHistory = config.redirectHistory || [];
|
|
1689
|
+
let currentUrl = config.url || "";
|
|
1690
|
+
for (let i = 0;i < allResponses.length - 1; i++) {
|
|
1691
|
+
const resp = allResponses[i];
|
|
1692
|
+
const respHeaders = this.parseHeaders(resp.headers);
|
|
1693
|
+
const statusMatch = resp.headers.match(/HTTP\/[\d.]+ (\d+)/);
|
|
1694
|
+
const statusCode = statusMatch ? parseInt(statusMatch[1]) : 0;
|
|
1695
|
+
const locationHeader = respHeaders["location"] || "";
|
|
1696
|
+
config.redirectHistory.push({
|
|
1697
|
+
url: currentUrl,
|
|
1698
|
+
statusCode,
|
|
1699
|
+
statusText: this.getStatusText(statusCode),
|
|
1700
|
+
headers: new RezoHeaders(respHeaders),
|
|
1701
|
+
method: config.method || "GET",
|
|
1702
|
+
cookies: [],
|
|
1703
|
+
duration: 0,
|
|
1704
|
+
request: originalRequest
|
|
1705
|
+
});
|
|
1706
|
+
if (locationHeader) {
|
|
1707
|
+
try {
|
|
1708
|
+
currentUrl = new URL(locationHeader, currentUrl).toString();
|
|
1709
|
+
} catch {
|
|
1710
|
+
currentUrl = locationHeader;
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
config.redirectCount = allResponses.length - 1;
|
|
1672
1715
|
}
|
|
1673
1716
|
const statusMatch = headerSection.match(/HTTP\/[\d.]+ (\d+)/);
|
|
1674
1717
|
const status = statusMatch ? parseInt(statusMatch[1]) : parseInt(stats["http_code"]) || 200;
|
|
1675
1718
|
const statusText = this.getStatusText(status);
|
|
1676
1719
|
const headers = this.parseHeaders(headerSection);
|
|
1677
1720
|
const rezoHeaders = new RezoHeaders(headers);
|
|
1678
|
-
|
|
1721
|
+
rezoHeaders.delete("set-cookie");
|
|
1722
|
+
const responseCookies = this.parseCookiesFromStrings(allSetCookies, config.url || "");
|
|
1679
1723
|
const mergedCookieArray = mergeRequestAndResponseCookies(config.requestCookies, responseCookies.array);
|
|
1680
1724
|
let cookies;
|
|
1681
1725
|
if (mergedCookieArray.length > 0) {
|
|
@@ -1714,7 +1758,7 @@ class CurlResponseParser {
|
|
|
1714
1758
|
const isSecure = config.url?.startsWith("https") || false;
|
|
1715
1759
|
config.adapterUsed = "curl";
|
|
1716
1760
|
config.isSecure = isSecure;
|
|
1717
|
-
config.finalUrl = stats["
|
|
1761
|
+
config.finalUrl = stats["url_effective"] || config.url || "";
|
|
1718
1762
|
if (!config.network) {
|
|
1719
1763
|
config.network = {};
|
|
1720
1764
|
}
|
|
@@ -1728,7 +1772,7 @@ class CurlResponseParser {
|
|
|
1728
1772
|
config.transfer.bodySize = responseBody.length;
|
|
1729
1773
|
config.transfer.headerSize = headerSection.length;
|
|
1730
1774
|
config.responseCookies = cookies;
|
|
1731
|
-
const finalUrl = stats["
|
|
1775
|
+
const finalUrl = stats["url_effective"] || config.url || "";
|
|
1732
1776
|
const urls = buildUrlTree(config, finalUrl);
|
|
1733
1777
|
const timingDurations = getTimingDurations(config);
|
|
1734
1778
|
debugLog.responseHeaders(config, headers);
|
|
@@ -1812,6 +1856,55 @@ class CurlResponseParser {
|
|
|
1812
1856
|
setCookiesString: cookieArray
|
|
1813
1857
|
};
|
|
1814
1858
|
}
|
|
1859
|
+
static parseAllHttpResponses(output) {
|
|
1860
|
+
const responses = [];
|
|
1861
|
+
const httpPattern = /HTTP\/[\d.]+ \d+/g;
|
|
1862
|
+
const matches = [];
|
|
1863
|
+
let match;
|
|
1864
|
+
while ((match = httpPattern.exec(output)) !== null) {
|
|
1865
|
+
matches.push(match.index);
|
|
1866
|
+
}
|
|
1867
|
+
if (matches.length === 0) {
|
|
1868
|
+
return [{ headers: "", body: output }];
|
|
1869
|
+
}
|
|
1870
|
+
for (let i = 0;i < matches.length; i++) {
|
|
1871
|
+
const start = matches[i];
|
|
1872
|
+
const end = i < matches.length - 1 ? matches[i + 1] : output.length;
|
|
1873
|
+
const segment = output.slice(start, end);
|
|
1874
|
+
const separator = segment.indexOf(`\r
|
|
1875
|
+
\r
|
|
1876
|
+
`);
|
|
1877
|
+
if (separator !== -1) {
|
|
1878
|
+
const headers = segment.slice(0, separator);
|
|
1879
|
+
const body = segment.slice(separator + 4);
|
|
1880
|
+
responses.push({ headers, body });
|
|
1881
|
+
} else {
|
|
1882
|
+
responses.push({ headers: segment.trim(), body: "" });
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
if (responses.length > 1) {
|
|
1886
|
+
const lastBody = responses[responses.length - 1].body;
|
|
1887
|
+
for (let i = 0;i < responses.length - 1; i++) {
|
|
1888
|
+
responses[i].body = "";
|
|
1889
|
+
}
|
|
1890
|
+
responses[responses.length - 1].body = lastBody;
|
|
1891
|
+
}
|
|
1892
|
+
return responses;
|
|
1893
|
+
}
|
|
1894
|
+
static parseCookiesFromStrings(setCookieStrings, url) {
|
|
1895
|
+
if (setCookieStrings.length === 0) {
|
|
1896
|
+
return {
|
|
1897
|
+
array: [],
|
|
1898
|
+
serialized: [],
|
|
1899
|
+
netscape: "",
|
|
1900
|
+
string: "",
|
|
1901
|
+
setCookiesString: []
|
|
1902
|
+
};
|
|
1903
|
+
}
|
|
1904
|
+
const jar = new RezoCookieJar;
|
|
1905
|
+
jar.setCookiesSync(setCookieStrings, url);
|
|
1906
|
+
return jar.cookies();
|
|
1907
|
+
}
|
|
1815
1908
|
}
|
|
1816
1909
|
|
|
1817
1910
|
class CurlExecutor {
|
|
@@ -2148,6 +2241,9 @@ export async function executeRequest(options, defaultOptions, jar) {
|
|
|
2148
2241
|
}
|
|
2149
2242
|
}
|
|
2150
2243
|
}
|
|
2244
|
+
if (!config.requestId) {
|
|
2245
|
+
config.requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
|
|
2246
|
+
}
|
|
2151
2247
|
debugLog.requestStart(config, requestUrl, method);
|
|
2152
2248
|
let streamResponse;
|
|
2153
2249
|
let downloadResponse;
|
|
@@ -2201,6 +2297,11 @@ export async function executeRequest(options, defaultOptions, jar) {
|
|
|
2201
2297
|
const duration = perform.now();
|
|
2202
2298
|
debugLog.response(config, response.status, response.statusText, duration);
|
|
2203
2299
|
debugLog.cookies(config, response.cookies?.array?.length || 0);
|
|
2300
|
+
if (response.cookies?.setCookiesString?.length > 0 && jar) {
|
|
2301
|
+
try {
|
|
2302
|
+
jar.setCookiesSync(response.cookies.setCookiesString, response.finalUrl || requestUrl);
|
|
2303
|
+
} catch (e) {}
|
|
2304
|
+
}
|
|
2204
2305
|
if (cache) {
|
|
2205
2306
|
if (response.status === 304 && cachedEntry) {
|
|
2206
2307
|
const responseHeaders = response.headers instanceof RezoHeaders ? Object.fromEntries(response.headers.entries()) : response.headers;
|
|
@@ -2220,21 +2321,31 @@ export async function executeRequest(options, defaultOptions, jar) {
|
|
|
2220
2321
|
if (config.retry) {
|
|
2221
2322
|
const maxRetries = config.retry.maxRetries || 0;
|
|
2222
2323
|
const statusCodes = config.retry.statusCodes || [408, 429, 500, 502, 503, 504];
|
|
2223
|
-
if (
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2324
|
+
if (statusCodes.includes(response.status)) {
|
|
2325
|
+
if (config.retryAttempts < maxRetries) {
|
|
2326
|
+
config.retryAttempts++;
|
|
2327
|
+
if (!config.errors)
|
|
2328
|
+
config.errors = [];
|
|
2329
|
+
config.errors.push({
|
|
2330
|
+
attempt: config.retryAttempts,
|
|
2331
|
+
error: httpError,
|
|
2332
|
+
duration: perform.now()
|
|
2333
|
+
});
|
|
2334
|
+
perform.reset();
|
|
2335
|
+
const retryDelay = config.retry.retryDelay || 0;
|
|
2336
|
+
const incrementDelay = config.retry.incrementDelay || false;
|
|
2337
|
+
const delay = incrementDelay ? retryDelay * config.retryAttempts : retryDelay;
|
|
2338
|
+
debugLog.retry(config, config.retryAttempts, maxRetries, response.status, delay);
|
|
2339
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
2340
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
2341
|
+
await hook(config, httpError, config.retryAttempts);
|
|
2342
|
+
}
|
|
2231
2343
|
}
|
|
2344
|
+
if (delay > 0) {
|
|
2345
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
2346
|
+
}
|
|
2347
|
+
return executeRequest(options, defaultOptions, jar);
|
|
2232
2348
|
}
|
|
2233
|
-
if (delay > 0) {
|
|
2234
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
2235
|
-
}
|
|
2236
|
-
config.retryAttempts++;
|
|
2237
|
-
return executeRequest(options, defaultOptions, jar);
|
|
2238
2349
|
}
|
|
2239
2350
|
debugLog.maxRetries(config, maxRetries);
|
|
2240
2351
|
}
|
|
@@ -4355,6 +4355,71 @@ export declare class Rezo {
|
|
|
4355
4355
|
* @see {@link cookieJar} - Access the underlying RezoCookieJar for more control
|
|
4356
4356
|
*/
|
|
4357
4357
|
clearCookies(): void;
|
|
4358
|
+
/**
|
|
4359
|
+
* Convert a Rezo request configuration to a cURL command string.
|
|
4360
|
+
*
|
|
4361
|
+
* Generates a valid cURL command that can be executed in a terminal to
|
|
4362
|
+
* reproduce the same HTTP request. Useful for:
|
|
4363
|
+
* - Debugging and sharing requests
|
|
4364
|
+
* - Documentation and examples
|
|
4365
|
+
* - Testing requests outside of Node.js
|
|
4366
|
+
* - Exporting requests to other tools
|
|
4367
|
+
*
|
|
4368
|
+
* @param config - Request configuration object
|
|
4369
|
+
* @returns A cURL command string
|
|
4370
|
+
*
|
|
4371
|
+
* @example
|
|
4372
|
+
* ```typescript
|
|
4373
|
+
* const curl = Rezo.toCurl({
|
|
4374
|
+
* url: 'https://api.example.com/users',
|
|
4375
|
+
* method: 'POST',
|
|
4376
|
+
* headers: { 'Content-Type': 'application/json' },
|
|
4377
|
+
* body: { name: 'John', email: 'john@example.com' }
|
|
4378
|
+
* });
|
|
4379
|
+
* // Output: curl -X POST -H 'content-type: application/json' --data-raw '{"name":"John","email":"john@example.com"}' -L --compressed 'https://api.example.com/users'
|
|
4380
|
+
* ```
|
|
4381
|
+
*/
|
|
4382
|
+
static toCurl(config: RezoRequestConfig | RezoRequestOptions): string;
|
|
4383
|
+
/**
|
|
4384
|
+
* Parse a cURL command string into a Rezo request configuration.
|
|
4385
|
+
*
|
|
4386
|
+
* Converts a cURL command into a configuration object that can be
|
|
4387
|
+
* passed directly to Rezo request methods. Useful for:
|
|
4388
|
+
* - Importing requests from browser DevTools
|
|
4389
|
+
* - Converting curl examples from API documentation
|
|
4390
|
+
* - Migrating scripts from curl to Rezo
|
|
4391
|
+
*
|
|
4392
|
+
* Supports common cURL options:
|
|
4393
|
+
* - `-X, --request` - HTTP method
|
|
4394
|
+
* - `-H, --header` - Request headers
|
|
4395
|
+
* - `-d, --data, --data-raw, --data-binary` - Request body
|
|
4396
|
+
* - `-u, --user` - Basic authentication
|
|
4397
|
+
* - `-x, --proxy` - Proxy configuration
|
|
4398
|
+
* - `--socks5, --socks4` - SOCKS proxy
|
|
4399
|
+
* - `-L, --location` - Follow redirects
|
|
4400
|
+
* - `--max-redirs` - Maximum redirects
|
|
4401
|
+
* - `--max-time` - Request timeout
|
|
4402
|
+
* - `-k, --insecure` - Skip TLS verification
|
|
4403
|
+
* - `-A, --user-agent` - User agent header
|
|
4404
|
+
*
|
|
4405
|
+
* @param curlCommand - A cURL command string
|
|
4406
|
+
* @returns A request configuration object
|
|
4407
|
+
*
|
|
4408
|
+
* @example
|
|
4409
|
+
* ```typescript
|
|
4410
|
+
* // From browser DevTools "Copy as cURL"
|
|
4411
|
+
* const config = Rezo.fromCurl(`
|
|
4412
|
+
* curl 'https://api.example.com/data' \\
|
|
4413
|
+
* -H 'Authorization: Bearer token123' \\
|
|
4414
|
+
* -H 'Content-Type: application/json'
|
|
4415
|
+
* `);
|
|
4416
|
+
*
|
|
4417
|
+
* // Use with Rezo
|
|
4418
|
+
* const rezo = new Rezo();
|
|
4419
|
+
* const response = await rezo.request(config);
|
|
4420
|
+
* ```
|
|
4421
|
+
*/
|
|
4422
|
+
static fromCurl(curlCommand: string): RezoRequestOptions;
|
|
4358
4423
|
}
|
|
4359
4424
|
/**
|
|
4360
4425
|
* Extended Rezo instance with Axios-compatible static helpers.
|
|
@@ -4355,6 +4355,71 @@ export declare class Rezo {
|
|
|
4355
4355
|
* @see {@link cookieJar} - Access the underlying RezoCookieJar for more control
|
|
4356
4356
|
*/
|
|
4357
4357
|
clearCookies(): void;
|
|
4358
|
+
/**
|
|
4359
|
+
* Convert a Rezo request configuration to a cURL command string.
|
|
4360
|
+
*
|
|
4361
|
+
* Generates a valid cURL command that can be executed in a terminal to
|
|
4362
|
+
* reproduce the same HTTP request. Useful for:
|
|
4363
|
+
* - Debugging and sharing requests
|
|
4364
|
+
* - Documentation and examples
|
|
4365
|
+
* - Testing requests outside of Node.js
|
|
4366
|
+
* - Exporting requests to other tools
|
|
4367
|
+
*
|
|
4368
|
+
* @param config - Request configuration object
|
|
4369
|
+
* @returns A cURL command string
|
|
4370
|
+
*
|
|
4371
|
+
* @example
|
|
4372
|
+
* ```typescript
|
|
4373
|
+
* const curl = Rezo.toCurl({
|
|
4374
|
+
* url: 'https://api.example.com/users',
|
|
4375
|
+
* method: 'POST',
|
|
4376
|
+
* headers: { 'Content-Type': 'application/json' },
|
|
4377
|
+
* body: { name: 'John', email: 'john@example.com' }
|
|
4378
|
+
* });
|
|
4379
|
+
* // Output: curl -X POST -H 'content-type: application/json' --data-raw '{"name":"John","email":"john@example.com"}' -L --compressed 'https://api.example.com/users'
|
|
4380
|
+
* ```
|
|
4381
|
+
*/
|
|
4382
|
+
static toCurl(config: RezoRequestConfig | RezoRequestOptions): string;
|
|
4383
|
+
/**
|
|
4384
|
+
* Parse a cURL command string into a Rezo request configuration.
|
|
4385
|
+
*
|
|
4386
|
+
* Converts a cURL command into a configuration object that can be
|
|
4387
|
+
* passed directly to Rezo request methods. Useful for:
|
|
4388
|
+
* - Importing requests from browser DevTools
|
|
4389
|
+
* - Converting curl examples from API documentation
|
|
4390
|
+
* - Migrating scripts from curl to Rezo
|
|
4391
|
+
*
|
|
4392
|
+
* Supports common cURL options:
|
|
4393
|
+
* - `-X, --request` - HTTP method
|
|
4394
|
+
* - `-H, --header` - Request headers
|
|
4395
|
+
* - `-d, --data, --data-raw, --data-binary` - Request body
|
|
4396
|
+
* - `-u, --user` - Basic authentication
|
|
4397
|
+
* - `-x, --proxy` - Proxy configuration
|
|
4398
|
+
* - `--socks5, --socks4` - SOCKS proxy
|
|
4399
|
+
* - `-L, --location` - Follow redirects
|
|
4400
|
+
* - `--max-redirs` - Maximum redirects
|
|
4401
|
+
* - `--max-time` - Request timeout
|
|
4402
|
+
* - `-k, --insecure` - Skip TLS verification
|
|
4403
|
+
* - `-A, --user-agent` - User agent header
|
|
4404
|
+
*
|
|
4405
|
+
* @param curlCommand - A cURL command string
|
|
4406
|
+
* @returns A request configuration object
|
|
4407
|
+
*
|
|
4408
|
+
* @example
|
|
4409
|
+
* ```typescript
|
|
4410
|
+
* // From browser DevTools "Copy as cURL"
|
|
4411
|
+
* const config = Rezo.fromCurl(`
|
|
4412
|
+
* curl 'https://api.example.com/data' \\
|
|
4413
|
+
* -H 'Authorization: Bearer token123' \\
|
|
4414
|
+
* -H 'Content-Type: application/json'
|
|
4415
|
+
* `);
|
|
4416
|
+
*
|
|
4417
|
+
* // Use with Rezo
|
|
4418
|
+
* const rezo = new Rezo();
|
|
4419
|
+
* const response = await rezo.request(config);
|
|
4420
|
+
* ```
|
|
4421
|
+
*/
|
|
4422
|
+
static fromCurl(curlCommand: string): RezoRequestOptions;
|
|
4358
4423
|
}
|
|
4359
4424
|
/**
|
|
4360
4425
|
* Extended Rezo instance with Axios-compatible static helpers.
|