rezo 1.0.42 → 1.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +131 -29
- package/dist/adapters/curl.js +131 -29
- package/dist/adapters/entries/curl.d.ts +65 -0
- package/dist/adapters/entries/fetch.d.ts +65 -0
- package/dist/adapters/entries/http.d.ts +65 -0
- package/dist/adapters/entries/http2.d.ts +65 -0
- package/dist/adapters/entries/react-native.d.ts +65 -0
- package/dist/adapters/entries/xhr.d.ts +65 -0
- package/dist/adapters/http2.cjs +209 -22
- package/dist/adapters/http2.js +209 -22
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +9 -13
- package/dist/cache/index.js +0 -2
- package/dist/core/rezo.cjs +7 -0
- package/dist/core/rezo.js +7 -0
- package/dist/crawler/addon/decodo/index.cjs +1 -0
- package/dist/crawler/addon/decodo/index.js +1 -0
- package/dist/crawler/crawler-options.cjs +1 -0
- package/dist/crawler/crawler-options.js +1 -0
- package/dist/crawler/crawler.cjs +1070 -0
- package/dist/crawler/crawler.js +1068 -0
- package/dist/crawler/index.cjs +40 -0
- package/dist/{plugin → crawler}/index.js +4 -2
- package/dist/crawler/plugin/file-cacher.cjs +19 -0
- package/dist/crawler/plugin/file-cacher.js +19 -0
- package/dist/crawler/plugin/index.cjs +1 -0
- package/dist/crawler/plugin/index.js +1 -0
- package/dist/crawler/plugin/navigation-history.cjs +43 -0
- package/dist/crawler/plugin/navigation-history.js +43 -0
- package/dist/crawler/plugin/robots-txt.cjs +2 -0
- package/dist/crawler/plugin/robots-txt.js +2 -0
- package/dist/crawler/plugin/url-store.cjs +18 -0
- package/dist/crawler/plugin/url-store.js +18 -0
- package/dist/crawler.d.ts +511 -183
- package/dist/entries/crawler.cjs +5 -5
- package/dist/entries/crawler.js +2 -2
- package/dist/index.cjs +27 -24
- package/dist/index.d.ts +73 -0
- package/dist/index.js +1 -0
- package/dist/internal/agents/base.cjs +113 -0
- package/dist/internal/agents/base.js +110 -0
- package/dist/internal/agents/http-proxy.cjs +89 -0
- package/dist/internal/agents/http-proxy.js +86 -0
- package/dist/internal/agents/https-proxy.cjs +176 -0
- package/dist/internal/agents/https-proxy.js +173 -0
- package/dist/internal/agents/index.cjs +10 -0
- package/dist/internal/agents/index.js +5 -0
- package/dist/internal/agents/socks-client.cjs +571 -0
- package/dist/internal/agents/socks-client.js +567 -0
- package/dist/internal/agents/socks-proxy.cjs +75 -0
- package/dist/internal/agents/socks-proxy.js +72 -0
- package/dist/platform/browser.d.ts +65 -0
- package/dist/platform/bun.d.ts +65 -0
- package/dist/platform/deno.d.ts +65 -0
- package/dist/platform/node.d.ts +65 -0
- package/dist/platform/react-native.d.ts +65 -0
- package/dist/platform/worker.d.ts +65 -0
- package/dist/proxy/index.cjs +18 -16
- package/dist/proxy/index.js +17 -12
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/buildError.cjs +11 -2
- package/dist/responses/buildError.js +11 -2
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/curl.cjs +317 -0
- package/dist/utils/curl.js +314 -0
- package/package.json +2 -6
- package/dist/cache/file-cacher.cjs +0 -264
- package/dist/cache/file-cacher.js +0 -261
- package/dist/cache/url-store.cjs +0 -288
- package/dist/cache/url-store.js +0 -285
- package/dist/plugin/addon/decodo/index.cjs +0 -1
- package/dist/plugin/addon/decodo/index.js +0 -1
- package/dist/plugin/crawler-options.cjs +0 -1
- package/dist/plugin/crawler-options.js +0 -1
- package/dist/plugin/crawler.cjs +0 -519
- package/dist/plugin/crawler.js +0 -517
- package/dist/plugin/index.cjs +0 -36
- /package/dist/{plugin → crawler}/addon/decodo/options.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/decodo/options.js +0 -0
- /package/dist/{plugin → crawler}/addon/decodo/types.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/decodo/types.js +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/index.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/index.js +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/options.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/options.js +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/types.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/types.js +0 -0
- /package/dist/{plugin → crawler}/scraper.cjs +0 -0
- /package/dist/{plugin → crawler}/scraper.js +0 -0
package/dist/adapters/curl.cjs
CHANGED
|
@@ -1485,14 +1485,15 @@ class CurlCommandBuilder {
|
|
|
1485
1485
|
case "socks5h":
|
|
1486
1486
|
this.addArg("--socks5", hostPort);
|
|
1487
1487
|
if (auth) {
|
|
1488
|
-
this.addArg("--socks5-
|
|
1488
|
+
this.addArg("--socks5-basic");
|
|
1489
|
+
this.addArg("--proxy-user", auth);
|
|
1489
1490
|
}
|
|
1490
1491
|
break;
|
|
1491
1492
|
case "socks4":
|
|
1492
1493
|
case "socks4a":
|
|
1493
1494
|
this.addArg("--socks4", hostPort);
|
|
1494
1495
|
if (username) {
|
|
1495
|
-
this.addArg("--
|
|
1496
|
+
this.addArg("--proxy-user", username);
|
|
1496
1497
|
}
|
|
1497
1498
|
break;
|
|
1498
1499
|
case "http":
|
|
@@ -1635,6 +1636,7 @@ class CurlCommandBuilder {
|
|
|
1635
1636
|
"local_ip:%{local_ip}",
|
|
1636
1637
|
"local_port:%{local_port}",
|
|
1637
1638
|
"redirect_url:%{redirect_url}",
|
|
1639
|
+
"url_effective:%{url_effective}",
|
|
1638
1640
|
"ssl_verify_result:%{ssl_verify_result}",
|
|
1639
1641
|
"content_type:%{content_type}",
|
|
1640
1642
|
"---CURL_STATS_END---"
|
|
@@ -1656,27 +1658,68 @@ class CurlResponseParser {
|
|
|
1656
1658
|
body = stdout.slice(0, statsStart);
|
|
1657
1659
|
for (const line of statsSection.split(`
|
|
1658
1660
|
`)) {
|
|
1659
|
-
const
|
|
1660
|
-
if (
|
|
1661
|
-
|
|
1661
|
+
const colonIndex = line.indexOf(":");
|
|
1662
|
+
if (colonIndex !== -1) {
|
|
1663
|
+
const key = line.slice(0, colonIndex).trim();
|
|
1664
|
+
const value = line.slice(colonIndex + 1).trim();
|
|
1665
|
+
if (key && value !== undefined) {
|
|
1666
|
+
stats[key] = value;
|
|
1667
|
+
}
|
|
1662
1668
|
}
|
|
1663
1669
|
}
|
|
1664
1670
|
}
|
|
1665
|
-
const
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
let
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1671
|
+
const allResponses = this.parseAllHttpResponses(body);
|
|
1672
|
+
const finalResponse = allResponses[allResponses.length - 1] || { headers: "", body };
|
|
1673
|
+
let headerSection = finalResponse.headers;
|
|
1674
|
+
let responseBody = finalResponse.body;
|
|
1675
|
+
const allSetCookies = [];
|
|
1676
|
+
for (const resp of allResponses) {
|
|
1677
|
+
const respHeaders = this.parseHeaders(resp.headers);
|
|
1678
|
+
const setCookieHeader = respHeaders["set-cookie"];
|
|
1679
|
+
if (setCookieHeader) {
|
|
1680
|
+
if (Array.isArray(setCookieHeader)) {
|
|
1681
|
+
allSetCookies.push(...setCookieHeader);
|
|
1682
|
+
} else {
|
|
1683
|
+
allSetCookies.push(setCookieHeader);
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
if (allResponses.length > 1) {
|
|
1688
|
+
config.redirectHistory = config.redirectHistory || [];
|
|
1689
|
+
let currentUrl = config.url || "";
|
|
1690
|
+
for (let i = 0;i < allResponses.length - 1; i++) {
|
|
1691
|
+
const resp = allResponses[i];
|
|
1692
|
+
const respHeaders = this.parseHeaders(resp.headers);
|
|
1693
|
+
const statusMatch = resp.headers.match(/HTTP\/[\d.]+ (\d+)/);
|
|
1694
|
+
const statusCode = statusMatch ? parseInt(statusMatch[1]) : 0;
|
|
1695
|
+
const locationHeader = respHeaders["location"] || "";
|
|
1696
|
+
config.redirectHistory.push({
|
|
1697
|
+
url: currentUrl,
|
|
1698
|
+
statusCode,
|
|
1699
|
+
statusText: this.getStatusText(statusCode),
|
|
1700
|
+
headers: new RezoHeaders(respHeaders),
|
|
1701
|
+
method: config.method || "GET",
|
|
1702
|
+
cookies: [],
|
|
1703
|
+
duration: 0,
|
|
1704
|
+
request: originalRequest
|
|
1705
|
+
});
|
|
1706
|
+
if (locationHeader) {
|
|
1707
|
+
try {
|
|
1708
|
+
currentUrl = new URL(locationHeader, currentUrl).toString();
|
|
1709
|
+
} catch {
|
|
1710
|
+
currentUrl = locationHeader;
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
config.redirectCount = allResponses.length - 1;
|
|
1673
1715
|
}
|
|
1674
1716
|
const statusMatch = headerSection.match(/HTTP\/[\d.]+ (\d+)/);
|
|
1675
1717
|
const status = statusMatch ? parseInt(statusMatch[1]) : parseInt(stats["http_code"]) || 200;
|
|
1676
1718
|
const statusText = this.getStatusText(status);
|
|
1677
1719
|
const headers = this.parseHeaders(headerSection);
|
|
1678
1720
|
const rezoHeaders = new RezoHeaders(headers);
|
|
1679
|
-
|
|
1721
|
+
rezoHeaders.delete("set-cookie");
|
|
1722
|
+
const responseCookies = this.parseCookiesFromStrings(allSetCookies, config.url || "");
|
|
1680
1723
|
const mergedCookieArray = mergeRequestAndResponseCookies(config.requestCookies, responseCookies.array);
|
|
1681
1724
|
let cookies;
|
|
1682
1725
|
if (mergedCookieArray.length > 0) {
|
|
@@ -1715,7 +1758,7 @@ class CurlResponseParser {
|
|
|
1715
1758
|
const isSecure = config.url?.startsWith("https") || false;
|
|
1716
1759
|
config.adapterUsed = "curl";
|
|
1717
1760
|
config.isSecure = isSecure;
|
|
1718
|
-
config.finalUrl = stats["
|
|
1761
|
+
config.finalUrl = stats["url_effective"] || config.url || "";
|
|
1719
1762
|
if (!config.network) {
|
|
1720
1763
|
config.network = {};
|
|
1721
1764
|
}
|
|
@@ -1729,7 +1772,7 @@ class CurlResponseParser {
|
|
|
1729
1772
|
config.transfer.bodySize = responseBody.length;
|
|
1730
1773
|
config.transfer.headerSize = headerSection.length;
|
|
1731
1774
|
config.responseCookies = cookies;
|
|
1732
|
-
const finalUrl = stats["
|
|
1775
|
+
const finalUrl = stats["url_effective"] || config.url || "";
|
|
1733
1776
|
const urls = buildUrlTree(config, finalUrl);
|
|
1734
1777
|
const timingDurations = getTimingDurations(config);
|
|
1735
1778
|
debugLog.responseHeaders(config, headers);
|
|
@@ -1813,6 +1856,55 @@ class CurlResponseParser {
|
|
|
1813
1856
|
setCookiesString: cookieArray
|
|
1814
1857
|
};
|
|
1815
1858
|
}
|
|
1859
|
+
static parseAllHttpResponses(output) {
|
|
1860
|
+
const responses = [];
|
|
1861
|
+
const httpPattern = /HTTP\/[\d.]+ \d+/g;
|
|
1862
|
+
const matches = [];
|
|
1863
|
+
let match;
|
|
1864
|
+
while ((match = httpPattern.exec(output)) !== null) {
|
|
1865
|
+
matches.push(match.index);
|
|
1866
|
+
}
|
|
1867
|
+
if (matches.length === 0) {
|
|
1868
|
+
return [{ headers: "", body: output }];
|
|
1869
|
+
}
|
|
1870
|
+
for (let i = 0;i < matches.length; i++) {
|
|
1871
|
+
const start = matches[i];
|
|
1872
|
+
const end = i < matches.length - 1 ? matches[i + 1] : output.length;
|
|
1873
|
+
const segment = output.slice(start, end);
|
|
1874
|
+
const separator = segment.indexOf(`\r
|
|
1875
|
+
\r
|
|
1876
|
+
`);
|
|
1877
|
+
if (separator !== -1) {
|
|
1878
|
+
const headers = segment.slice(0, separator);
|
|
1879
|
+
const body = segment.slice(separator + 4);
|
|
1880
|
+
responses.push({ headers, body });
|
|
1881
|
+
} else {
|
|
1882
|
+
responses.push({ headers: segment.trim(), body: "" });
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
if (responses.length > 1) {
|
|
1886
|
+
const lastBody = responses[responses.length - 1].body;
|
|
1887
|
+
for (let i = 0;i < responses.length - 1; i++) {
|
|
1888
|
+
responses[i].body = "";
|
|
1889
|
+
}
|
|
1890
|
+
responses[responses.length - 1].body = lastBody;
|
|
1891
|
+
}
|
|
1892
|
+
return responses;
|
|
1893
|
+
}
|
|
1894
|
+
static parseCookiesFromStrings(setCookieStrings, url) {
|
|
1895
|
+
if (setCookieStrings.length === 0) {
|
|
1896
|
+
return {
|
|
1897
|
+
array: [],
|
|
1898
|
+
serialized: [],
|
|
1899
|
+
netscape: "",
|
|
1900
|
+
string: "",
|
|
1901
|
+
setCookiesString: []
|
|
1902
|
+
};
|
|
1903
|
+
}
|
|
1904
|
+
const jar = new RezoCookieJar;
|
|
1905
|
+
jar.setCookiesSync(setCookieStrings, url);
|
|
1906
|
+
return jar.cookies();
|
|
1907
|
+
}
|
|
1816
1908
|
}
|
|
1817
1909
|
|
|
1818
1910
|
class CurlExecutor {
|
|
@@ -2229,21 +2321,31 @@ async function executeRequest(options, defaultOptions, jar) {
|
|
|
2229
2321
|
if (config.retry) {
|
|
2230
2322
|
const maxRetries = config.retry.maxRetries || 0;
|
|
2231
2323
|
const statusCodes = config.retry.statusCodes || [408, 429, 500, 502, 503, 504];
|
|
2232
|
-
if (
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2324
|
+
if (statusCodes.includes(response.status)) {
|
|
2325
|
+
if (config.retryAttempts < maxRetries) {
|
|
2326
|
+
config.retryAttempts++;
|
|
2327
|
+
if (!config.errors)
|
|
2328
|
+
config.errors = [];
|
|
2329
|
+
config.errors.push({
|
|
2330
|
+
attempt: config.retryAttempts,
|
|
2331
|
+
error: httpError,
|
|
2332
|
+
duration: perform.now()
|
|
2333
|
+
});
|
|
2334
|
+
perform.reset();
|
|
2335
|
+
const retryDelay = config.retry.retryDelay || 0;
|
|
2336
|
+
const incrementDelay = config.retry.incrementDelay || false;
|
|
2337
|
+
const delay = incrementDelay ? retryDelay * config.retryAttempts : retryDelay;
|
|
2338
|
+
debugLog.retry(config, config.retryAttempts, maxRetries, response.status, delay);
|
|
2339
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
2340
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
2341
|
+
await hook(config, httpError, config.retryAttempts);
|
|
2342
|
+
}
|
|
2240
2343
|
}
|
|
2344
|
+
if (delay > 0) {
|
|
2345
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
2346
|
+
}
|
|
2347
|
+
return executeRequest(options, defaultOptions, jar);
|
|
2241
2348
|
}
|
|
2242
|
-
if (delay > 0) {
|
|
2243
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
2244
|
-
}
|
|
2245
|
-
config.retryAttempts++;
|
|
2246
|
-
return executeRequest(options, defaultOptions, jar);
|
|
2247
2349
|
}
|
|
2248
2350
|
debugLog.maxRetries(config, maxRetries);
|
|
2249
2351
|
}
|
package/dist/adapters/curl.js
CHANGED
|
@@ -1485,14 +1485,15 @@ class CurlCommandBuilder {
|
|
|
1485
1485
|
case "socks5h":
|
|
1486
1486
|
this.addArg("--socks5", hostPort);
|
|
1487
1487
|
if (auth) {
|
|
1488
|
-
this.addArg("--socks5-
|
|
1488
|
+
this.addArg("--socks5-basic");
|
|
1489
|
+
this.addArg("--proxy-user", auth);
|
|
1489
1490
|
}
|
|
1490
1491
|
break;
|
|
1491
1492
|
case "socks4":
|
|
1492
1493
|
case "socks4a":
|
|
1493
1494
|
this.addArg("--socks4", hostPort);
|
|
1494
1495
|
if (username) {
|
|
1495
|
-
this.addArg("--
|
|
1496
|
+
this.addArg("--proxy-user", username);
|
|
1496
1497
|
}
|
|
1497
1498
|
break;
|
|
1498
1499
|
case "http":
|
|
@@ -1635,6 +1636,7 @@ class CurlCommandBuilder {
|
|
|
1635
1636
|
"local_ip:%{local_ip}",
|
|
1636
1637
|
"local_port:%{local_port}",
|
|
1637
1638
|
"redirect_url:%{redirect_url}",
|
|
1639
|
+
"url_effective:%{url_effective}",
|
|
1638
1640
|
"ssl_verify_result:%{ssl_verify_result}",
|
|
1639
1641
|
"content_type:%{content_type}",
|
|
1640
1642
|
"---CURL_STATS_END---"
|
|
@@ -1656,27 +1658,68 @@ class CurlResponseParser {
|
|
|
1656
1658
|
body = stdout.slice(0, statsStart);
|
|
1657
1659
|
for (const line of statsSection.split(`
|
|
1658
1660
|
`)) {
|
|
1659
|
-
const
|
|
1660
|
-
if (
|
|
1661
|
-
|
|
1661
|
+
const colonIndex = line.indexOf(":");
|
|
1662
|
+
if (colonIndex !== -1) {
|
|
1663
|
+
const key = line.slice(0, colonIndex).trim();
|
|
1664
|
+
const value = line.slice(colonIndex + 1).trim();
|
|
1665
|
+
if (key && value !== undefined) {
|
|
1666
|
+
stats[key] = value;
|
|
1667
|
+
}
|
|
1662
1668
|
}
|
|
1663
1669
|
}
|
|
1664
1670
|
}
|
|
1665
|
-
const
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
let
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1671
|
+
const allResponses = this.parseAllHttpResponses(body);
|
|
1672
|
+
const finalResponse = allResponses[allResponses.length - 1] || { headers: "", body };
|
|
1673
|
+
let headerSection = finalResponse.headers;
|
|
1674
|
+
let responseBody = finalResponse.body;
|
|
1675
|
+
const allSetCookies = [];
|
|
1676
|
+
for (const resp of allResponses) {
|
|
1677
|
+
const respHeaders = this.parseHeaders(resp.headers);
|
|
1678
|
+
const setCookieHeader = respHeaders["set-cookie"];
|
|
1679
|
+
if (setCookieHeader) {
|
|
1680
|
+
if (Array.isArray(setCookieHeader)) {
|
|
1681
|
+
allSetCookies.push(...setCookieHeader);
|
|
1682
|
+
} else {
|
|
1683
|
+
allSetCookies.push(setCookieHeader);
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
if (allResponses.length > 1) {
|
|
1688
|
+
config.redirectHistory = config.redirectHistory || [];
|
|
1689
|
+
let currentUrl = config.url || "";
|
|
1690
|
+
for (let i = 0;i < allResponses.length - 1; i++) {
|
|
1691
|
+
const resp = allResponses[i];
|
|
1692
|
+
const respHeaders = this.parseHeaders(resp.headers);
|
|
1693
|
+
const statusMatch = resp.headers.match(/HTTP\/[\d.]+ (\d+)/);
|
|
1694
|
+
const statusCode = statusMatch ? parseInt(statusMatch[1]) : 0;
|
|
1695
|
+
const locationHeader = respHeaders["location"] || "";
|
|
1696
|
+
config.redirectHistory.push({
|
|
1697
|
+
url: currentUrl,
|
|
1698
|
+
statusCode,
|
|
1699
|
+
statusText: this.getStatusText(statusCode),
|
|
1700
|
+
headers: new RezoHeaders(respHeaders),
|
|
1701
|
+
method: config.method || "GET",
|
|
1702
|
+
cookies: [],
|
|
1703
|
+
duration: 0,
|
|
1704
|
+
request: originalRequest
|
|
1705
|
+
});
|
|
1706
|
+
if (locationHeader) {
|
|
1707
|
+
try {
|
|
1708
|
+
currentUrl = new URL(locationHeader, currentUrl).toString();
|
|
1709
|
+
} catch {
|
|
1710
|
+
currentUrl = locationHeader;
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
config.redirectCount = allResponses.length - 1;
|
|
1673
1715
|
}
|
|
1674
1716
|
const statusMatch = headerSection.match(/HTTP\/[\d.]+ (\d+)/);
|
|
1675
1717
|
const status = statusMatch ? parseInt(statusMatch[1]) : parseInt(stats["http_code"]) || 200;
|
|
1676
1718
|
const statusText = this.getStatusText(status);
|
|
1677
1719
|
const headers = this.parseHeaders(headerSection);
|
|
1678
1720
|
const rezoHeaders = new RezoHeaders(headers);
|
|
1679
|
-
|
|
1721
|
+
rezoHeaders.delete("set-cookie");
|
|
1722
|
+
const responseCookies = this.parseCookiesFromStrings(allSetCookies, config.url || "");
|
|
1680
1723
|
const mergedCookieArray = mergeRequestAndResponseCookies(config.requestCookies, responseCookies.array);
|
|
1681
1724
|
let cookies;
|
|
1682
1725
|
if (mergedCookieArray.length > 0) {
|
|
@@ -1715,7 +1758,7 @@ class CurlResponseParser {
|
|
|
1715
1758
|
const isSecure = config.url?.startsWith("https") || false;
|
|
1716
1759
|
config.adapterUsed = "curl";
|
|
1717
1760
|
config.isSecure = isSecure;
|
|
1718
|
-
config.finalUrl = stats["
|
|
1761
|
+
config.finalUrl = stats["url_effective"] || config.url || "";
|
|
1719
1762
|
if (!config.network) {
|
|
1720
1763
|
config.network = {};
|
|
1721
1764
|
}
|
|
@@ -1729,7 +1772,7 @@ class CurlResponseParser {
|
|
|
1729
1772
|
config.transfer.bodySize = responseBody.length;
|
|
1730
1773
|
config.transfer.headerSize = headerSection.length;
|
|
1731
1774
|
config.responseCookies = cookies;
|
|
1732
|
-
const finalUrl = stats["
|
|
1775
|
+
const finalUrl = stats["url_effective"] || config.url || "";
|
|
1733
1776
|
const urls = buildUrlTree(config, finalUrl);
|
|
1734
1777
|
const timingDurations = getTimingDurations(config);
|
|
1735
1778
|
debugLog.responseHeaders(config, headers);
|
|
@@ -1813,6 +1856,55 @@ class CurlResponseParser {
|
|
|
1813
1856
|
setCookiesString: cookieArray
|
|
1814
1857
|
};
|
|
1815
1858
|
}
|
|
1859
|
+
static parseAllHttpResponses(output) {
|
|
1860
|
+
const responses = [];
|
|
1861
|
+
const httpPattern = /HTTP\/[\d.]+ \d+/g;
|
|
1862
|
+
const matches = [];
|
|
1863
|
+
let match;
|
|
1864
|
+
while ((match = httpPattern.exec(output)) !== null) {
|
|
1865
|
+
matches.push(match.index);
|
|
1866
|
+
}
|
|
1867
|
+
if (matches.length === 0) {
|
|
1868
|
+
return [{ headers: "", body: output }];
|
|
1869
|
+
}
|
|
1870
|
+
for (let i = 0;i < matches.length; i++) {
|
|
1871
|
+
const start = matches[i];
|
|
1872
|
+
const end = i < matches.length - 1 ? matches[i + 1] : output.length;
|
|
1873
|
+
const segment = output.slice(start, end);
|
|
1874
|
+
const separator = segment.indexOf(`\r
|
|
1875
|
+
\r
|
|
1876
|
+
`);
|
|
1877
|
+
if (separator !== -1) {
|
|
1878
|
+
const headers = segment.slice(0, separator);
|
|
1879
|
+
const body = segment.slice(separator + 4);
|
|
1880
|
+
responses.push({ headers, body });
|
|
1881
|
+
} else {
|
|
1882
|
+
responses.push({ headers: segment.trim(), body: "" });
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
if (responses.length > 1) {
|
|
1886
|
+
const lastBody = responses[responses.length - 1].body;
|
|
1887
|
+
for (let i = 0;i < responses.length - 1; i++) {
|
|
1888
|
+
responses[i].body = "";
|
|
1889
|
+
}
|
|
1890
|
+
responses[responses.length - 1].body = lastBody;
|
|
1891
|
+
}
|
|
1892
|
+
return responses;
|
|
1893
|
+
}
|
|
1894
|
+
static parseCookiesFromStrings(setCookieStrings, url) {
|
|
1895
|
+
if (setCookieStrings.length === 0) {
|
|
1896
|
+
return {
|
|
1897
|
+
array: [],
|
|
1898
|
+
serialized: [],
|
|
1899
|
+
netscape: "",
|
|
1900
|
+
string: "",
|
|
1901
|
+
setCookiesString: []
|
|
1902
|
+
};
|
|
1903
|
+
}
|
|
1904
|
+
const jar = new RezoCookieJar;
|
|
1905
|
+
jar.setCookiesSync(setCookieStrings, url);
|
|
1906
|
+
return jar.cookies();
|
|
1907
|
+
}
|
|
1816
1908
|
}
|
|
1817
1909
|
|
|
1818
1910
|
class CurlExecutor {
|
|
@@ -2229,21 +2321,31 @@ export async function executeRequest(options, defaultOptions, jar) {
|
|
|
2229
2321
|
if (config.retry) {
|
|
2230
2322
|
const maxRetries = config.retry.maxRetries || 0;
|
|
2231
2323
|
const statusCodes = config.retry.statusCodes || [408, 429, 500, 502, 503, 504];
|
|
2232
|
-
if (
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2324
|
+
if (statusCodes.includes(response.status)) {
|
|
2325
|
+
if (config.retryAttempts < maxRetries) {
|
|
2326
|
+
config.retryAttempts++;
|
|
2327
|
+
if (!config.errors)
|
|
2328
|
+
config.errors = [];
|
|
2329
|
+
config.errors.push({
|
|
2330
|
+
attempt: config.retryAttempts,
|
|
2331
|
+
error: httpError,
|
|
2332
|
+
duration: perform.now()
|
|
2333
|
+
});
|
|
2334
|
+
perform.reset();
|
|
2335
|
+
const retryDelay = config.retry.retryDelay || 0;
|
|
2336
|
+
const incrementDelay = config.retry.incrementDelay || false;
|
|
2337
|
+
const delay = incrementDelay ? retryDelay * config.retryAttempts : retryDelay;
|
|
2338
|
+
debugLog.retry(config, config.retryAttempts, maxRetries, response.status, delay);
|
|
2339
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
2340
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
2341
|
+
await hook(config, httpError, config.retryAttempts);
|
|
2342
|
+
}
|
|
2240
2343
|
}
|
|
2344
|
+
if (delay > 0) {
|
|
2345
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
2346
|
+
}
|
|
2347
|
+
return executeRequest(options, defaultOptions, jar);
|
|
2241
2348
|
}
|
|
2242
|
-
if (delay > 0) {
|
|
2243
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
2244
|
-
}
|
|
2245
|
-
config.retryAttempts++;
|
|
2246
|
-
return executeRequest(options, defaultOptions, jar);
|
|
2247
2349
|
}
|
|
2248
2350
|
debugLog.maxRetries(config, maxRetries);
|
|
2249
2351
|
}
|
|
@@ -4355,6 +4355,71 @@ export declare class Rezo {
|
|
|
4355
4355
|
* @see {@link cookieJar} - Access the underlying RezoCookieJar for more control
|
|
4356
4356
|
*/
|
|
4357
4357
|
clearCookies(): void;
|
|
4358
|
+
/**
|
|
4359
|
+
* Convert a Rezo request configuration to a cURL command string.
|
|
4360
|
+
*
|
|
4361
|
+
* Generates a valid cURL command that can be executed in a terminal to
|
|
4362
|
+
* reproduce the same HTTP request. Useful for:
|
|
4363
|
+
* - Debugging and sharing requests
|
|
4364
|
+
* - Documentation and examples
|
|
4365
|
+
* - Testing requests outside of Node.js
|
|
4366
|
+
* - Exporting requests to other tools
|
|
4367
|
+
*
|
|
4368
|
+
* @param config - Request configuration object
|
|
4369
|
+
* @returns A cURL command string
|
|
4370
|
+
*
|
|
4371
|
+
* @example
|
|
4372
|
+
* ```typescript
|
|
4373
|
+
* const curl = Rezo.toCurl({
|
|
4374
|
+
* url: 'https://api.example.com/users',
|
|
4375
|
+
* method: 'POST',
|
|
4376
|
+
* headers: { 'Content-Type': 'application/json' },
|
|
4377
|
+
* body: { name: 'John', email: 'john@example.com' }
|
|
4378
|
+
* });
|
|
4379
|
+
* // Output: curl -X POST -H 'content-type: application/json' --data-raw '{"name":"John","email":"john@example.com"}' -L --compressed 'https://api.example.com/users'
|
|
4380
|
+
* ```
|
|
4381
|
+
*/
|
|
4382
|
+
static toCurl(config: RezoRequestConfig | RezoRequestOptions): string;
|
|
4383
|
+
/**
|
|
4384
|
+
* Parse a cURL command string into a Rezo request configuration.
|
|
4385
|
+
*
|
|
4386
|
+
* Converts a cURL command into a configuration object that can be
|
|
4387
|
+
* passed directly to Rezo request methods. Useful for:
|
|
4388
|
+
* - Importing requests from browser DevTools
|
|
4389
|
+
* - Converting curl examples from API documentation
|
|
4390
|
+
* - Migrating scripts from curl to Rezo
|
|
4391
|
+
*
|
|
4392
|
+
* Supports common cURL options:
|
|
4393
|
+
* - `-X, --request` - HTTP method
|
|
4394
|
+
* - `-H, --header` - Request headers
|
|
4395
|
+
* - `-d, --data, --data-raw, --data-binary` - Request body
|
|
4396
|
+
* - `-u, --user` - Basic authentication
|
|
4397
|
+
* - `-x, --proxy` - Proxy configuration
|
|
4398
|
+
* - `--socks5, --socks4` - SOCKS proxy
|
|
4399
|
+
* - `-L, --location` - Follow redirects
|
|
4400
|
+
* - `--max-redirs` - Maximum redirects
|
|
4401
|
+
* - `--max-time` - Request timeout
|
|
4402
|
+
* - `-k, --insecure` - Skip TLS verification
|
|
4403
|
+
* - `-A, --user-agent` - User agent header
|
|
4404
|
+
*
|
|
4405
|
+
* @param curlCommand - A cURL command string
|
|
4406
|
+
* @returns A request configuration object
|
|
4407
|
+
*
|
|
4408
|
+
* @example
|
|
4409
|
+
* ```typescript
|
|
4410
|
+
* // From browser DevTools "Copy as cURL"
|
|
4411
|
+
* const config = Rezo.fromCurl(`
|
|
4412
|
+
* curl 'https://api.example.com/data' \\
|
|
4413
|
+
* -H 'Authorization: Bearer token123' \\
|
|
4414
|
+
* -H 'Content-Type: application/json'
|
|
4415
|
+
* `);
|
|
4416
|
+
*
|
|
4417
|
+
* // Use with Rezo
|
|
4418
|
+
* const rezo = new Rezo();
|
|
4419
|
+
* const response = await rezo.request(config);
|
|
4420
|
+
* ```
|
|
4421
|
+
*/
|
|
4422
|
+
static fromCurl(curlCommand: string): RezoRequestOptions;
|
|
4358
4423
|
}
|
|
4359
4424
|
/**
|
|
4360
4425
|
* Extended Rezo instance with Axios-compatible static helpers.
|
|
@@ -4355,6 +4355,71 @@ export declare class Rezo {
|
|
|
4355
4355
|
* @see {@link cookieJar} - Access the underlying RezoCookieJar for more control
|
|
4356
4356
|
*/
|
|
4357
4357
|
clearCookies(): void;
|
|
4358
|
+
/**
|
|
4359
|
+
* Convert a Rezo request configuration to a cURL command string.
|
|
4360
|
+
*
|
|
4361
|
+
* Generates a valid cURL command that can be executed in a terminal to
|
|
4362
|
+
* reproduce the same HTTP request. Useful for:
|
|
4363
|
+
* - Debugging and sharing requests
|
|
4364
|
+
* - Documentation and examples
|
|
4365
|
+
* - Testing requests outside of Node.js
|
|
4366
|
+
* - Exporting requests to other tools
|
|
4367
|
+
*
|
|
4368
|
+
* @param config - Request configuration object
|
|
4369
|
+
* @returns A cURL command string
|
|
4370
|
+
*
|
|
4371
|
+
* @example
|
|
4372
|
+
* ```typescript
|
|
4373
|
+
* const curl = Rezo.toCurl({
|
|
4374
|
+
* url: 'https://api.example.com/users',
|
|
4375
|
+
* method: 'POST',
|
|
4376
|
+
* headers: { 'Content-Type': 'application/json' },
|
|
4377
|
+
* body: { name: 'John', email: 'john@example.com' }
|
|
4378
|
+
* });
|
|
4379
|
+
* // Output: curl -X POST -H 'content-type: application/json' --data-raw '{"name":"John","email":"john@example.com"}' -L --compressed 'https://api.example.com/users'
|
|
4380
|
+
* ```
|
|
4381
|
+
*/
|
|
4382
|
+
static toCurl(config: RezoRequestConfig | RezoRequestOptions): string;
|
|
4383
|
+
/**
|
|
4384
|
+
* Parse a cURL command string into a Rezo request configuration.
|
|
4385
|
+
*
|
|
4386
|
+
* Converts a cURL command into a configuration object that can be
|
|
4387
|
+
* passed directly to Rezo request methods. Useful for:
|
|
4388
|
+
* - Importing requests from browser DevTools
|
|
4389
|
+
* - Converting curl examples from API documentation
|
|
4390
|
+
* - Migrating scripts from curl to Rezo
|
|
4391
|
+
*
|
|
4392
|
+
* Supports common cURL options:
|
|
4393
|
+
* - `-X, --request` - HTTP method
|
|
4394
|
+
* - `-H, --header` - Request headers
|
|
4395
|
+
* - `-d, --data, --data-raw, --data-binary` - Request body
|
|
4396
|
+
* - `-u, --user` - Basic authentication
|
|
4397
|
+
* - `-x, --proxy` - Proxy configuration
|
|
4398
|
+
* - `--socks5, --socks4` - SOCKS proxy
|
|
4399
|
+
* - `-L, --location` - Follow redirects
|
|
4400
|
+
* - `--max-redirs` - Maximum redirects
|
|
4401
|
+
* - `--max-time` - Request timeout
|
|
4402
|
+
* - `-k, --insecure` - Skip TLS verification
|
|
4403
|
+
* - `-A, --user-agent` - User agent header
|
|
4404
|
+
*
|
|
4405
|
+
* @param curlCommand - A cURL command string
|
|
4406
|
+
* @returns A request configuration object
|
|
4407
|
+
*
|
|
4408
|
+
* @example
|
|
4409
|
+
* ```typescript
|
|
4410
|
+
* // From browser DevTools "Copy as cURL"
|
|
4411
|
+
* const config = Rezo.fromCurl(`
|
|
4412
|
+
* curl 'https://api.example.com/data' \\
|
|
4413
|
+
* -H 'Authorization: Bearer token123' \\
|
|
4414
|
+
* -H 'Content-Type: application/json'
|
|
4415
|
+
* `);
|
|
4416
|
+
*
|
|
4417
|
+
* // Use with Rezo
|
|
4418
|
+
* const rezo = new Rezo();
|
|
4419
|
+
* const response = await rezo.request(config);
|
|
4420
|
+
* ```
|
|
4421
|
+
*/
|
|
4422
|
+
static fromCurl(curlCommand: string): RezoRequestOptions;
|
|
4358
4423
|
}
|
|
4359
4424
|
/**
|
|
4360
4425
|
* Extended Rezo instance with Axios-compatible static helpers.
|
|
@@ -4355,6 +4355,71 @@ export declare class Rezo {
|
|
|
4355
4355
|
* @see {@link cookieJar} - Access the underlying RezoCookieJar for more control
|
|
4356
4356
|
*/
|
|
4357
4357
|
clearCookies(): void;
|
|
4358
|
+
/**
|
|
4359
|
+
* Convert a Rezo request configuration to a cURL command string.
|
|
4360
|
+
*
|
|
4361
|
+
* Generates a valid cURL command that can be executed in a terminal to
|
|
4362
|
+
* reproduce the same HTTP request. Useful for:
|
|
4363
|
+
* - Debugging and sharing requests
|
|
4364
|
+
* - Documentation and examples
|
|
4365
|
+
* - Testing requests outside of Node.js
|
|
4366
|
+
* - Exporting requests to other tools
|
|
4367
|
+
*
|
|
4368
|
+
* @param config - Request configuration object
|
|
4369
|
+
* @returns A cURL command string
|
|
4370
|
+
*
|
|
4371
|
+
* @example
|
|
4372
|
+
* ```typescript
|
|
4373
|
+
* const curl = Rezo.toCurl({
|
|
4374
|
+
* url: 'https://api.example.com/users',
|
|
4375
|
+
* method: 'POST',
|
|
4376
|
+
* headers: { 'Content-Type': 'application/json' },
|
|
4377
|
+
* body: { name: 'John', email: 'john@example.com' }
|
|
4378
|
+
* });
|
|
4379
|
+
* // Output: curl -X POST -H 'content-type: application/json' --data-raw '{"name":"John","email":"john@example.com"}' -L --compressed 'https://api.example.com/users'
|
|
4380
|
+
* ```
|
|
4381
|
+
*/
|
|
4382
|
+
static toCurl(config: RezoRequestConfig | RezoRequestOptions): string;
|
|
4383
|
+
/**
|
|
4384
|
+
* Parse a cURL command string into a Rezo request configuration.
|
|
4385
|
+
*
|
|
4386
|
+
* Converts a cURL command into a configuration object that can be
|
|
4387
|
+
* passed directly to Rezo request methods. Useful for:
|
|
4388
|
+
* - Importing requests from browser DevTools
|
|
4389
|
+
* - Converting curl examples from API documentation
|
|
4390
|
+
* - Migrating scripts from curl to Rezo
|
|
4391
|
+
*
|
|
4392
|
+
* Supports common cURL options:
|
|
4393
|
+
* - `-X, --request` - HTTP method
|
|
4394
|
+
* - `-H, --header` - Request headers
|
|
4395
|
+
* - `-d, --data, --data-raw, --data-binary` - Request body
|
|
4396
|
+
* - `-u, --user` - Basic authentication
|
|
4397
|
+
* - `-x, --proxy` - Proxy configuration
|
|
4398
|
+
* - `--socks5, --socks4` - SOCKS proxy
|
|
4399
|
+
* - `-L, --location` - Follow redirects
|
|
4400
|
+
* - `--max-redirs` - Maximum redirects
|
|
4401
|
+
* - `--max-time` - Request timeout
|
|
4402
|
+
* - `-k, --insecure` - Skip TLS verification
|
|
4403
|
+
* - `-A, --user-agent` - User agent header
|
|
4404
|
+
*
|
|
4405
|
+
* @param curlCommand - A cURL command string
|
|
4406
|
+
* @returns A request configuration object
|
|
4407
|
+
*
|
|
4408
|
+
* @example
|
|
4409
|
+
* ```typescript
|
|
4410
|
+
* // From browser DevTools "Copy as cURL"
|
|
4411
|
+
* const config = Rezo.fromCurl(`
|
|
4412
|
+
* curl 'https://api.example.com/data' \\
|
|
4413
|
+
* -H 'Authorization: Bearer token123' \\
|
|
4414
|
+
* -H 'Content-Type: application/json'
|
|
4415
|
+
* `);
|
|
4416
|
+
*
|
|
4417
|
+
* // Use with Rezo
|
|
4418
|
+
* const rezo = new Rezo();
|
|
4419
|
+
* const response = await rezo.request(config);
|
|
4420
|
+
* ```
|
|
4421
|
+
*/
|
|
4422
|
+
static fromCurl(curlCommand: string): RezoRequestOptions;
|
|
4358
4423
|
}
|
|
4359
4424
|
/**
|
|
4360
4425
|
* Extended Rezo instance with Axios-compatible static helpers.
|