rezo 1.0.41 → 1.0.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/dist/adapters/curl.cjs +143 -32
  2. package/dist/adapters/curl.js +143 -32
  3. package/dist/adapters/entries/curl.d.ts +65 -0
  4. package/dist/adapters/entries/fetch.d.ts +65 -0
  5. package/dist/adapters/entries/http.d.ts +65 -0
  6. package/dist/adapters/entries/http2.d.ts +65 -0
  7. package/dist/adapters/entries/react-native.d.ts +65 -0
  8. package/dist/adapters/entries/xhr.d.ts +65 -0
  9. package/dist/adapters/fetch.cjs +98 -12
  10. package/dist/adapters/fetch.js +98 -12
  11. package/dist/adapters/http.cjs +26 -14
  12. package/dist/adapters/http.js +26 -14
  13. package/dist/adapters/http2.cjs +756 -227
  14. package/dist/adapters/http2.js +756 -227
  15. package/dist/adapters/index.cjs +6 -6
  16. package/dist/adapters/xhr.cjs +94 -2
  17. package/dist/adapters/xhr.js +94 -2
  18. package/dist/cache/dns-cache.cjs +5 -3
  19. package/dist/cache/dns-cache.js +5 -3
  20. package/dist/cache/file-cacher.cjs +7 -1
  21. package/dist/cache/file-cacher.js +7 -1
  22. package/dist/cache/index.cjs +15 -13
  23. package/dist/cache/index.js +1 -0
  24. package/dist/cache/navigation-history.cjs +298 -0
  25. package/dist/cache/navigation-history.js +296 -0
  26. package/dist/cache/url-store.cjs +7 -1
  27. package/dist/cache/url-store.js +7 -1
  28. package/dist/core/rezo.cjs +7 -0
  29. package/dist/core/rezo.js +7 -0
  30. package/dist/crawler.d.ts +196 -11
  31. package/dist/entries/crawler.cjs +5 -5
  32. package/dist/index.cjs +27 -24
  33. package/dist/index.d.ts +73 -0
  34. package/dist/index.js +1 -0
  35. package/dist/internal/agents/base.cjs +113 -0
  36. package/dist/internal/agents/base.js +110 -0
  37. package/dist/internal/agents/http-proxy.cjs +89 -0
  38. package/dist/internal/agents/http-proxy.js +86 -0
  39. package/dist/internal/agents/https-proxy.cjs +176 -0
  40. package/dist/internal/agents/https-proxy.js +173 -0
  41. package/dist/internal/agents/index.cjs +10 -0
  42. package/dist/internal/agents/index.js +5 -0
  43. package/dist/internal/agents/socks-client.cjs +571 -0
  44. package/dist/internal/agents/socks-client.js +567 -0
  45. package/dist/internal/agents/socks-proxy.cjs +75 -0
  46. package/dist/internal/agents/socks-proxy.js +72 -0
  47. package/dist/platform/browser.d.ts +65 -0
  48. package/dist/platform/bun.d.ts +65 -0
  49. package/dist/platform/deno.d.ts +65 -0
  50. package/dist/platform/node.d.ts +65 -0
  51. package/dist/platform/react-native.d.ts +65 -0
  52. package/dist/platform/worker.d.ts +65 -0
  53. package/dist/plugin/crawler-options.cjs +1 -1
  54. package/dist/plugin/crawler-options.js +1 -1
  55. package/dist/plugin/crawler.cjs +192 -1
  56. package/dist/plugin/crawler.js +192 -1
  57. package/dist/plugin/index.cjs +36 -36
  58. package/dist/proxy/index.cjs +18 -16
  59. package/dist/proxy/index.js +17 -12
  60. package/dist/queue/index.cjs +8 -8
  61. package/dist/responses/buildError.cjs +11 -2
  62. package/dist/responses/buildError.js +11 -2
  63. package/dist/responses/universal/index.cjs +11 -11
  64. package/dist/utils/agent-pool.cjs +1 -17
  65. package/dist/utils/agent-pool.js +1 -17
  66. package/dist/utils/curl.cjs +317 -0
  67. package/dist/utils/curl.js +314 -0
  68. package/package.json +1 -1
@@ -195,7 +195,7 @@ function buildUrlTree(config, finalUrl) {
195
195
  }
196
196
  return urls.length > 0 ? urls : [finalUrl];
197
197
  }
198
- async function updateCookies(config, cookieStrings, url) {
198
+ async function updateCookies(config, cookieStrings, url, rootJar) {
199
199
  if (!cookieStrings || cookieStrings.length === 0)
200
200
  return;
201
201
  const tempJar = new RezoCookieJar;
@@ -233,8 +233,9 @@ async function updateCookies(config, cookieStrings, url) {
233
233
  acceptedCookies.push(...parsedCookies.array);
234
234
  }
235
235
  const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
236
- if (!config.disableCookieJar && config.cookieJar) {
237
- config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
236
+ const jarToUpdate = rootJar || config.cookieJar;
237
+ if (!config.disableCookieJar && jarToUpdate) {
238
+ jarToUpdate.setCookiesSync(acceptedCookieStrings, url);
238
239
  }
239
240
  if (config.useCookies) {
240
241
  const existingArray = config.responseCookies?.array || [];
@@ -1484,14 +1485,15 @@ class CurlCommandBuilder {
1484
1485
  case "socks5h":
1485
1486
  this.addArg("--socks5", hostPort);
1486
1487
  if (auth) {
1487
- this.addArg("--socks5-user", auth);
1488
+ this.addArg("--socks5-basic");
1489
+ this.addArg("--proxy-user", auth);
1488
1490
  }
1489
1491
  break;
1490
1492
  case "socks4":
1491
1493
  case "socks4a":
1492
1494
  this.addArg("--socks4", hostPort);
1493
1495
  if (username) {
1494
- this.addArg("--socks4-user", username);
1496
+ this.addArg("--proxy-user", username);
1495
1497
  }
1496
1498
  break;
1497
1499
  case "http":
@@ -1634,6 +1636,7 @@ class CurlCommandBuilder {
1634
1636
  "local_ip:%{local_ip}",
1635
1637
  "local_port:%{local_port}",
1636
1638
  "redirect_url:%{redirect_url}",
1639
+ "url_effective:%{url_effective}",
1637
1640
  "ssl_verify_result:%{ssl_verify_result}",
1638
1641
  "content_type:%{content_type}",
1639
1642
  "---CURL_STATS_END---"
@@ -1655,27 +1658,68 @@ class CurlResponseParser {
1655
1658
  body = stdout.slice(0, statsStart);
1656
1659
  for (const line of statsSection.split(`
1657
1660
  `)) {
1658
- const [key, value] = line.split(":");
1659
- if (key && value !== undefined) {
1660
- stats[key.trim()] = value.trim();
1661
+ const colonIndex = line.indexOf(":");
1662
+ if (colonIndex !== -1) {
1663
+ const key = line.slice(0, colonIndex).trim();
1664
+ const value = line.slice(colonIndex + 1).trim();
1665
+ if (key && value !== undefined) {
1666
+ stats[key] = value;
1667
+ }
1661
1668
  }
1662
1669
  }
1663
1670
  }
1664
- const headerBodySplit = body.indexOf(`\r
1665
- \r
1666
- `);
1667
- let headerSection = "";
1668
- let responseBody = body;
1669
- if (headerBodySplit !== -1) {
1670
- headerSection = body.slice(0, headerBodySplit);
1671
- responseBody = body.slice(headerBodySplit + 4);
1671
+ const allResponses = this.parseAllHttpResponses(body);
1672
+ const finalResponse = allResponses[allResponses.length - 1] || { headers: "", body };
1673
+ let headerSection = finalResponse.headers;
1674
+ let responseBody = finalResponse.body;
1675
+ const allSetCookies = [];
1676
+ for (const resp of allResponses) {
1677
+ const respHeaders = this.parseHeaders(resp.headers);
1678
+ const setCookieHeader = respHeaders["set-cookie"];
1679
+ if (setCookieHeader) {
1680
+ if (Array.isArray(setCookieHeader)) {
1681
+ allSetCookies.push(...setCookieHeader);
1682
+ } else {
1683
+ allSetCookies.push(setCookieHeader);
1684
+ }
1685
+ }
1686
+ }
1687
+ if (allResponses.length > 1) {
1688
+ config.redirectHistory = config.redirectHistory || [];
1689
+ let currentUrl = config.url || "";
1690
+ for (let i = 0;i < allResponses.length - 1; i++) {
1691
+ const resp = allResponses[i];
1692
+ const respHeaders = this.parseHeaders(resp.headers);
1693
+ const statusMatch = resp.headers.match(/HTTP\/[\d.]+ (\d+)/);
1694
+ const statusCode = statusMatch ? parseInt(statusMatch[1]) : 0;
1695
+ const locationHeader = respHeaders["location"] || "";
1696
+ config.redirectHistory.push({
1697
+ url: currentUrl,
1698
+ statusCode,
1699
+ statusText: this.getStatusText(statusCode),
1700
+ headers: new RezoHeaders(respHeaders),
1701
+ method: config.method || "GET",
1702
+ cookies: [],
1703
+ duration: 0,
1704
+ request: originalRequest
1705
+ });
1706
+ if (locationHeader) {
1707
+ try {
1708
+ currentUrl = new URL(locationHeader, currentUrl).toString();
1709
+ } catch {
1710
+ currentUrl = locationHeader;
1711
+ }
1712
+ }
1713
+ }
1714
+ config.redirectCount = allResponses.length - 1;
1672
1715
  }
1673
1716
  const statusMatch = headerSection.match(/HTTP\/[\d.]+ (\d+)/);
1674
1717
  const status = statusMatch ? parseInt(statusMatch[1]) : parseInt(stats["http_code"]) || 200;
1675
1718
  const statusText = this.getStatusText(status);
1676
1719
  const headers = this.parseHeaders(headerSection);
1677
1720
  const rezoHeaders = new RezoHeaders(headers);
1678
- const responseCookies = this.parseCookies(headers);
1721
+ rezoHeaders.delete("set-cookie");
1722
+ const responseCookies = this.parseCookiesFromStrings(allSetCookies, config.url || "");
1679
1723
  const mergedCookieArray = mergeRequestAndResponseCookies(config.requestCookies, responseCookies.array);
1680
1724
  let cookies;
1681
1725
  if (mergedCookieArray.length > 0) {
@@ -1714,7 +1758,7 @@ class CurlResponseParser {
1714
1758
  const isSecure = config.url?.startsWith("https") || false;
1715
1759
  config.adapterUsed = "curl";
1716
1760
  config.isSecure = isSecure;
1717
- config.finalUrl = stats["redirect_url"] || config.url || "";
1761
+ config.finalUrl = stats["url_effective"] || config.url || "";
1718
1762
  if (!config.network) {
1719
1763
  config.network = {};
1720
1764
  }
@@ -1728,7 +1772,7 @@ class CurlResponseParser {
1728
1772
  config.transfer.bodySize = responseBody.length;
1729
1773
  config.transfer.headerSize = headerSection.length;
1730
1774
  config.responseCookies = cookies;
1731
- const finalUrl = stats["redirect_url"] || config.url || "";
1775
+ const finalUrl = stats["url_effective"] || config.url || "";
1732
1776
  const urls = buildUrlTree(config, finalUrl);
1733
1777
  const timingDurations = getTimingDurations(config);
1734
1778
  debugLog.responseHeaders(config, headers);
@@ -1812,6 +1856,55 @@ class CurlResponseParser {
1812
1856
  setCookiesString: cookieArray
1813
1857
  };
1814
1858
  }
1859
+ static parseAllHttpResponses(output) {
1860
+ const responses = [];
1861
+ const httpPattern = /HTTP\/[\d.]+ \d+/g;
1862
+ const matches = [];
1863
+ let match;
1864
+ while ((match = httpPattern.exec(output)) !== null) {
1865
+ matches.push(match.index);
1866
+ }
1867
+ if (matches.length === 0) {
1868
+ return [{ headers: "", body: output }];
1869
+ }
1870
+ for (let i = 0;i < matches.length; i++) {
1871
+ const start = matches[i];
1872
+ const end = i < matches.length - 1 ? matches[i + 1] : output.length;
1873
+ const segment = output.slice(start, end);
1874
+ const separator = segment.indexOf(`\r
1875
+ \r
1876
+ `);
1877
+ if (separator !== -1) {
1878
+ const headers = segment.slice(0, separator);
1879
+ const body = segment.slice(separator + 4);
1880
+ responses.push({ headers, body });
1881
+ } else {
1882
+ responses.push({ headers: segment.trim(), body: "" });
1883
+ }
1884
+ }
1885
+ if (responses.length > 1) {
1886
+ const lastBody = responses[responses.length - 1].body;
1887
+ for (let i = 0;i < responses.length - 1; i++) {
1888
+ responses[i].body = "";
1889
+ }
1890
+ responses[responses.length - 1].body = lastBody;
1891
+ }
1892
+ return responses;
1893
+ }
1894
+ static parseCookiesFromStrings(setCookieStrings, url) {
1895
+ if (setCookieStrings.length === 0) {
1896
+ return {
1897
+ array: [],
1898
+ serialized: [],
1899
+ netscape: "",
1900
+ string: "",
1901
+ setCookiesString: []
1902
+ };
1903
+ }
1904
+ const jar = new RezoCookieJar;
1905
+ jar.setCookiesSync(setCookieStrings, url);
1906
+ return jar.cookies();
1907
+ }
1815
1908
  }
1816
1909
 
1817
1910
  class CurlExecutor {
@@ -2148,6 +2241,9 @@ async function executeRequest(options, defaultOptions, jar) {
2148
2241
  }
2149
2242
  }
2150
2243
  }
2244
+ if (!config.requestId) {
2245
+ config.requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
2246
+ }
2151
2247
  debugLog.requestStart(config, requestUrl, method);
2152
2248
  let streamResponse;
2153
2249
  let downloadResponse;
@@ -2201,6 +2297,11 @@ async function executeRequest(options, defaultOptions, jar) {
2201
2297
  const duration = perform.now();
2202
2298
  debugLog.response(config, response.status, response.statusText, duration);
2203
2299
  debugLog.cookies(config, response.cookies?.array?.length || 0);
2300
+ if (response.cookies?.setCookiesString?.length > 0 && jar) {
2301
+ try {
2302
+ jar.setCookiesSync(response.cookies.setCookiesString, response.finalUrl || requestUrl);
2303
+ } catch (e) {}
2304
+ }
2204
2305
  if (cache) {
2205
2306
  if (response.status === 304 && cachedEntry) {
2206
2307
  const responseHeaders = response.headers instanceof RezoHeaders ? Object.fromEntries(response.headers.entries()) : response.headers;
@@ -2220,21 +2321,31 @@ async function executeRequest(options, defaultOptions, jar) {
2220
2321
  if (config.retry) {
2221
2322
  const maxRetries = config.retry.maxRetries || 0;
2222
2323
  const statusCodes = config.retry.statusCodes || [408, 429, 500, 502, 503, 504];
2223
- if (config.retryAttempts < maxRetries && statusCodes.includes(response.status)) {
2224
- const retryDelay = config.retry.retryDelay || 0;
2225
- const incrementDelay = config.retry.incrementDelay || false;
2226
- const delay = incrementDelay ? retryDelay * (config.retryAttempts + 1) : retryDelay;
2227
- debugLog.retry(config, config.retryAttempts + 1, maxRetries, response.status, delay);
2228
- if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
2229
- for (const hook of config.hooks.beforeRetry) {
2230
- await hook(config, httpError, config.retryAttempts + 1);
2324
+ if (statusCodes.includes(response.status)) {
2325
+ if (config.retryAttempts < maxRetries) {
2326
+ config.retryAttempts++;
2327
+ if (!config.errors)
2328
+ config.errors = [];
2329
+ config.errors.push({
2330
+ attempt: config.retryAttempts,
2331
+ error: httpError,
2332
+ duration: perform.now()
2333
+ });
2334
+ perform.reset();
2335
+ const retryDelay = config.retry.retryDelay || 0;
2336
+ const incrementDelay = config.retry.incrementDelay || false;
2337
+ const delay = incrementDelay ? retryDelay * config.retryAttempts : retryDelay;
2338
+ debugLog.retry(config, config.retryAttempts, maxRetries, response.status, delay);
2339
+ if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
2340
+ for (const hook of config.hooks.beforeRetry) {
2341
+ await hook(config, httpError, config.retryAttempts);
2342
+ }
2231
2343
  }
2344
+ if (delay > 0) {
2345
+ await new Promise((resolve) => setTimeout(resolve, delay));
2346
+ }
2347
+ return executeRequest(options, defaultOptions, jar);
2232
2348
  }
2233
- if (delay > 0) {
2234
- await new Promise((resolve) => setTimeout(resolve, delay));
2235
- }
2236
- config.retryAttempts++;
2237
- return executeRequest(options, defaultOptions, jar);
2238
2349
  }
2239
2350
  debugLog.maxRetries(config, maxRetries);
2240
2351
  }
@@ -195,7 +195,7 @@ function buildUrlTree(config, finalUrl) {
195
195
  }
196
196
  return urls.length > 0 ? urls : [finalUrl];
197
197
  }
198
- async function updateCookies(config, cookieStrings, url) {
198
+ async function updateCookies(config, cookieStrings, url, rootJar) {
199
199
  if (!cookieStrings || cookieStrings.length === 0)
200
200
  return;
201
201
  const tempJar = new RezoCookieJar;
@@ -233,8 +233,9 @@ async function updateCookies(config, cookieStrings, url) {
233
233
  acceptedCookies.push(...parsedCookies.array);
234
234
  }
235
235
  const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
236
- if (!config.disableCookieJar && config.cookieJar) {
237
- config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
236
+ const jarToUpdate = rootJar || config.cookieJar;
237
+ if (!config.disableCookieJar && jarToUpdate) {
238
+ jarToUpdate.setCookiesSync(acceptedCookieStrings, url);
238
239
  }
239
240
  if (config.useCookies) {
240
241
  const existingArray = config.responseCookies?.array || [];
@@ -1484,14 +1485,15 @@ class CurlCommandBuilder {
1484
1485
  case "socks5h":
1485
1486
  this.addArg("--socks5", hostPort);
1486
1487
  if (auth) {
1487
- this.addArg("--socks5-user", auth);
1488
+ this.addArg("--socks5-basic");
1489
+ this.addArg("--proxy-user", auth);
1488
1490
  }
1489
1491
  break;
1490
1492
  case "socks4":
1491
1493
  case "socks4a":
1492
1494
  this.addArg("--socks4", hostPort);
1493
1495
  if (username) {
1494
- this.addArg("--socks4-user", username);
1496
+ this.addArg("--proxy-user", username);
1495
1497
  }
1496
1498
  break;
1497
1499
  case "http":
@@ -1634,6 +1636,7 @@ class CurlCommandBuilder {
1634
1636
  "local_ip:%{local_ip}",
1635
1637
  "local_port:%{local_port}",
1636
1638
  "redirect_url:%{redirect_url}",
1639
+ "url_effective:%{url_effective}",
1637
1640
  "ssl_verify_result:%{ssl_verify_result}",
1638
1641
  "content_type:%{content_type}",
1639
1642
  "---CURL_STATS_END---"
@@ -1655,27 +1658,68 @@ class CurlResponseParser {
1655
1658
  body = stdout.slice(0, statsStart);
1656
1659
  for (const line of statsSection.split(`
1657
1660
  `)) {
1658
- const [key, value] = line.split(":");
1659
- if (key && value !== undefined) {
1660
- stats[key.trim()] = value.trim();
1661
+ const colonIndex = line.indexOf(":");
1662
+ if (colonIndex !== -1) {
1663
+ const key = line.slice(0, colonIndex).trim();
1664
+ const value = line.slice(colonIndex + 1).trim();
1665
+ if (key && value !== undefined) {
1666
+ stats[key] = value;
1667
+ }
1661
1668
  }
1662
1669
  }
1663
1670
  }
1664
- const headerBodySplit = body.indexOf(`\r
1665
- \r
1666
- `);
1667
- let headerSection = "";
1668
- let responseBody = body;
1669
- if (headerBodySplit !== -1) {
1670
- headerSection = body.slice(0, headerBodySplit);
1671
- responseBody = body.slice(headerBodySplit + 4);
1671
+ const allResponses = this.parseAllHttpResponses(body);
1672
+ const finalResponse = allResponses[allResponses.length - 1] || { headers: "", body };
1673
+ let headerSection = finalResponse.headers;
1674
+ let responseBody = finalResponse.body;
1675
+ const allSetCookies = [];
1676
+ for (const resp of allResponses) {
1677
+ const respHeaders = this.parseHeaders(resp.headers);
1678
+ const setCookieHeader = respHeaders["set-cookie"];
1679
+ if (setCookieHeader) {
1680
+ if (Array.isArray(setCookieHeader)) {
1681
+ allSetCookies.push(...setCookieHeader);
1682
+ } else {
1683
+ allSetCookies.push(setCookieHeader);
1684
+ }
1685
+ }
1686
+ }
1687
+ if (allResponses.length > 1) {
1688
+ config.redirectHistory = config.redirectHistory || [];
1689
+ let currentUrl = config.url || "";
1690
+ for (let i = 0;i < allResponses.length - 1; i++) {
1691
+ const resp = allResponses[i];
1692
+ const respHeaders = this.parseHeaders(resp.headers);
1693
+ const statusMatch = resp.headers.match(/HTTP\/[\d.]+ (\d+)/);
1694
+ const statusCode = statusMatch ? parseInt(statusMatch[1]) : 0;
1695
+ const locationHeader = respHeaders["location"] || "";
1696
+ config.redirectHistory.push({
1697
+ url: currentUrl,
1698
+ statusCode,
1699
+ statusText: this.getStatusText(statusCode),
1700
+ headers: new RezoHeaders(respHeaders),
1701
+ method: config.method || "GET",
1702
+ cookies: [],
1703
+ duration: 0,
1704
+ request: originalRequest
1705
+ });
1706
+ if (locationHeader) {
1707
+ try {
1708
+ currentUrl = new URL(locationHeader, currentUrl).toString();
1709
+ } catch {
1710
+ currentUrl = locationHeader;
1711
+ }
1712
+ }
1713
+ }
1714
+ config.redirectCount = allResponses.length - 1;
1672
1715
  }
1673
1716
  const statusMatch = headerSection.match(/HTTP\/[\d.]+ (\d+)/);
1674
1717
  const status = statusMatch ? parseInt(statusMatch[1]) : parseInt(stats["http_code"]) || 200;
1675
1718
  const statusText = this.getStatusText(status);
1676
1719
  const headers = this.parseHeaders(headerSection);
1677
1720
  const rezoHeaders = new RezoHeaders(headers);
1678
- const responseCookies = this.parseCookies(headers);
1721
+ rezoHeaders.delete("set-cookie");
1722
+ const responseCookies = this.parseCookiesFromStrings(allSetCookies, config.url || "");
1679
1723
  const mergedCookieArray = mergeRequestAndResponseCookies(config.requestCookies, responseCookies.array);
1680
1724
  let cookies;
1681
1725
  if (mergedCookieArray.length > 0) {
@@ -1714,7 +1758,7 @@ class CurlResponseParser {
1714
1758
  const isSecure = config.url?.startsWith("https") || false;
1715
1759
  config.adapterUsed = "curl";
1716
1760
  config.isSecure = isSecure;
1717
- config.finalUrl = stats["redirect_url"] || config.url || "";
1761
+ config.finalUrl = stats["url_effective"] || config.url || "";
1718
1762
  if (!config.network) {
1719
1763
  config.network = {};
1720
1764
  }
@@ -1728,7 +1772,7 @@ class CurlResponseParser {
1728
1772
  config.transfer.bodySize = responseBody.length;
1729
1773
  config.transfer.headerSize = headerSection.length;
1730
1774
  config.responseCookies = cookies;
1731
- const finalUrl = stats["redirect_url"] || config.url || "";
1775
+ const finalUrl = stats["url_effective"] || config.url || "";
1732
1776
  const urls = buildUrlTree(config, finalUrl);
1733
1777
  const timingDurations = getTimingDurations(config);
1734
1778
  debugLog.responseHeaders(config, headers);
@@ -1812,6 +1856,55 @@ class CurlResponseParser {
1812
1856
  setCookiesString: cookieArray
1813
1857
  };
1814
1858
  }
1859
+ static parseAllHttpResponses(output) {
1860
+ const responses = [];
1861
+ const httpPattern = /HTTP\/[\d.]+ \d+/g;
1862
+ const matches = [];
1863
+ let match;
1864
+ while ((match = httpPattern.exec(output)) !== null) {
1865
+ matches.push(match.index);
1866
+ }
1867
+ if (matches.length === 0) {
1868
+ return [{ headers: "", body: output }];
1869
+ }
1870
+ for (let i = 0;i < matches.length; i++) {
1871
+ const start = matches[i];
1872
+ const end = i < matches.length - 1 ? matches[i + 1] : output.length;
1873
+ const segment = output.slice(start, end);
1874
+ const separator = segment.indexOf(`\r
1875
+ \r
1876
+ `);
1877
+ if (separator !== -1) {
1878
+ const headers = segment.slice(0, separator);
1879
+ const body = segment.slice(separator + 4);
1880
+ responses.push({ headers, body });
1881
+ } else {
1882
+ responses.push({ headers: segment.trim(), body: "" });
1883
+ }
1884
+ }
1885
+ if (responses.length > 1) {
1886
+ const lastBody = responses[responses.length - 1].body;
1887
+ for (let i = 0;i < responses.length - 1; i++) {
1888
+ responses[i].body = "";
1889
+ }
1890
+ responses[responses.length - 1].body = lastBody;
1891
+ }
1892
+ return responses;
1893
+ }
1894
+ static parseCookiesFromStrings(setCookieStrings, url) {
1895
+ if (setCookieStrings.length === 0) {
1896
+ return {
1897
+ array: [],
1898
+ serialized: [],
1899
+ netscape: "",
1900
+ string: "",
1901
+ setCookiesString: []
1902
+ };
1903
+ }
1904
+ const jar = new RezoCookieJar;
1905
+ jar.setCookiesSync(setCookieStrings, url);
1906
+ return jar.cookies();
1907
+ }
1815
1908
  }
1816
1909
 
1817
1910
  class CurlExecutor {
@@ -2148,6 +2241,9 @@ export async function executeRequest(options, defaultOptions, jar) {
2148
2241
  }
2149
2242
  }
2150
2243
  }
2244
+ if (!config.requestId) {
2245
+ config.requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
2246
+ }
2151
2247
  debugLog.requestStart(config, requestUrl, method);
2152
2248
  let streamResponse;
2153
2249
  let downloadResponse;
@@ -2201,6 +2297,11 @@ export async function executeRequest(options, defaultOptions, jar) {
2201
2297
  const duration = perform.now();
2202
2298
  debugLog.response(config, response.status, response.statusText, duration);
2203
2299
  debugLog.cookies(config, response.cookies?.array?.length || 0);
2300
+ if (response.cookies?.setCookiesString?.length > 0 && jar) {
2301
+ try {
2302
+ jar.setCookiesSync(response.cookies.setCookiesString, response.finalUrl || requestUrl);
2303
+ } catch (e) {}
2304
+ }
2204
2305
  if (cache) {
2205
2306
  if (response.status === 304 && cachedEntry) {
2206
2307
  const responseHeaders = response.headers instanceof RezoHeaders ? Object.fromEntries(response.headers.entries()) : response.headers;
@@ -2220,21 +2321,31 @@ export async function executeRequest(options, defaultOptions, jar) {
2220
2321
  if (config.retry) {
2221
2322
  const maxRetries = config.retry.maxRetries || 0;
2222
2323
  const statusCodes = config.retry.statusCodes || [408, 429, 500, 502, 503, 504];
2223
- if (config.retryAttempts < maxRetries && statusCodes.includes(response.status)) {
2224
- const retryDelay = config.retry.retryDelay || 0;
2225
- const incrementDelay = config.retry.incrementDelay || false;
2226
- const delay = incrementDelay ? retryDelay * (config.retryAttempts + 1) : retryDelay;
2227
- debugLog.retry(config, config.retryAttempts + 1, maxRetries, response.status, delay);
2228
- if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
2229
- for (const hook of config.hooks.beforeRetry) {
2230
- await hook(config, httpError, config.retryAttempts + 1);
2324
+ if (statusCodes.includes(response.status)) {
2325
+ if (config.retryAttempts < maxRetries) {
2326
+ config.retryAttempts++;
2327
+ if (!config.errors)
2328
+ config.errors = [];
2329
+ config.errors.push({
2330
+ attempt: config.retryAttempts,
2331
+ error: httpError,
2332
+ duration: perform.now()
2333
+ });
2334
+ perform.reset();
2335
+ const retryDelay = config.retry.retryDelay || 0;
2336
+ const incrementDelay = config.retry.incrementDelay || false;
2337
+ const delay = incrementDelay ? retryDelay * config.retryAttempts : retryDelay;
2338
+ debugLog.retry(config, config.retryAttempts, maxRetries, response.status, delay);
2339
+ if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
2340
+ for (const hook of config.hooks.beforeRetry) {
2341
+ await hook(config, httpError, config.retryAttempts);
2342
+ }
2231
2343
  }
2344
+ if (delay > 0) {
2345
+ await new Promise((resolve) => setTimeout(resolve, delay));
2346
+ }
2347
+ return executeRequest(options, defaultOptions, jar);
2232
2348
  }
2233
- if (delay > 0) {
2234
- await new Promise((resolve) => setTimeout(resolve, delay));
2235
- }
2236
- config.retryAttempts++;
2237
- return executeRequest(options, defaultOptions, jar);
2238
2349
  }
2239
2350
  debugLog.maxRetries(config, maxRetries);
2240
2351
  }
@@ -4355,6 +4355,71 @@ export declare class Rezo {
4355
4355
  * @see {@link cookieJar} - Access the underlying RezoCookieJar for more control
4356
4356
  */
4357
4357
  clearCookies(): void;
4358
+ /**
4359
+ * Convert a Rezo request configuration to a cURL command string.
4360
+ *
4361
+ * Generates a valid cURL command that can be executed in a terminal to
4362
+ * reproduce the same HTTP request. Useful for:
4363
+ * - Debugging and sharing requests
4364
+ * - Documentation and examples
4365
+ * - Testing requests outside of Node.js
4366
+ * - Exporting requests to other tools
4367
+ *
4368
+ * @param config - Request configuration object
4369
+ * @returns A cURL command string
4370
+ *
4371
+ * @example
4372
+ * ```typescript
4373
+ * const curl = Rezo.toCurl({
4374
+ * url: 'https://api.example.com/users',
4375
+ * method: 'POST',
4376
+ * headers: { 'Content-Type': 'application/json' },
4377
+ * body: { name: 'John', email: 'john@example.com' }
4378
+ * });
4379
+ * // Output: curl -X POST -H 'content-type: application/json' --data-raw '{"name":"John","email":"john@example.com"}' -L --compressed 'https://api.example.com/users'
4380
+ * ```
4381
+ */
4382
+ static toCurl(config: RezoRequestConfig | RezoRequestOptions): string;
4383
+ /**
4384
+ * Parse a cURL command string into a Rezo request configuration.
4385
+ *
4386
+ * Converts a cURL command into a configuration object that can be
4387
+ * passed directly to Rezo request methods. Useful for:
4388
+ * - Importing requests from browser DevTools
4389
+ * - Converting curl examples from API documentation
4390
+ * - Migrating scripts from curl to Rezo
4391
+ *
4392
+ * Supports common cURL options:
4393
+ * - `-X, --request` - HTTP method
4394
+ * - `-H, --header` - Request headers
4395
+ * - `-d, --data, --data-raw, --data-binary` - Request body
4396
+ * - `-u, --user` - Basic authentication
4397
+ * - `-x, --proxy` - Proxy configuration
4398
+ * - `--socks5, --socks4` - SOCKS proxy
4399
+ * - `-L, --location` - Follow redirects
4400
+ * - `--max-redirs` - Maximum redirects
4401
+ * - `--max-time` - Request timeout
4402
+ * - `-k, --insecure` - Skip TLS verification
4403
+ * - `-A, --user-agent` - User agent header
4404
+ *
4405
+ * @param curlCommand - A cURL command string
4406
+ * @returns A request configuration object
4407
+ *
4408
+ * @example
4409
+ * ```typescript
4410
+ * // From browser DevTools "Copy as cURL"
4411
+ * const config = Rezo.fromCurl(`
4412
+ * curl 'https://api.example.com/data' \\
4413
+ * -H 'Authorization: Bearer token123' \\
4414
+ * -H 'Content-Type: application/json'
4415
+ * `);
4416
+ *
4417
+ * // Use with Rezo
4418
+ * const rezo = new Rezo();
4419
+ * const response = await rezo.request(config);
4420
+ * ```
4421
+ */
4422
+ static fromCurl(curlCommand: string): RezoRequestOptions;
4358
4423
  }
4359
4424
  /**
4360
4425
  * Extended Rezo instance with Axios-compatible static helpers.
@@ -4355,6 +4355,71 @@ export declare class Rezo {
4355
4355
  * @see {@link cookieJar} - Access the underlying RezoCookieJar for more control
4356
4356
  */
4357
4357
  clearCookies(): void;
4358
+ /**
4359
+ * Convert a Rezo request configuration to a cURL command string.
4360
+ *
4361
+ * Generates a valid cURL command that can be executed in a terminal to
4362
+ * reproduce the same HTTP request. Useful for:
4363
+ * - Debugging and sharing requests
4364
+ * - Documentation and examples
4365
+ * - Testing requests outside of Node.js
4366
+ * - Exporting requests to other tools
4367
+ *
4368
+ * @param config - Request configuration object
4369
+ * @returns A cURL command string
4370
+ *
4371
+ * @example
4372
+ * ```typescript
4373
+ * const curl = Rezo.toCurl({
4374
+ * url: 'https://api.example.com/users',
4375
+ * method: 'POST',
4376
+ * headers: { 'Content-Type': 'application/json' },
4377
+ * body: { name: 'John', email: 'john@example.com' }
4378
+ * });
4379
+ * // Output: curl -X POST -H 'content-type: application/json' --data-raw '{"name":"John","email":"john@example.com"}' -L --compressed 'https://api.example.com/users'
4380
+ * ```
4381
+ */
4382
+ static toCurl(config: RezoRequestConfig | RezoRequestOptions): string;
4383
+ /**
4384
+ * Parse a cURL command string into a Rezo request configuration.
4385
+ *
4386
+ * Converts a cURL command into a configuration object that can be
4387
+ * passed directly to Rezo request methods. Useful for:
4388
+ * - Importing requests from browser DevTools
4389
+ * - Converting curl examples from API documentation
4390
+ * - Migrating scripts from curl to Rezo
4391
+ *
4392
+ * Supports common cURL options:
4393
+ * - `-X, --request` - HTTP method
4394
+ * - `-H, --header` - Request headers
4395
+ * - `-d, --data, --data-raw, --data-binary` - Request body
4396
+ * - `-u, --user` - Basic authentication
4397
+ * - `-x, --proxy` - Proxy configuration
4398
+ * - `--socks5, --socks4` - SOCKS proxy
4399
+ * - `-L, --location` - Follow redirects
4400
+ * - `--max-redirs` - Maximum redirects
4401
+ * - `--max-time` - Request timeout
4402
+ * - `-k, --insecure` - Skip TLS verification
4403
+ * - `-A, --user-agent` - User agent header
4404
+ *
4405
+ * @param curlCommand - A cURL command string
4406
+ * @returns A request configuration object
4407
+ *
4408
+ * @example
4409
+ * ```typescript
4410
+ * // From browser DevTools "Copy as cURL"
4411
+ * const config = Rezo.fromCurl(`
4412
+ * curl 'https://api.example.com/data' \\
4413
+ * -H 'Authorization: Bearer token123' \\
4414
+ * -H 'Content-Type: application/json'
4415
+ * `);
4416
+ *
4417
+ * // Use with Rezo
4418
+ * const rezo = new Rezo();
4419
+ * const response = await rezo.request(config);
4420
+ * ```
4421
+ */
4422
+ static fromCurl(curlCommand: string): RezoRequestOptions;
4358
4423
  }
4359
4424
  /**
4360
4425
  * Extended Rezo instance with Axios-compatible static helpers.