rezo 1.0.139 → 1.0.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl-exit-code.cjs +66 -0
- package/dist/adapters/curl-exit-code.js +63 -0
- package/dist/adapters/curl-redirect-hop.cjs +14 -0
- package/dist/adapters/curl-redirect-hop.js +11 -0
- package/dist/adapters/curl-stealth.cjs +120 -0
- package/dist/adapters/curl-stealth.js +113 -0
- package/dist/adapters/curl.cjs +1241 -381
- package/dist/adapters/curl.js +1246 -381
- package/dist/adapters/download-target-transaction.cjs +263 -0
- package/dist/adapters/download-target-transaction.js +267 -0
- package/dist/adapters/entries/curl.d.ts +945 -549
- package/dist/adapters/entries/fetch.cjs +6 -1
- package/dist/adapters/entries/fetch.d.ts +683 -386
- package/dist/adapters/entries/fetch.js +6 -1
- package/dist/adapters/entries/http.d.ts +684 -387
- package/dist/adapters/entries/http2.d.ts +699 -387
- package/dist/adapters/entries/react-native.cjs +6 -6
- package/dist/adapters/entries/react-native.d.ts +683 -386
- package/dist/adapters/entries/xhr.cjs +6 -1
- package/dist/adapters/entries/xhr.d.ts +683 -386
- package/dist/adapters/entries/xhr.js +6 -1
- package/dist/adapters/fetch-request-body.cjs +51 -0
- package/dist/adapters/fetch-request-body.js +49 -0
- package/dist/adapters/fetch.cjs +1492 -459
- package/dist/adapters/fetch.js +1505 -459
- package/dist/adapters/http.cjs +2150 -986
- package/dist/adapters/http.js +2156 -986
- package/dist/adapters/http2-session-identity.cjs +61 -0
- package/dist/adapters/http2-session-identity.js +59 -0
- package/dist/adapters/http2.cjs +2011 -575
- package/dist/adapters/http2.js +2018 -575
- package/dist/adapters/index.cjs +10 -10
- package/dist/adapters/index.d.ts +188 -33
- package/dist/adapters/multipart-request-body.cjs +12 -0
- package/dist/adapters/multipart-request-body.js +10 -0
- package/dist/adapters/node-request-body.cjs +63 -0
- package/dist/adapters/node-request-body.js +58 -0
- package/dist/adapters/react-native.cjs +1645 -905
- package/dist/adapters/react-native.js +1655 -905
- package/dist/adapters/xhr.cjs +757 -245
- package/dist/adapters/xhr.js +762 -245
- package/dist/cache/bound-response-cache.cjs +20 -0
- package/dist/cache/bound-response-cache.js +17 -0
- package/dist/cache/dns-cache.cjs +62 -46
- package/dist/cache/dns-cache.js +62 -46
- package/dist/cache/index.cjs +9 -9
- package/dist/cache/lru-cache.cjs +5 -0
- package/dist/cache/lru-cache.js +5 -0
- package/dist/cache/response-cache-identity.cjs +207 -0
- package/dist/cache/response-cache-identity.js +201 -0
- package/dist/cache/response-cache-ownership.cjs +19 -0
- package/dist/cache/response-cache-ownership.js +15 -0
- package/dist/cache/response-cache-persistence.cjs +706 -0
- package/dist/cache/response-cache-persistence.js +705 -0
- package/dist/cache/response-cache-readiness.cjs +10 -0
- package/dist/cache/response-cache-readiness.js +7 -0
- package/dist/cache/response-cache.cjs +132 -147
- package/dist/cache/response-cache.js +132 -147
- package/dist/cache/universal-response-cache.cjs +52 -26
- package/dist/cache/universal-response-cache.js +52 -26
- package/dist/cookies/cookie-jar.cjs +7 -6
- package/dist/cookies/cookie-jar.js +3 -2
- package/dist/cookies/index.cjs +10 -10
- package/dist/core/adapter-capabilities.cjs +137 -0
- package/dist/core/adapter-capabilities.js +129 -0
- package/dist/core/hooks.cjs +27 -0
- package/dist/core/hooks.js +25 -0
- package/dist/core/rezo.cjs +177 -61
- package/dist/core/rezo.js +182 -61
- package/dist/crawler/addon/decodo/index.cjs +1 -1
- package/dist/crawler/addon/decodo/index.js +1 -1
- package/dist/crawler/addon/decodo/options.js +1 -1
- package/dist/crawler/addon/oxylabs/index.cjs +1 -1
- package/dist/crawler/addon/oxylabs/index.js +1 -1
- package/dist/crawler/addon/oxylabs/options.js +1 -1
- package/dist/crawler/crawler-options.cjs +1 -1
- package/dist/crawler/crawler-options.js +1 -1
- package/dist/crawler/index.cjs +42 -42
- package/dist/crawler/plugin/capped-array.js +1 -1
- package/dist/crawler/plugin/capped-map.js +1 -1
- package/dist/crawler/plugin/file-cacher.cjs +4 -4
- package/dist/crawler/plugin/file-cacher.js +1 -1
- package/dist/crawler/plugin/health-metrics.cjs +1 -1
- package/dist/crawler/plugin/health-metrics.js +2 -2
- package/dist/crawler/plugin/index.cjs +1 -1
- package/dist/crawler/plugin/index.js +1 -1
- package/dist/crawler/plugin/memory-monitor.js +1 -1
- package/dist/crawler/plugin/navigation-history.cjs +4 -4
- package/dist/crawler/plugin/result-stream.js +1 -1
- package/dist/crawler/plugin/robots-txt.js +1 -1
- package/dist/crawler/plugin/sqlite-utils.cjs +1 -1
- package/dist/crawler/plugin/sqlite-utils.js +1 -1
- package/dist/crawler/plugin/url-store.js +1 -1
- package/dist/crawler/scraper.cjs +1 -1
- package/dist/crawler/scraper.js +1 -1
- package/dist/crawler.d.ts +677 -330
- package/dist/entries/crawler.cjs +24 -24
- package/dist/errors/rezo-error.cjs +168 -51
- package/dist/errors/rezo-error.js +168 -51
- package/dist/index.cjs +58 -58
- package/dist/index.d.ts +691 -390
- package/dist/internal/agents/bun-socks-http.cjs +22 -6
- package/dist/internal/agents/bun-socks-http.js +22 -6
- package/dist/internal/agents/https-proxy.cjs +15 -1
- package/dist/internal/agents/https-proxy.js +15 -1
- package/dist/internal/agents/index.cjs +18 -14
- package/dist/internal/agents/index.js +1 -0
- package/dist/internal/agents/pending-proxy-handshake.cjs +30 -0
- package/dist/internal/agents/pending-proxy-handshake.js +26 -0
- package/dist/internal/agents/socks-client.cjs +2 -1
- package/dist/internal/agents/socks-client.js +2 -1
- package/dist/internal/agents/socks-proxy.cjs +20 -9
- package/dist/internal/agents/socks-proxy.js +20 -9
- package/dist/internal/agents/wait-for-socket-connection.cjs +26 -0
- package/dist/internal/agents/wait-for-socket-connection.js +24 -0
- package/dist/platform/browser.cjs +7 -2
- package/dist/platform/browser.d.ts +683 -386
- package/dist/platform/browser.js +7 -2
- package/dist/platform/bun.d.ts +683 -386
- package/dist/platform/deno.d.ts +683 -386
- package/dist/platform/native-tls-capabilities.cjs +5 -0
- package/dist/platform/native-tls-capabilities.js +3 -0
- package/dist/platform/node.d.ts +683 -386
- package/dist/platform/react-native-providers.cjs +697 -190
- package/dist/platform/react-native-providers.js +697 -190
- package/dist/platform/react-native.cjs +6 -6
- package/dist/platform/react-native.d.ts +683 -386
- package/dist/platform/worker.d.ts +683 -386
- package/dist/proxy/index.cjs +9 -6
- package/dist/proxy/index.js +4 -2
- package/dist/proxy/manager.cjs +16 -5
- package/dist/proxy/manager.js +14 -4
- package/dist/proxy/parse.cjs +1 -1
- package/dist/proxy/parse.js +1 -1
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/buildError.cjs +14 -1
- package/dist/responses/buildError.js +13 -1
- package/dist/responses/buildResponse.cjs +46 -30
- package/dist/responses/buildResponse.js +44 -31
- package/dist/responses/sanitize-config.cjs +140 -0
- package/dist/responses/sanitize-config.js +138 -0
- package/dist/responses/universal/event-emitter.cjs +16 -10
- package/dist/responses/universal/event-emitter.js +15 -10
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/responses/universal/stream.cjs +20 -4
- package/dist/responses/universal/stream.js +23 -4
- package/dist/shared/combine-wait-interrupts.cjs +26 -0
- package/dist/shared/combine-wait-interrupts.js +24 -0
- package/dist/shared/contain-lifecycle-hook.cjs +19 -0
- package/dist/shared/contain-lifecycle-hook.js +17 -0
- package/dist/shared/create-fetch-request-deadline.cjs +98 -0
- package/dist/shared/create-fetch-request-deadline.js +96 -0
- package/dist/shared/create-fetch-timeout-error.cjs +12 -0
- package/dist/shared/create-fetch-timeout-error.js +10 -0
- package/dist/shared/create-staged-timeout-error.cjs +16 -0
- package/dist/shared/create-staged-timeout-error.js +14 -0
- package/dist/shared/create-total-deadline.cjs +31 -0
- package/dist/shared/create-total-deadline.js +29 -0
- package/dist/shared/index.cjs +17 -0
- package/dist/shared/index.js +13 -0
- package/dist/shared/resolve-response-type.cjs +48 -0
- package/dist/shared/resolve-response-type.js +45 -0
- package/dist/shared/status-attempt-continues.cjs +10 -0
- package/dist/shared/status-attempt-continues.js +8 -0
- package/dist/stealth/index.cjs +17 -17
- package/dist/stealth/index.d.ts +81 -36
- package/dist/stealth/profiles/chrome-profiles.cjs +57 -2
- package/dist/stealth/profiles/chrome-profiles.js +57 -1
- package/dist/stealth/profiles/constants.cjs +46 -16
- package/dist/stealth/profiles/constants.js +45 -16
- package/dist/stealth/profiles/edge-profiles.cjs +83 -2
- package/dist/stealth/profiles/edge-profiles.js +83 -2
- package/dist/stealth/profiles/firefox-profiles.cjs +48 -1
- package/dist/stealth/profiles/firefox-profiles.js +48 -1
- package/dist/stealth/profiles/index.cjs +53 -14
- package/dist/stealth/profiles/index.js +41 -4
- package/dist/stealth/profiles/safari-profiles.cjs +48 -1
- package/dist/stealth/profiles/safari-profiles.js +48 -1
- package/dist/stealth/resolver.cjs +65 -14
- package/dist/stealth/resolver.js +64 -14
- package/dist/stealth/tls-fingerprint.cjs +41 -9
- package/dist/stealth/tls-fingerprint.js +40 -9
- package/dist/stealth/tls-fingerprint.universal.cjs +13 -0
- package/dist/stealth/tls-fingerprint.universal.js +10 -0
- package/dist/stealth/tls-material.cjs +77 -0
- package/dist/stealth/tls-material.js +74 -0
- package/dist/stealth/universal.cjs +17 -0
- package/dist/stealth/universal.d.ts +312 -0
- package/dist/stealth/universal.js +13 -0
- package/dist/utils/agent-pool.cjs +75 -22
- package/dist/utils/agent-pool.js +75 -22
- package/dist/utils/compression.cjs +76 -135
- package/dist/utils/compression.js +76 -135
- package/dist/utils/debug-error-dump.cjs +81 -33
- package/dist/utils/debug-error-dump.js +81 -33
- package/dist/utils/form-data-encoding.cjs +61 -0
- package/dist/utils/form-data-encoding.js +59 -0
- package/dist/utils/form-data.cjs +22 -35
- package/dist/utils/form-data.js +22 -35
- package/dist/utils/headers.cjs +52 -0
- package/dist/utils/headers.js +52 -1
- package/dist/utils/http-config.cjs +156 -50
- package/dist/utils/http-config.js +156 -50
- package/dist/utils/node-runtime.cjs +14 -8
- package/dist/utils/node-runtime.js +14 -8
- package/dist/utils/rate-limit-wait.cjs +82 -5
- package/dist/utils/rate-limit-wait.js +82 -5
- package/dist/utils/redirect-header-policy.cjs +202 -0
- package/dist/utils/redirect-header-policy.js +206 -0
- package/dist/utils/request-body.cjs +71 -0
- package/dist/utils/request-body.js +60 -0
- package/dist/utils/request-fetch-options.cjs +124 -0
- package/dist/utils/request-fetch-options.js +116 -0
- package/dist/utils/request-input-errors.cjs +10 -0
- package/dist/utils/request-input-errors.js +7 -0
- package/dist/utils/request-input-policy.cjs +133 -0
- package/dist/utils/request-input-policy.js +131 -0
- package/dist/utils/request-input-url.cjs +49 -0
- package/dist/utils/request-input-url.js +47 -0
- package/dist/utils/request-input.cjs +98 -0
- package/dist/utils/request-input.js +95 -0
- package/dist/utils/socket-telemetry.cjs +44 -23
- package/dist/utils/socket-telemetry.js +44 -23
- package/dist/utils/staged-timeout.cjs +18 -18
- package/dist/utils/staged-timeout.js +18 -18
- package/dist/utils/tools.cjs +795 -1
- package/dist/utils/tools.js +791 -0
- package/dist/utils/zstd-frame-validator.cjs +205 -0
- package/dist/utils/zstd-frame-validator.js +202 -0
- package/dist/version.cjs +1 -1
- package/dist/version.js +1 -1
- package/dist/wget/index.cjs +51 -51
- package/dist/wget/index.d.ts +677 -330
- package/package.json +32 -2
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
const EXIT_CODE_TO_ERROR_CODE = {
|
|
2
|
+
1: "ERR_INVALID_PROTOCOL",
|
|
3
|
+
3: "ERR_INVALID_URL",
|
|
4
|
+
5: "REZ_PROXY_CONNECTION_FAILED",
|
|
5
|
+
6: "ENOTFOUND",
|
|
6
|
+
8: "HPE_INVALID_CONSTANT",
|
|
7
|
+
16: "ERR_HTTP2_ERROR",
|
|
8
|
+
18: "ERR_STREAM_PREMATURE_CLOSE",
|
|
9
|
+
22: "REZ_HTTP_ERROR",
|
|
10
|
+
23: "REZ_DOWNLOAD_FAILED",
|
|
11
|
+
26: "REZ_UPLOAD_FAILED",
|
|
12
|
+
35: "EPROTO",
|
|
13
|
+
47: "REZ_MAX_REDIRECTS_EXCEEDED",
|
|
14
|
+
52: "ECONNRESET",
|
|
15
|
+
53: "EPROTO",
|
|
16
|
+
55: "EPIPE",
|
|
17
|
+
56: "ECONNRESET",
|
|
18
|
+
58: "ERR_TLS_INVALID_CLIENT_CERTIFICATE",
|
|
19
|
+
59: "EPROTO",
|
|
20
|
+
61: "REZ_DECOMPRESSION_ERROR",
|
|
21
|
+
63: "REZ_RESPONSE_TOO_LARGE",
|
|
22
|
+
77: "ERR_TLS_CA_BUNDLE_UNREADABLE",
|
|
23
|
+
91: "REZ_SOCKS_PROTOCOL_ERROR",
|
|
24
|
+
92: "ECONNRESET",
|
|
25
|
+
97: "REZ_PROXY_TARGET_UNREACHABLE",
|
|
26
|
+
99: "ERR_TLS_INVALID_CLIENT_CERTIFICATE"
|
|
27
|
+
};
|
|
28
|
+
const CONNECT_FAILURE_PATTERNS = [
|
|
29
|
+
[/connection refused/i, "ECONNREFUSED"],
|
|
30
|
+
[/no route to host/i, "EHOSTUNREACH"],
|
|
31
|
+
[/network (is )?unreachable/i, "ENETUNREACH"],
|
|
32
|
+
[/timed? ?out/i, "ETIMEDOUT"]
|
|
33
|
+
];
|
|
34
|
+
const VERIFICATION_FAILURE_PATTERNS = [
|
|
35
|
+
[/no alternative certificate subject name matches|does not match target host name/i, "ERR_TLS_CERT_ALTNAME_INVALID"],
|
|
36
|
+
[/certificate has expired/i, "CERT_HAS_EXPIRED"],
|
|
37
|
+
[/certificate is not yet valid/i, "CERT_NOT_YET_VALID"],
|
|
38
|
+
[/self[- ]signed certificate in certificate chain/i, "SELF_SIGNED_CERT_IN_CHAIN"],
|
|
39
|
+
[/self[- ]signed certificate/i, "DEPTH_ZERO_SELF_SIGNED_CERT"],
|
|
40
|
+
[/unable to get (local )?issuer certificate/i, "UNABLE_TO_GET_ISSUER_CERT_LOCALLY"]
|
|
41
|
+
];
|
|
42
|
+
const CURL_EXIT_COULD_NOT_CONNECT = 7;
|
|
43
|
+
const CURL_EXIT_PEER_VERIFICATION_FAILED = 60;
|
|
44
|
+
const CURL_EXIT_PEER_VERIFICATION_FAILED_LEGACY = 51;
|
|
45
|
+
function firstMatchingCode(stderr, patterns, fallback) {
|
|
46
|
+
for (const [pattern, code] of patterns) {
|
|
47
|
+
if (pattern.test(stderr))
|
|
48
|
+
return code;
|
|
49
|
+
}
|
|
50
|
+
return fallback;
|
|
51
|
+
}
|
|
52
|
+
function classifyCurlExitCode(exitCode, stderr) {
|
|
53
|
+
if (exitCode === CURL_EXIT_COULD_NOT_CONNECT) {
|
|
54
|
+
return firstMatchingCode(stderr, CONNECT_FAILURE_PATTERNS, "ECONNREFUSED");
|
|
55
|
+
}
|
|
56
|
+
if (exitCode === CURL_EXIT_PEER_VERIFICATION_FAILED || exitCode === CURL_EXIT_PEER_VERIFICATION_FAILED_LEGACY) {
|
|
57
|
+
return firstMatchingCode(stderr, VERIFICATION_FAILURE_PATTERNS, "UNABLE_TO_VERIFY_LEAF_SIGNATURE");
|
|
58
|
+
}
|
|
59
|
+
return EXIT_CODE_TO_ERROR_CODE[exitCode] ?? "REZ_UNKNOWN_ERROR";
|
|
60
|
+
}
|
|
61
|
+
function isCurlTransferInterruption(exitCode) {
|
|
62
|
+
return exitCode === 18 || exitCode === 55 || exitCode === 56 || exitCode === 92;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
exports.classifyCurlExitCode = classifyCurlExitCode;
|
|
66
|
+
exports.isCurlTransferInterruption = isCurlTransferInterruption;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
const EXIT_CODE_TO_ERROR_CODE = {
|
|
2
|
+
1: "ERR_INVALID_PROTOCOL",
|
|
3
|
+
3: "ERR_INVALID_URL",
|
|
4
|
+
5: "REZ_PROXY_CONNECTION_FAILED",
|
|
5
|
+
6: "ENOTFOUND",
|
|
6
|
+
8: "HPE_INVALID_CONSTANT",
|
|
7
|
+
16: "ERR_HTTP2_ERROR",
|
|
8
|
+
18: "ERR_STREAM_PREMATURE_CLOSE",
|
|
9
|
+
22: "REZ_HTTP_ERROR",
|
|
10
|
+
23: "REZ_DOWNLOAD_FAILED",
|
|
11
|
+
26: "REZ_UPLOAD_FAILED",
|
|
12
|
+
35: "EPROTO",
|
|
13
|
+
47: "REZ_MAX_REDIRECTS_EXCEEDED",
|
|
14
|
+
52: "ECONNRESET",
|
|
15
|
+
53: "EPROTO",
|
|
16
|
+
55: "EPIPE",
|
|
17
|
+
56: "ECONNRESET",
|
|
18
|
+
58: "ERR_TLS_INVALID_CLIENT_CERTIFICATE",
|
|
19
|
+
59: "EPROTO",
|
|
20
|
+
61: "REZ_DECOMPRESSION_ERROR",
|
|
21
|
+
63: "REZ_RESPONSE_TOO_LARGE",
|
|
22
|
+
77: "ERR_TLS_CA_BUNDLE_UNREADABLE",
|
|
23
|
+
91: "REZ_SOCKS_PROTOCOL_ERROR",
|
|
24
|
+
92: "ECONNRESET",
|
|
25
|
+
97: "REZ_PROXY_TARGET_UNREACHABLE",
|
|
26
|
+
99: "ERR_TLS_INVALID_CLIENT_CERTIFICATE"
|
|
27
|
+
};
|
|
28
|
+
const CONNECT_FAILURE_PATTERNS = [
|
|
29
|
+
[/connection refused/i, "ECONNREFUSED"],
|
|
30
|
+
[/no route to host/i, "EHOSTUNREACH"],
|
|
31
|
+
[/network (is )?unreachable/i, "ENETUNREACH"],
|
|
32
|
+
[/timed? ?out/i, "ETIMEDOUT"]
|
|
33
|
+
];
|
|
34
|
+
const VERIFICATION_FAILURE_PATTERNS = [
|
|
35
|
+
[/no alternative certificate subject name matches|does not match target host name/i, "ERR_TLS_CERT_ALTNAME_INVALID"],
|
|
36
|
+
[/certificate has expired/i, "CERT_HAS_EXPIRED"],
|
|
37
|
+
[/certificate is not yet valid/i, "CERT_NOT_YET_VALID"],
|
|
38
|
+
[/self[- ]signed certificate in certificate chain/i, "SELF_SIGNED_CERT_IN_CHAIN"],
|
|
39
|
+
[/self[- ]signed certificate/i, "DEPTH_ZERO_SELF_SIGNED_CERT"],
|
|
40
|
+
[/unable to get (local )?issuer certificate/i, "UNABLE_TO_GET_ISSUER_CERT_LOCALLY"]
|
|
41
|
+
];
|
|
42
|
+
const CURL_EXIT_COULD_NOT_CONNECT = 7;
|
|
43
|
+
const CURL_EXIT_PEER_VERIFICATION_FAILED = 60;
|
|
44
|
+
const CURL_EXIT_PEER_VERIFICATION_FAILED_LEGACY = 51;
|
|
45
|
+
function firstMatchingCode(stderr, patterns, fallback) {
|
|
46
|
+
for (const [pattern, code] of patterns) {
|
|
47
|
+
if (pattern.test(stderr))
|
|
48
|
+
return code;
|
|
49
|
+
}
|
|
50
|
+
return fallback;
|
|
51
|
+
}
|
|
52
|
+
export function classifyCurlExitCode(exitCode, stderr) {
|
|
53
|
+
if (exitCode === CURL_EXIT_COULD_NOT_CONNECT) {
|
|
54
|
+
return firstMatchingCode(stderr, CONNECT_FAILURE_PATTERNS, "ECONNREFUSED");
|
|
55
|
+
}
|
|
56
|
+
if (exitCode === CURL_EXIT_PEER_VERIFICATION_FAILED || exitCode === CURL_EXIT_PEER_VERIFICATION_FAILED_LEGACY) {
|
|
57
|
+
return firstMatchingCode(stderr, VERIFICATION_FAILURE_PATTERNS, "UNABLE_TO_VERIFY_LEAF_SIGNATURE");
|
|
58
|
+
}
|
|
59
|
+
return EXIT_CODE_TO_ERROR_CODE[exitCode] ?? "REZ_UNKNOWN_ERROR";
|
|
60
|
+
}
|
|
61
|
+
export function isCurlTransferInterruption(exitCode) {
|
|
62
|
+
return exitCode === 18 || exitCode === 55 || exitCode === 56 || exitCode === 92;
|
|
63
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
function isFollowableRedirectStatus(status) {
|
|
2
|
+
return status >= 300 && status < 400 && status !== 304;
|
|
3
|
+
}
|
|
4
|
+
function planCurlRedirectHop(status, method, location, currentUrl) {
|
|
5
|
+
if (!isFollowableRedirectStatus(status) || !location)
|
|
6
|
+
return null;
|
|
7
|
+
const url = new URL(location, currentUrl).toString();
|
|
8
|
+
if (status === 301 || status === 302 || status === 303)
|
|
9
|
+
return { dropBody: true, method: "GET", url };
|
|
10
|
+
return { dropBody: false, method: method.toUpperCase(), url };
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
exports.isFollowableRedirectStatus = isFollowableRedirectStatus;
|
|
14
|
+
exports.planCurlRedirectHop = planCurlRedirectHop;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export function isFollowableRedirectStatus(status) {
|
|
2
|
+
return status >= 300 && status < 400 && status !== 304;
|
|
3
|
+
}
|
|
4
|
+
export function planCurlRedirectHop(status, method, location, currentUrl) {
|
|
5
|
+
if (!isFollowableRedirectStatus(status) || !location)
|
|
6
|
+
return null;
|
|
7
|
+
const url = new URL(location, currentUrl).toString();
|
|
8
|
+
if (status === 301 || status === 302 || status === 303)
|
|
9
|
+
return { dropBody: true, method: "GET", url };
|
|
10
|
+
return { dropBody: false, method: method.toUpperCase(), url };
|
|
11
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
const { spawn } = require("node:child_process");
|
|
2
|
+
const { createServer } = require("node:net");
|
|
3
|
+
const OPENSSL_FAMILY = new Set(["OpenSSL", "quictls"]);
|
|
4
|
+
const BACKEND_TOKEN = /^\(?(OpenSSL|quictls|LibreSSL|BoringSSL|AWS-LC|wolfSSL|GnuTLS|mbedTLS|Schannel|SecureTransport|rustls|bearssl)(?:\/([\w.+-]+))?\)?$/u;
|
|
5
|
+
const CURL_ROUTE_BOUNDARIES = Object.freeze([
|
|
6
|
+
"sigalgs",
|
|
7
|
+
"renegotiationScsv",
|
|
8
|
+
"extensionOrder",
|
|
9
|
+
"grease",
|
|
10
|
+
"h2Settings",
|
|
11
|
+
"h2ConnectionWindow",
|
|
12
|
+
"h2PseudoHeaderOrder"
|
|
13
|
+
]);
|
|
14
|
+
function parseCurlTlsBackends(versionLine) {
|
|
15
|
+
const backends = [];
|
|
16
|
+
for (const token of versionLine.trim().split(/\s+/u)) {
|
|
17
|
+
const match = token.match(BACKEND_TOKEN);
|
|
18
|
+
if (match)
|
|
19
|
+
backends.push({ name: match[1], version: match[2] ?? null });
|
|
20
|
+
}
|
|
21
|
+
return backends;
|
|
22
|
+
}
|
|
23
|
+
function splitCipherSuites(ciphers) {
|
|
24
|
+
const tls12 = [];
|
|
25
|
+
const tls13 = [];
|
|
26
|
+
for (const suite of ciphers.split(":")) {
|
|
27
|
+
if (!suite || suite.startsWith("@"))
|
|
28
|
+
continue;
|
|
29
|
+
(suite.startsWith("TLS_") ? tls13 : tls12).push(suite);
|
|
30
|
+
}
|
|
31
|
+
return { tls12, tls13 };
|
|
32
|
+
}
|
|
33
|
+
function supportsKeyShareMarkers(backend) {
|
|
34
|
+
if (!backend || !OPENSSL_FAMILY.has(backend.name) || !backend.version)
|
|
35
|
+
return false;
|
|
36
|
+
const [major = 0, minor = 0] = backend.version.split(".").map((part) => Number.parseInt(part, 10));
|
|
37
|
+
return major > 3 || major === 3 && minor >= 5;
|
|
38
|
+
}
|
|
39
|
+
function stealthTlsArguments(profile, keyShareMarkers = false) {
|
|
40
|
+
const { tls12, tls13 } = splitCipherSuites(profile.tls.ciphers);
|
|
41
|
+
const args = [];
|
|
42
|
+
args.push(profile.tls.minVersion === "TLSv1.3" ? "--tlsv1.3" : "--tlsv1.2");
|
|
43
|
+
args.push("--tls-max", profile.tls.maxVersion === "TLSv1.2" ? "1.2" : "1.3");
|
|
44
|
+
if (tls13.length > 0)
|
|
45
|
+
args.push("--tls13-ciphers", tls13.join(":"));
|
|
46
|
+
if (tls12.length > 0)
|
|
47
|
+
args.push("--ciphers", tls12.join(":"));
|
|
48
|
+
const groups = profile.tls.ecdhCurve.split(":").filter((group) => group.length > 0).map((group) => keyShareMarkers ? group : group.replace(/^\*/u, ""));
|
|
49
|
+
if (groups.length > 0)
|
|
50
|
+
args.push("--curves", groups.join(":"));
|
|
51
|
+
return args;
|
|
52
|
+
}
|
|
53
|
+
const acceptanceByMaterial = new Map;
|
|
54
|
+
function probeCurlTlsAcceptance(tlsArgs, curlCommand = "curl") {
|
|
55
|
+
const key = `${curlCommand} ${tlsArgs.join(" ")}`;
|
|
56
|
+
let pending = acceptanceByMaterial.get(key);
|
|
57
|
+
if (!pending) {
|
|
58
|
+
pending = runAcceptanceProbe(tlsArgs, curlCommand);
|
|
59
|
+
acceptanceByMaterial.set(key, pending);
|
|
60
|
+
pending.catch(() => acceptanceByMaterial.delete(key));
|
|
61
|
+
}
|
|
62
|
+
return pending;
|
|
63
|
+
}
|
|
64
|
+
async function runAcceptanceProbe(tlsArgs, curlCommand) {
|
|
65
|
+
const listener = createServer((socket) => socket.destroy());
|
|
66
|
+
await new Promise((resolve, reject) => {
|
|
67
|
+
listener.once("error", reject);
|
|
68
|
+
listener.listen(0, "127.0.0.1", () => resolve());
|
|
69
|
+
});
|
|
70
|
+
const address = listener.address();
|
|
71
|
+
const port = typeof address === "object" && address ? address.port : 0;
|
|
72
|
+
try {
|
|
73
|
+
return await new Promise((resolve, reject) => {
|
|
74
|
+
const child = spawn(curlCommand, ["-sS", "--max-time", "10", "-o", "/dev/null", ...tlsArgs, `https://127.0.0.1:${port}/`], { stdio: ["ignore", "ignore", "pipe"] });
|
|
75
|
+
let stderr = "";
|
|
76
|
+
child.stderr.on("data", (chunk) => {
|
|
77
|
+
stderr += chunk.toString();
|
|
78
|
+
});
|
|
79
|
+
child.once("error", reject);
|
|
80
|
+
child.once("close", (code) => {
|
|
81
|
+
if (code === 59)
|
|
82
|
+
resolve({ accepted: false, detail: stderr.trim().split(`
|
|
83
|
+
`).pop() ?? `curl exit ${code}` });
|
|
84
|
+
else
|
|
85
|
+
resolve({ accepted: true });
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
} finally {
|
|
89
|
+
await new Promise((resolve) => listener.close(() => resolve()));
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
async function mapStealthProfileToCurl(profile, capabilities, curlCommand = "curl") {
|
|
93
|
+
const reasons = [];
|
|
94
|
+
if (capabilities.tlsBackends.length !== 1) {
|
|
95
|
+
reasons.push(capabilities.tlsBackends.length === 0 ? `curl ${capabilities.curlVersion} names no TLS backend, so its cipher and group syntax is unknown` : `curl ${capabilities.curlVersion} is a MultiSSL build (${capabilities.tlsBackends.map((backend) => backend.name).join(", ")}); the active backend cannot be known from here`);
|
|
96
|
+
} else if (!OPENSSL_FAMILY.has(capabilities.tlsBackends[0].name)) {
|
|
97
|
+
reasons.push(`curl ${capabilities.curlVersion} uses ${capabilities.tlsBackends[0].name}, which does not take the profile's OpenSSL cipher and group lists`);
|
|
98
|
+
}
|
|
99
|
+
const wantsHttp2 = profile.tls.alpnProtocols.includes("h2");
|
|
100
|
+
if (wantsHttp2 && !capabilities.http2)
|
|
101
|
+
reasons.push(`profile offers h2 in ALPN but curl ${capabilities.curlVersion} was built without HTTP/2`);
|
|
102
|
+
if (reasons.length > 0)
|
|
103
|
+
return { ok: false, reasons };
|
|
104
|
+
const tlsArgs = stealthTlsArguments(profile, supportsKeyShareMarkers(capabilities.tlsBackends[0]));
|
|
105
|
+
const acceptance = await probeCurlTlsAcceptance(tlsArgs, curlCommand);
|
|
106
|
+
if (!acceptance.accepted) {
|
|
107
|
+
return { ok: false, reasons: [`curl ${capabilities.curlVersion} (${capabilities.tlsBackends[0].name}/${capabilities.tlsBackends[0].version ?? "?"}) rejects the profile's TLS material: ${acceptance.detail}`] };
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
ok: true,
|
|
111
|
+
mapping: { tlsArgs, httpVersionArg: wantsHttp2 ? "--http2" : "--http1.1", useHttp2Headers: wantsHttp2, notExpressible: CURL_ROUTE_BOUNDARIES }
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
exports.parseCurlTlsBackends = parseCurlTlsBackends;
|
|
116
|
+
exports.splitCipherSuites = splitCipherSuites;
|
|
117
|
+
exports.supportsKeyShareMarkers = supportsKeyShareMarkers;
|
|
118
|
+
exports.stealthTlsArguments = stealthTlsArguments;
|
|
119
|
+
exports.probeCurlTlsAcceptance = probeCurlTlsAcceptance;
|
|
120
|
+
exports.mapStealthProfileToCurl = mapStealthProfileToCurl;
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { createServer } from "node:net";
|
|
3
|
+
const OPENSSL_FAMILY = new Set(["OpenSSL", "quictls"]);
|
|
4
|
+
const BACKEND_TOKEN = /^\(?(OpenSSL|quictls|LibreSSL|BoringSSL|AWS-LC|wolfSSL|GnuTLS|mbedTLS|Schannel|SecureTransport|rustls|bearssl)(?:\/([\w.+-]+))?\)?$/u;
|
|
5
|
+
const CURL_ROUTE_BOUNDARIES = Object.freeze([
|
|
6
|
+
"sigalgs",
|
|
7
|
+
"renegotiationScsv",
|
|
8
|
+
"extensionOrder",
|
|
9
|
+
"grease",
|
|
10
|
+
"h2Settings",
|
|
11
|
+
"h2ConnectionWindow",
|
|
12
|
+
"h2PseudoHeaderOrder"
|
|
13
|
+
]);
|
|
14
|
+
export function parseCurlTlsBackends(versionLine) {
|
|
15
|
+
const backends = [];
|
|
16
|
+
for (const token of versionLine.trim().split(/\s+/u)) {
|
|
17
|
+
const match = token.match(BACKEND_TOKEN);
|
|
18
|
+
if (match)
|
|
19
|
+
backends.push({ name: match[1], version: match[2] ?? null });
|
|
20
|
+
}
|
|
21
|
+
return backends;
|
|
22
|
+
}
|
|
23
|
+
export function splitCipherSuites(ciphers) {
|
|
24
|
+
const tls12 = [];
|
|
25
|
+
const tls13 = [];
|
|
26
|
+
for (const suite of ciphers.split(":")) {
|
|
27
|
+
if (!suite || suite.startsWith("@"))
|
|
28
|
+
continue;
|
|
29
|
+
(suite.startsWith("TLS_") ? tls13 : tls12).push(suite);
|
|
30
|
+
}
|
|
31
|
+
return { tls12, tls13 };
|
|
32
|
+
}
|
|
33
|
+
export function supportsKeyShareMarkers(backend) {
|
|
34
|
+
if (!backend || !OPENSSL_FAMILY.has(backend.name) || !backend.version)
|
|
35
|
+
return false;
|
|
36
|
+
const [major = 0, minor = 0] = backend.version.split(".").map((part) => Number.parseInt(part, 10));
|
|
37
|
+
return major > 3 || major === 3 && minor >= 5;
|
|
38
|
+
}
|
|
39
|
+
export function stealthTlsArguments(profile, keyShareMarkers = false) {
|
|
40
|
+
const { tls12, tls13 } = splitCipherSuites(profile.tls.ciphers);
|
|
41
|
+
const args = [];
|
|
42
|
+
args.push(profile.tls.minVersion === "TLSv1.3" ? "--tlsv1.3" : "--tlsv1.2");
|
|
43
|
+
args.push("--tls-max", profile.tls.maxVersion === "TLSv1.2" ? "1.2" : "1.3");
|
|
44
|
+
if (tls13.length > 0)
|
|
45
|
+
args.push("--tls13-ciphers", tls13.join(":"));
|
|
46
|
+
if (tls12.length > 0)
|
|
47
|
+
args.push("--ciphers", tls12.join(":"));
|
|
48
|
+
const groups = profile.tls.ecdhCurve.split(":").filter((group) => group.length > 0).map((group) => keyShareMarkers ? group : group.replace(/^\*/u, ""));
|
|
49
|
+
if (groups.length > 0)
|
|
50
|
+
args.push("--curves", groups.join(":"));
|
|
51
|
+
return args;
|
|
52
|
+
}
|
|
53
|
+
const acceptanceByMaterial = new Map;
|
|
54
|
+
export function probeCurlTlsAcceptance(tlsArgs, curlCommand = "curl") {
|
|
55
|
+
const key = `${curlCommand} ${tlsArgs.join(" ")}`;
|
|
56
|
+
let pending = acceptanceByMaterial.get(key);
|
|
57
|
+
if (!pending) {
|
|
58
|
+
pending = runAcceptanceProbe(tlsArgs, curlCommand);
|
|
59
|
+
acceptanceByMaterial.set(key, pending);
|
|
60
|
+
pending.catch(() => acceptanceByMaterial.delete(key));
|
|
61
|
+
}
|
|
62
|
+
return pending;
|
|
63
|
+
}
|
|
64
|
+
async function runAcceptanceProbe(tlsArgs, curlCommand) {
|
|
65
|
+
const listener = createServer((socket) => socket.destroy());
|
|
66
|
+
await new Promise((resolve, reject) => {
|
|
67
|
+
listener.once("error", reject);
|
|
68
|
+
listener.listen(0, "127.0.0.1", () => resolve());
|
|
69
|
+
});
|
|
70
|
+
const address = listener.address();
|
|
71
|
+
const port = typeof address === "object" && address ? address.port : 0;
|
|
72
|
+
try {
|
|
73
|
+
return await new Promise((resolve, reject) => {
|
|
74
|
+
const child = spawn(curlCommand, ["-sS", "--max-time", "10", "-o", "/dev/null", ...tlsArgs, `https://127.0.0.1:${port}/`], { stdio: ["ignore", "ignore", "pipe"] });
|
|
75
|
+
let stderr = "";
|
|
76
|
+
child.stderr.on("data", (chunk) => {
|
|
77
|
+
stderr += chunk.toString();
|
|
78
|
+
});
|
|
79
|
+
child.once("error", reject);
|
|
80
|
+
child.once("close", (code) => {
|
|
81
|
+
if (code === 59)
|
|
82
|
+
resolve({ accepted: false, detail: stderr.trim().split(`
|
|
83
|
+
`).pop() ?? `curl exit ${code}` });
|
|
84
|
+
else
|
|
85
|
+
resolve({ accepted: true });
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
} finally {
|
|
89
|
+
await new Promise((resolve) => listener.close(() => resolve()));
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
export async function mapStealthProfileToCurl(profile, capabilities, curlCommand = "curl") {
|
|
93
|
+
const reasons = [];
|
|
94
|
+
if (capabilities.tlsBackends.length !== 1) {
|
|
95
|
+
reasons.push(capabilities.tlsBackends.length === 0 ? `curl ${capabilities.curlVersion} names no TLS backend, so its cipher and group syntax is unknown` : `curl ${capabilities.curlVersion} is a MultiSSL build (${capabilities.tlsBackends.map((backend) => backend.name).join(", ")}); the active backend cannot be known from here`);
|
|
96
|
+
} else if (!OPENSSL_FAMILY.has(capabilities.tlsBackends[0].name)) {
|
|
97
|
+
reasons.push(`curl ${capabilities.curlVersion} uses ${capabilities.tlsBackends[0].name}, which does not take the profile's OpenSSL cipher and group lists`);
|
|
98
|
+
}
|
|
99
|
+
const wantsHttp2 = profile.tls.alpnProtocols.includes("h2");
|
|
100
|
+
if (wantsHttp2 && !capabilities.http2)
|
|
101
|
+
reasons.push(`profile offers h2 in ALPN but curl ${capabilities.curlVersion} was built without HTTP/2`);
|
|
102
|
+
if (reasons.length > 0)
|
|
103
|
+
return { ok: false, reasons };
|
|
104
|
+
const tlsArgs = stealthTlsArguments(profile, supportsKeyShareMarkers(capabilities.tlsBackends[0]));
|
|
105
|
+
const acceptance = await probeCurlTlsAcceptance(tlsArgs, curlCommand);
|
|
106
|
+
if (!acceptance.accepted) {
|
|
107
|
+
return { ok: false, reasons: [`curl ${capabilities.curlVersion} (${capabilities.tlsBackends[0].name}/${capabilities.tlsBackends[0].version ?? "?"}) rejects the profile's TLS material: ${acceptance.detail}`] };
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
ok: true,
|
|
111
|
+
mapping: { tlsArgs, httpVersionArg: wantsHttp2 ? "--http2" : "--http1.1", useHttp2Headers: wantsHttp2, notExpressible: CURL_ROUTE_BOUNDARIES }
|
|
112
|
+
};
|
|
113
|
+
}
|