recker 1.0.2-0 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +0 -2
- package/README.md +121 -72
- package/dist/cache/memory-storage.d.ts.map +1 -1
- package/dist/cache/memory-storage.js +7 -1
- package/dist/constants/http-status.d.ts +74 -0
- package/dist/constants/http-status.d.ts.map +1 -0
- package/dist/constants/http-status.js +156 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +6 -6
- package/dist/cookies/memory-cookie-jar.d.ts +31 -0
- package/dist/cookies/memory-cookie-jar.d.ts.map +1 -0
- package/dist/cookies/memory-cookie-jar.js +210 -0
- package/dist/core/client.d.ts +9 -0
- package/dist/core/client.d.ts.map +1 -1
- package/dist/core/client.js +252 -53
- package/dist/core/errors.d.ts +18 -2
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +66 -5
- package/dist/core/index.d.ts +6 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +5 -0
- package/dist/core/request-promise.d.ts.map +1 -1
- package/dist/core/request-promise.js +8 -2
- package/dist/core/request.d.ts +7 -1
- package/dist/core/request.d.ts.map +1 -1
- package/dist/core/request.js +32 -0
- package/dist/core/response.d.ts +2 -0
- package/dist/core/response.d.ts.map +1 -1
- package/dist/core/response.js +44 -19
- package/dist/events/request-events.d.ts +48 -0
- package/dist/events/request-events.d.ts.map +1 -0
- package/dist/events/request-events.js +85 -0
- package/dist/index.d.ts +28 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +28 -2
- package/dist/mcp/client.d.ts.map +1 -1
- package/dist/mcp/client.js +16 -5
- package/dist/mcp/contract.d.ts +77 -0
- package/dist/mcp/contract.d.ts.map +1 -0
- package/dist/mcp/contract.js +278 -0
- package/dist/mcp/types.d.ts +1 -0
- package/dist/mcp/types.d.ts.map +1 -1
- package/dist/plugins/auth.d.ts +45 -0
- package/dist/plugins/auth.d.ts.map +1 -0
- package/dist/plugins/auth.js +268 -0
- package/dist/plugins/cache.d.ts +7 -1
- package/dist/plugins/cache.d.ts.map +1 -1
- package/dist/plugins/cache.js +470 -49
- package/dist/plugins/circuit-breaker.js +1 -1
- package/dist/plugins/compression.d.ts.map +1 -1
- package/dist/plugins/compression.js +3 -3
- package/dist/plugins/dedup.d.ts.map +1 -1
- package/dist/plugins/dedup.js +2 -1
- package/dist/plugins/graphql.d.ts +4 -3
- package/dist/plugins/graphql.d.ts.map +1 -1
- package/dist/plugins/graphql.js +24 -5
- package/dist/plugins/grpc-web.d.ts +80 -0
- package/dist/plugins/grpc-web.d.ts.map +1 -0
- package/dist/plugins/grpc-web.js +261 -0
- package/dist/plugins/har-player.d.ts.map +1 -1
- package/dist/plugins/har-player.js +11 -2
- package/dist/plugins/hls.d.ts +33 -0
- package/dist/plugins/hls.d.ts.map +1 -0
- package/dist/plugins/hls.js +225 -0
- package/dist/plugins/http2-push.d.ts +64 -0
- package/dist/plugins/http2-push.d.ts.map +1 -0
- package/dist/plugins/http2-push.js +274 -0
- package/dist/plugins/http3.d.ts +76 -0
- package/dist/plugins/http3.d.ts.map +1 -0
- package/dist/plugins/http3.js +231 -0
- package/dist/plugins/interface-rotator.d.ts +10 -0
- package/dist/plugins/interface-rotator.d.ts.map +1 -0
- package/dist/plugins/interface-rotator.js +57 -0
- package/dist/plugins/jsonrpc.d.ts +76 -0
- package/dist/plugins/jsonrpc.d.ts.map +1 -0
- package/dist/plugins/jsonrpc.js +143 -0
- package/dist/plugins/logger.d.ts +8 -5
- package/dist/plugins/logger.d.ts.map +1 -1
- package/dist/plugins/logger.js +66 -30
- package/dist/plugins/odata.d.ts +182 -0
- package/dist/plugins/odata.d.ts.map +1 -0
- package/dist/plugins/odata.js +561 -0
- package/dist/plugins/retry.d.ts +1 -0
- package/dist/plugins/retry.d.ts.map +1 -1
- package/dist/plugins/retry.js +26 -2
- package/dist/plugins/scrape.d.ts +22 -0
- package/dist/plugins/scrape.d.ts.map +1 -0
- package/dist/plugins/scrape.js +87 -0
- package/dist/plugins/soap.d.ts +73 -0
- package/dist/plugins/soap.d.ts.map +1 -0
- package/dist/plugins/soap.js +347 -0
- package/dist/plugins/user-agent.d.ts +8 -0
- package/dist/plugins/user-agent.d.ts.map +1 -0
- package/dist/plugins/user-agent.js +46 -0
- package/dist/plugins/xml.d.ts +10 -0
- package/dist/plugins/xml.d.ts.map +1 -0
- package/dist/plugins/xml.js +194 -0
- package/dist/presets/anthropic.d.ts +7 -0
- package/dist/presets/anthropic.d.ts.map +1 -0
- package/dist/presets/anthropic.js +17 -0
- package/dist/presets/azure-openai.d.ts +9 -0
- package/dist/presets/azure-openai.d.ts.map +1 -0
- package/dist/presets/azure-openai.js +25 -0
- package/dist/presets/cloudflare.d.ts +13 -0
- package/dist/presets/cloudflare.d.ts.map +1 -0
- package/dist/presets/cloudflare.js +39 -0
- package/dist/presets/cohere.d.ts +6 -0
- package/dist/presets/cohere.d.ts.map +1 -0
- package/dist/presets/cohere.js +16 -0
- package/dist/presets/deepseek.d.ts +6 -0
- package/dist/presets/deepseek.d.ts.map +1 -0
- package/dist/presets/deepseek.js +16 -0
- package/dist/presets/digitalocean.d.ts +6 -0
- package/dist/presets/digitalocean.d.ts.map +1 -0
- package/dist/presets/digitalocean.js +16 -0
- package/dist/presets/discord.d.ts +7 -0
- package/dist/presets/discord.d.ts.map +1 -0
- package/dist/presets/discord.js +17 -0
- package/dist/presets/fireworks.d.ts +6 -0
- package/dist/presets/fireworks.d.ts.map +1 -0
- package/dist/presets/fireworks.js +16 -0
- package/dist/presets/gemini.d.ts +6 -0
- package/dist/presets/gemini.d.ts.map +1 -0
- package/dist/presets/gemini.js +16 -0
- package/dist/presets/github.d.ts +7 -0
- package/dist/presets/github.d.ts.map +1 -0
- package/dist/presets/github.js +17 -0
- package/dist/presets/gitlab.d.ts +7 -0
- package/dist/presets/gitlab.d.ts.map +1 -0
- package/dist/presets/gitlab.js +16 -0
- package/dist/presets/groq.d.ts +6 -0
- package/dist/presets/groq.d.ts.map +1 -0
- package/dist/presets/groq.js +16 -0
- package/dist/presets/huggingface.d.ts +6 -0
- package/dist/presets/huggingface.d.ts.map +1 -0
- package/dist/presets/huggingface.js +16 -0
- package/dist/presets/index.d.ts +28 -0
- package/dist/presets/index.d.ts.map +1 -0
- package/dist/presets/index.js +27 -0
- package/dist/presets/linear.d.ts +6 -0
- package/dist/presets/linear.d.ts.map +1 -0
- package/dist/presets/linear.js +16 -0
- package/dist/presets/mistral.d.ts +6 -0
- package/dist/presets/mistral.d.ts.map +1 -0
- package/dist/presets/mistral.js +16 -0
- package/dist/presets/notion.d.ts +7 -0
- package/dist/presets/notion.d.ts.map +1 -0
- package/dist/presets/notion.js +17 -0
- package/dist/presets/openai.d.ts +8 -0
- package/dist/presets/openai.d.ts.map +1 -0
- package/dist/presets/openai.js +23 -0
- package/dist/presets/perplexity.d.ts +6 -0
- package/dist/presets/perplexity.d.ts.map +1 -0
- package/dist/presets/perplexity.js +16 -0
- package/dist/presets/registry.d.ts +20 -0
- package/dist/presets/registry.d.ts.map +1 -0
- package/dist/presets/registry.js +311 -0
- package/dist/presets/replicate.d.ts +6 -0
- package/dist/presets/replicate.d.ts.map +1 -0
- package/dist/presets/replicate.js +16 -0
- package/dist/presets/slack.d.ts +6 -0
- package/dist/presets/slack.d.ts.map +1 -0
- package/dist/presets/slack.js +16 -0
- package/dist/presets/stripe.d.ts +8 -0
- package/dist/presets/stripe.d.ts.map +1 -0
- package/dist/presets/stripe.js +23 -0
- package/dist/presets/supabase.d.ts +7 -0
- package/dist/presets/supabase.d.ts.map +1 -0
- package/dist/presets/supabase.js +18 -0
- package/dist/presets/together.d.ts +6 -0
- package/dist/presets/together.d.ts.map +1 -0
- package/dist/presets/together.js +16 -0
- package/dist/presets/twilio.d.ts +7 -0
- package/dist/presets/twilio.d.ts.map +1 -0
- package/dist/presets/twilio.js +17 -0
- package/dist/presets/vercel.d.ts +7 -0
- package/dist/presets/vercel.d.ts.map +1 -0
- package/dist/presets/vercel.js +23 -0
- package/dist/presets/xai.d.ts +7 -0
- package/dist/presets/xai.d.ts.map +1 -0
- package/dist/presets/xai.js +17 -0
- package/dist/protocols/ftp.d.ts +63 -0
- package/dist/protocols/ftp.d.ts.map +1 -0
- package/dist/protocols/ftp.js +388 -0
- package/dist/protocols/index.d.ts +4 -0
- package/dist/protocols/index.d.ts.map +1 -0
- package/dist/protocols/index.js +3 -0
- package/dist/protocols/sftp.d.ts +65 -0
- package/dist/protocols/sftp.d.ts.map +1 -0
- package/dist/protocols/sftp.js +346 -0
- package/dist/protocols/telnet.d.ts +50 -0
- package/dist/protocols/telnet.d.ts.map +1 -0
- package/dist/protocols/telnet.js +139 -0
- package/dist/runner/request-runner.d.ts.map +1 -1
- package/dist/runner/request-runner.js +1 -0
- package/dist/scrape/document.d.ts +44 -0
- package/dist/scrape/document.d.ts.map +1 -0
- package/dist/scrape/document.js +198 -0
- package/dist/scrape/element.d.ts +50 -0
- package/dist/scrape/element.d.ts.map +1 -0
- package/dist/scrape/element.js +176 -0
- package/dist/scrape/extractors.d.ts +17 -0
- package/dist/scrape/extractors.d.ts.map +1 -0
- package/dist/scrape/extractors.js +356 -0
- package/dist/scrape/index.d.ts +5 -0
- package/dist/scrape/index.d.ts.map +1 -0
- package/dist/scrape/index.js +3 -0
- package/dist/scrape/types.d.ts +108 -0
- package/dist/scrape/types.d.ts.map +1 -0
- package/dist/scrape/types.js +1 -0
- package/dist/testing/index.d.ts +3 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +1 -0
- package/dist/testing/mock.d.ts +58 -0
- package/dist/testing/mock.d.ts.map +1 -0
- package/dist/testing/mock.js +252 -0
- package/dist/transport/fetch.d.ts.map +1 -1
- package/dist/transport/fetch.js +12 -4
- package/dist/transport/undici.d.ts +17 -1
- package/dist/transport/undici.d.ts.map +1 -1
- package/dist/transport/undici.js +708 -47
- package/dist/types/index.d.ts +111 -10
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -1
- package/dist/types/logger.d.ts +17 -0
- package/dist/types/logger.d.ts.map +1 -0
- package/dist/types/logger.js +66 -0
- package/dist/utils/agent-manager.d.ts.map +1 -1
- package/dist/utils/agent-manager.js +20 -4
- package/dist/utils/body.d.ts.map +1 -1
- package/dist/utils/body.js +14 -2
- package/dist/utils/charset.d.ts +16 -0
- package/dist/utils/charset.d.ts.map +1 -0
- package/dist/utils/charset.js +169 -0
- package/dist/utils/client-pool.d.ts +21 -0
- package/dist/utils/client-pool.d.ts.map +1 -0
- package/dist/utils/client-pool.js +49 -0
- package/dist/utils/concurrency.d.ts.map +1 -1
- package/dist/utils/concurrency.js +8 -4
- package/dist/utils/dns-toolkit.d.ts +13 -0
- package/dist/utils/dns-toolkit.d.ts.map +1 -0
- package/dist/utils/dns-toolkit.js +48 -0
- package/dist/utils/doh.d.ts.map +1 -1
- package/dist/utils/doh.js +16 -3
- package/dist/utils/download.d.ts +15 -0
- package/dist/utils/download.d.ts.map +1 -0
- package/dist/utils/download.js +44 -0
- package/dist/utils/env-proxy.d.ts +13 -0
- package/dist/utils/env-proxy.d.ts.map +1 -0
- package/dist/utils/env-proxy.js +105 -0
- package/dist/utils/header-parser.d.ts +15 -1
- package/dist/utils/header-parser.d.ts.map +1 -1
- package/dist/utils/header-parser.js +161 -1
- package/dist/utils/link-header.d.ts +70 -0
- package/dist/utils/link-header.d.ts.map +1 -0
- package/dist/utils/link-header.js +190 -0
- package/dist/utils/progress.d.ts +7 -2
- package/dist/utils/progress.d.ts.map +1 -1
- package/dist/utils/progress.js +48 -15
- package/dist/utils/rdap.d.ts +17 -0
- package/dist/utils/rdap.d.ts.map +1 -0
- package/dist/utils/rdap.js +32 -0
- package/dist/utils/request-pool.d.ts.map +1 -1
- package/dist/utils/request-pool.js +4 -3
- package/dist/utils/sse.d.ts.map +1 -1
- package/dist/utils/sse.js +8 -2
- package/dist/utils/status-codes.d.ts +84 -0
- package/dist/utils/status-codes.d.ts.map +1 -0
- package/dist/utils/status-codes.js +204 -0
- package/dist/utils/streaming.d.ts.map +1 -1
- package/dist/utils/streaming.js +1 -0
- package/dist/utils/tls-inspector.d.ts +21 -0
- package/dist/utils/tls-inspector.d.ts.map +1 -0
- package/dist/utils/tls-inspector.js +39 -0
- package/dist/utils/try-fn.d.ts.map +1 -1
- package/dist/utils/try-fn.js +11 -5
- package/dist/utils/upload.d.ts +1 -0
- package/dist/utils/upload.d.ts.map +1 -1
- package/dist/utils/upload.js +20 -3
- package/dist/utils/user-agent.d.ts +9 -9
- package/dist/utils/user-agent.js +9 -9
- package/dist/utils/whois.d.ts.map +1 -1
- package/dist/utils/whois.js +11 -2
- package/dist/websocket/client.d.ts +29 -1
- package/dist/websocket/client.d.ts.map +1 -1
- package/dist/websocket/client.js +145 -13
- package/package.json +45 -8
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
const cookieNameRegExp = /^[\u0021-\u003A\u003C\u003E-\u007E]+$/;
|
|
2
|
+
const cookieValueRegExp = /^[\u0021-\u003A\u003C-\u007E]*$/;
|
|
3
|
+
const domainValueRegExp = /^([.]?[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)([.][a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/i;
|
|
4
|
+
const pathValueRegExp = /^[\u0020-\u003A\u003D-\u007E]*$/;
|
|
5
|
+
export class MemoryCookieJar {
|
|
6
|
+
cookies = new Map();
|
|
7
|
+
getCookieString(url) {
|
|
8
|
+
const parsedUrl = new URL(url);
|
|
9
|
+
const hostname = parsedUrl.hostname;
|
|
10
|
+
const pathname = parsedUrl.pathname || '/';
|
|
11
|
+
const isSecure = parsedUrl.protocol === 'https:';
|
|
12
|
+
const now = Date.now();
|
|
13
|
+
const matchingCookies = [];
|
|
14
|
+
for (const [domain, pathMap] of this.cookies.entries()) {
|
|
15
|
+
if (!this.domainMatches(hostname, domain))
|
|
16
|
+
continue;
|
|
17
|
+
for (const [path, nameMap] of pathMap.entries()) {
|
|
18
|
+
if (!this.pathMatches(pathname, path))
|
|
19
|
+
continue;
|
|
20
|
+
for (const cookie of nameMap.values()) {
|
|
21
|
+
if (this.isExpired(cookie, now)) {
|
|
22
|
+
nameMap.delete(cookie.name);
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
25
|
+
if (cookie.secure && !isSecure)
|
|
26
|
+
continue;
|
|
27
|
+
matchingCookies.push(cookie);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
matchingCookies.sort((a, b) => {
|
|
32
|
+
const pathDiff = b.path.length - a.path.length;
|
|
33
|
+
if (pathDiff !== 0)
|
|
34
|
+
return pathDiff;
|
|
35
|
+
return a.createdAt - b.createdAt;
|
|
36
|
+
});
|
|
37
|
+
return matchingCookies.map(c => `${c.name}=${c.value}`).join('; ');
|
|
38
|
+
}
|
|
39
|
+
setCookie(rawCookie, url) {
|
|
40
|
+
const parsedUrl = new URL(url);
|
|
41
|
+
const hostname = parsedUrl.hostname;
|
|
42
|
+
const pathname = parsedUrl.pathname || '/';
|
|
43
|
+
const cookie = this.parseCookie(rawCookie, hostname, pathname);
|
|
44
|
+
if (!cookie)
|
|
45
|
+
return;
|
|
46
|
+
if (!this.domainMatches(hostname, cookie.domain)) {
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
if (!this.cookies.has(cookie.domain)) {
|
|
50
|
+
this.cookies.set(cookie.domain, new Map());
|
|
51
|
+
}
|
|
52
|
+
const domainMap = this.cookies.get(cookie.domain);
|
|
53
|
+
if (!domainMap.has(cookie.path)) {
|
|
54
|
+
domainMap.set(cookie.path, new Map());
|
|
55
|
+
}
|
|
56
|
+
const pathMap = domainMap.get(cookie.path);
|
|
57
|
+
pathMap.set(cookie.name, cookie);
|
|
58
|
+
}
|
|
59
|
+
clear() {
|
|
60
|
+
this.cookies.clear();
|
|
61
|
+
}
|
|
62
|
+
clearDomain(domain) {
|
|
63
|
+
this.cookies.delete(domain);
|
|
64
|
+
}
|
|
65
|
+
getAllCookies() {
|
|
66
|
+
const all = [];
|
|
67
|
+
for (const domainMap of this.cookies.values()) {
|
|
68
|
+
for (const pathMap of domainMap.values()) {
|
|
69
|
+
for (const cookie of pathMap.values()) {
|
|
70
|
+
all.push(cookie);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return all;
|
|
75
|
+
}
|
|
76
|
+
parseCookie(setCookieStr, requestDomain, requestPath) {
|
|
77
|
+
const parts = setCookieStr.split(';').map(p => p.trim());
|
|
78
|
+
if (parts.length === 0 || !parts[0])
|
|
79
|
+
return null;
|
|
80
|
+
const [nameValue, ...attributes] = parts;
|
|
81
|
+
const eqIndex = nameValue.indexOf('=');
|
|
82
|
+
if (eqIndex === -1)
|
|
83
|
+
return null;
|
|
84
|
+
const name = nameValue.substring(0, eqIndex).trim();
|
|
85
|
+
const rawValue = nameValue.substring(eqIndex + 1).trim();
|
|
86
|
+
if (!name)
|
|
87
|
+
return null;
|
|
88
|
+
const value = this.decode(rawValue);
|
|
89
|
+
const cookie = {
|
|
90
|
+
name,
|
|
91
|
+
value,
|
|
92
|
+
domain: requestDomain,
|
|
93
|
+
path: this.defaultPath(requestPath),
|
|
94
|
+
secure: false,
|
|
95
|
+
httpOnly: false,
|
|
96
|
+
createdAt: Date.now()
|
|
97
|
+
};
|
|
98
|
+
for (const attr of attributes) {
|
|
99
|
+
const lowerAttr = attr.toLowerCase();
|
|
100
|
+
if (lowerAttr === 'secure') {
|
|
101
|
+
cookie.secure = true;
|
|
102
|
+
}
|
|
103
|
+
else if (lowerAttr === 'httponly') {
|
|
104
|
+
cookie.httpOnly = true;
|
|
105
|
+
}
|
|
106
|
+
else if (lowerAttr === 'partitioned') {
|
|
107
|
+
cookie.partitioned = true;
|
|
108
|
+
}
|
|
109
|
+
else if (lowerAttr.startsWith('domain=')) {
|
|
110
|
+
let domain = attr.substring(7).trim();
|
|
111
|
+
if (domain.startsWith('.')) {
|
|
112
|
+
domain = domain.substring(1);
|
|
113
|
+
}
|
|
114
|
+
if (domainValueRegExp.test(domain)) {
|
|
115
|
+
cookie.domain = domain.toLowerCase();
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
else if (lowerAttr.startsWith('path=')) {
|
|
119
|
+
const path = attr.substring(5).trim();
|
|
120
|
+
if (!path || pathValueRegExp.test(path)) {
|
|
121
|
+
cookie.path = path || '/';
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
else if (lowerAttr.startsWith('expires=')) {
|
|
125
|
+
const expiresStr = attr.substring(8).trim();
|
|
126
|
+
const expires = new Date(expiresStr);
|
|
127
|
+
if (Number.isFinite(expires.valueOf())) {
|
|
128
|
+
cookie.expires = expires;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
else if (lowerAttr.startsWith('max-age=')) {
|
|
132
|
+
const maxAgeStr = attr.substring(8).trim();
|
|
133
|
+
if (/^-?\d+$/.test(maxAgeStr)) {
|
|
134
|
+
cookie.maxAge = parseInt(maxAgeStr, 10);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
else if (lowerAttr.startsWith('samesite=')) {
|
|
138
|
+
const samesite = attr.substring(9).trim().toLowerCase();
|
|
139
|
+
if (samesite === 'strict')
|
|
140
|
+
cookie.sameSite = 'Strict';
|
|
141
|
+
else if (samesite === 'lax')
|
|
142
|
+
cookie.sameSite = 'Lax';
|
|
143
|
+
else if (samesite === 'none')
|
|
144
|
+
cookie.sameSite = 'None';
|
|
145
|
+
}
|
|
146
|
+
else if (lowerAttr.startsWith('priority=')) {
|
|
147
|
+
const priority = attr.substring(9).trim().toLowerCase();
|
|
148
|
+
if (priority === 'low')
|
|
149
|
+
cookie.priority = 'Low';
|
|
150
|
+
else if (priority === 'medium')
|
|
151
|
+
cookie.priority = 'Medium';
|
|
152
|
+
else if (priority === 'high')
|
|
153
|
+
cookie.priority = 'High';
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return cookie;
|
|
157
|
+
}
|
|
158
|
+
decode(str) {
|
|
159
|
+
if (str.indexOf('%') === -1)
|
|
160
|
+
return str;
|
|
161
|
+
try {
|
|
162
|
+
return decodeURIComponent(str);
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
return str;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
domainMatches(requestDomain, cookieDomain) {
|
|
169
|
+
const reqLower = requestDomain.toLowerCase();
|
|
170
|
+
const cookieLower = cookieDomain.toLowerCase();
|
|
171
|
+
if (reqLower === cookieLower)
|
|
172
|
+
return true;
|
|
173
|
+
if (reqLower.endsWith('.' + cookieLower))
|
|
174
|
+
return true;
|
|
175
|
+
return false;
|
|
176
|
+
}
|
|
177
|
+
pathMatches(requestPath, cookiePath) {
|
|
178
|
+
if (requestPath === cookiePath)
|
|
179
|
+
return true;
|
|
180
|
+
if (requestPath.startsWith(cookiePath)) {
|
|
181
|
+
if (cookiePath.endsWith('/'))
|
|
182
|
+
return true;
|
|
183
|
+
if (requestPath[cookiePath.length] === '/')
|
|
184
|
+
return true;
|
|
185
|
+
}
|
|
186
|
+
return false;
|
|
187
|
+
}
|
|
188
|
+
defaultPath(requestPath) {
|
|
189
|
+
if (!requestPath || !requestPath.startsWith('/')) {
|
|
190
|
+
return '/';
|
|
191
|
+
}
|
|
192
|
+
const lastSlash = requestPath.lastIndexOf('/');
|
|
193
|
+
if (lastSlash === 0) {
|
|
194
|
+
return '/';
|
|
195
|
+
}
|
|
196
|
+
return requestPath.substring(0, lastSlash);
|
|
197
|
+
}
|
|
198
|
+
isExpired(cookie, now) {
|
|
199
|
+
if (cookie.maxAge !== undefined) {
|
|
200
|
+
if (cookie.maxAge <= 0)
|
|
201
|
+
return true;
|
|
202
|
+
const expiresAt = cookie.createdAt + (cookie.maxAge * 1000);
|
|
203
|
+
return now >= expiresAt;
|
|
204
|
+
}
|
|
205
|
+
if (cookie.expires) {
|
|
206
|
+
return now >= cookie.expires.getTime();
|
|
207
|
+
}
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
}
|
package/dist/core/client.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { CacheOptions } from '../plugins/cache.js';
|
|
|
6
6
|
import { DedupOptions } from '../plugins/dedup.js';
|
|
7
7
|
import { ReckerWebSocket, type WebSocketOptions } from '../websocket/client.js';
|
|
8
8
|
import { type WhoisOptions, type WhoisResult } from '../utils/whois.js';
|
|
9
|
+
import { type ScrapePromise } from '../plugins/scrape.js';
|
|
9
10
|
interface ClientCacheConfig extends Omit<CacheOptions, 'storage'> {
|
|
10
11
|
storage?: CacheStorage;
|
|
11
12
|
driver?: 'memory' | 'file';
|
|
@@ -30,8 +31,15 @@ export declare class Client {
|
|
|
30
31
|
private agentManager?;
|
|
31
32
|
private concurrencyConfig;
|
|
32
33
|
private requestPool?;
|
|
34
|
+
private maxResponseSize?;
|
|
35
|
+
private cookieJar?;
|
|
36
|
+
private cookieIgnoreInvalid;
|
|
33
37
|
constructor(options?: ExtendedClientOptions);
|
|
34
38
|
private createLoggingMiddleware;
|
|
39
|
+
private createMaxSizeMiddleware;
|
|
40
|
+
private setupCookieJar;
|
|
41
|
+
private createCookieMiddleware;
|
|
42
|
+
private splitSetCookieHeader;
|
|
35
43
|
private composeMiddlewares;
|
|
36
44
|
private hooksMiddleware;
|
|
37
45
|
private httpErrorMiddleware;
|
|
@@ -91,6 +99,7 @@ export declare class Client {
|
|
|
91
99
|
unlock<T = unknown>(path: string, options?: Omit<RequestOptions, 'method'>): RequestPromise<T>;
|
|
92
100
|
link<T = unknown>(path: string, body?: any, options?: Omit<RequestOptions, 'method' | 'body'>): RequestPromise<T>;
|
|
93
101
|
unlink<T = unknown>(path: string, body?: any, options?: Omit<RequestOptions, 'method' | 'body'>): RequestPromise<T>;
|
|
102
|
+
scrape(path: string, options?: RequestOptions): ScrapePromise<ReckerResponse>;
|
|
94
103
|
paginate<T>(path: string, options?: RequestOptions & PaginationOptions<T>): AsyncGenerator<T>;
|
|
95
104
|
pages<T = any>(path: string, options?: RequestOptions & PaginationOptions): AsyncGenerator<PageResult<T>>;
|
|
96
105
|
page<T = any>(path: string, pageNumber: number, options?: RequestOptions & {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/core/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,UAAU,EAAgB,aAAa,EAAE,cAAc,EAAE,cAAc,EAAa,YAAY,EAAyC,UAAU,
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/core/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,UAAU,EAAgB,aAAa,EAAE,cAAc,EAAE,cAAc,EAAa,YAAY,EAAyC,UAAU,EAAmD,MAAM,mBAAmB,CAAC;AAIxP,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAStD,OAAO,EAAY,iBAAiB,EAAe,MAAM,0BAA0B,CAAC;AACpF,OAAO,EAAS,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAS,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAS,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAO1D,OAAO,EAAE,eAAe,EAAE,KAAK,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAChF,OAAO,EAA4C,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAElH,OAAO,EAA0B,KAAK,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIlF,UAAU,iBAAkB,SAAQ,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC;IAC/D,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,MAAM,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAGD,MAAM,WAAW,qBAAsB,SAAQ,aAAa;IAC1D,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,KAAK,CAAC,EAAE,iBAAiB,CAAC;IAC1B,KAAK,CAAC,EAAE,YAAY,CAAC;CACtB;AAED,qBAAa,MAAM;IACjB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,cAAc,CAAc;IACpC,OAAO,CAAC,aAAa,CAAkC;IACvD,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,OAAO,CAAkD;IACjE,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,YAAY,CAAU;IAC9B,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,iBAAiB,CAA8B;IACvD,OAAO,CAAC,WAAW,CAAC,CAAc;IAClC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,SAAS,CAAC,CAAY;IAC9B,OAAO,CAAC,mBAAmB,CAAkB;gBAEjC,OAAO,GAAE,qBAA0B;IAyK/C,OAAO,CAAC,uBAAuB;IA+C/B,OAAO,CAAC,uBAAuB;IA2B/B,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,sBAAsB;IAgD9B,OAAO,CAAC,oBAAoB;IAM5B,OAAO,CAAC,kBAAkB;IAqB1B,OAAO,CAAC,eAAe,CAyCtB;IAED,OAAO,CAAC,mBAAmB,CAQ1B;IAEM,GAAG,CAAC,UAAU,EAAE,UAAU;IAW1B,aAAa,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,aAAa,KAAK,aAAa,GAAG,IAAI,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAchG,aAAa,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,cAAc,KAAK,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAcvH,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,KAAK,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAajH,OAAO,CAAC,QAAQ;IA+EhB,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,cAAc,CAAC,CAAC,CAAC;IAoFnF,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAoDrE,KAAK,CAAC,CAAC,GAAG,cAAc,EAC5B,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,cAAc,CAAA;KAAE,CAAC,EAC3D,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;KAAO,GAC5F,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC;QAAC,KAAK,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAC;YAAC,QAAQ,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;IA4BtH,KAAK,CAAC,CAAC,GAAG,cAAc,EACtB,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,cAAc,CAAA;KAAE,CAAC,EAC3D,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;KAAO;;eA9BnD;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAC;YAAC,QAAQ,EAAE,MAAM,CAAA;SAAE;;IAwCnH,OAAO,CAAC,eAAe;IAiFvB,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,MAAM,CAAM;IAIjG,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,MAAM,CAAM;IAIhG,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,MAAM,CAAM;IAIlG,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAI9E,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAI5E,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAQ/E,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAQ7E,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAc/E,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAc7E,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,MAAM,CAAM;IAOrG,SAAS,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,MAAM,CAAM;IAYtG,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAc7E,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAc5E,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAQ5E,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,MAAM,CAAM;IAQjG,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAM;IAQ9E,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,MAAM,CAAM;IAQjG,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,MAAM,CAAM;IAwCnG,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,aAAa,CAAC,cAAc,CAAC;IAMjF,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,cAAc,GAAG,iBAAiB,CAAC,CAAC,CAAM,GAAG,cAAc,CAAC,CAAC,CAAC;IAqBjG,KAAK,CAAC,CAAC,GAAG,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,cAAc,GAAG,iBAAsB,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAiB7G,IAAI,CAAC,CAAC,GAAG,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,GAAE,cAAc,GAAG;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,cAAc,CAAC,CAAC,CAAC;IAanH,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,cAAc,GAAG,iBAAiB,CAAC,CAAC,CAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IA2BhG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,eAAe;IAyCxE,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,eAAe;IAiB3D,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC;IAclE,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC;CAGlF;AAED,wBAAgB,YAAY,CAAC,OAAO,GAAE,qBAA0B,UAE/D"}
|