recker 1.0.2-0 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +0 -2
- package/README.md +121 -72
- package/dist/cache/memory-storage.d.ts.map +1 -1
- package/dist/cache/memory-storage.js +7 -1
- package/dist/constants/http-status.d.ts +74 -0
- package/dist/constants/http-status.d.ts.map +1 -0
- package/dist/constants/http-status.js +156 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +6 -6
- package/dist/cookies/memory-cookie-jar.d.ts +31 -0
- package/dist/cookies/memory-cookie-jar.d.ts.map +1 -0
- package/dist/cookies/memory-cookie-jar.js +210 -0
- package/dist/core/client.d.ts +9 -0
- package/dist/core/client.d.ts.map +1 -1
- package/dist/core/client.js +252 -53
- package/dist/core/errors.d.ts +18 -2
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +66 -5
- package/dist/core/index.d.ts +6 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +5 -0
- package/dist/core/request-promise.d.ts.map +1 -1
- package/dist/core/request-promise.js +8 -2
- package/dist/core/request.d.ts +7 -1
- package/dist/core/request.d.ts.map +1 -1
- package/dist/core/request.js +32 -0
- package/dist/core/response.d.ts +2 -0
- package/dist/core/response.d.ts.map +1 -1
- package/dist/core/response.js +44 -19
- package/dist/events/request-events.d.ts +48 -0
- package/dist/events/request-events.d.ts.map +1 -0
- package/dist/events/request-events.js +85 -0
- package/dist/index.d.ts +28 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +28 -2
- package/dist/mcp/client.d.ts.map +1 -1
- package/dist/mcp/client.js +16 -5
- package/dist/mcp/contract.d.ts +77 -0
- package/dist/mcp/contract.d.ts.map +1 -0
- package/dist/mcp/contract.js +278 -0
- package/dist/mcp/types.d.ts +1 -0
- package/dist/mcp/types.d.ts.map +1 -1
- package/dist/plugins/auth.d.ts +45 -0
- package/dist/plugins/auth.d.ts.map +1 -0
- package/dist/plugins/auth.js +268 -0
- package/dist/plugins/cache.d.ts +7 -1
- package/dist/plugins/cache.d.ts.map +1 -1
- package/dist/plugins/cache.js +470 -49
- package/dist/plugins/circuit-breaker.js +1 -1
- package/dist/plugins/compression.d.ts.map +1 -1
- package/dist/plugins/compression.js +3 -3
- package/dist/plugins/dedup.d.ts.map +1 -1
- package/dist/plugins/dedup.js +2 -1
- package/dist/plugins/graphql.d.ts +4 -3
- package/dist/plugins/graphql.d.ts.map +1 -1
- package/dist/plugins/graphql.js +24 -5
- package/dist/plugins/grpc-web.d.ts +80 -0
- package/dist/plugins/grpc-web.d.ts.map +1 -0
- package/dist/plugins/grpc-web.js +261 -0
- package/dist/plugins/har-player.d.ts.map +1 -1
- package/dist/plugins/har-player.js +11 -2
- package/dist/plugins/hls.d.ts +33 -0
- package/dist/plugins/hls.d.ts.map +1 -0
- package/dist/plugins/hls.js +225 -0
- package/dist/plugins/http2-push.d.ts +64 -0
- package/dist/plugins/http2-push.d.ts.map +1 -0
- package/dist/plugins/http2-push.js +274 -0
- package/dist/plugins/http3.d.ts +76 -0
- package/dist/plugins/http3.d.ts.map +1 -0
- package/dist/plugins/http3.js +231 -0
- package/dist/plugins/interface-rotator.d.ts +10 -0
- package/dist/plugins/interface-rotator.d.ts.map +1 -0
- package/dist/plugins/interface-rotator.js +57 -0
- package/dist/plugins/jsonrpc.d.ts +76 -0
- package/dist/plugins/jsonrpc.d.ts.map +1 -0
- package/dist/plugins/jsonrpc.js +143 -0
- package/dist/plugins/logger.d.ts +8 -5
- package/dist/plugins/logger.d.ts.map +1 -1
- package/dist/plugins/logger.js +66 -30
- package/dist/plugins/odata.d.ts +182 -0
- package/dist/plugins/odata.d.ts.map +1 -0
- package/dist/plugins/odata.js +561 -0
- package/dist/plugins/retry.d.ts +1 -0
- package/dist/plugins/retry.d.ts.map +1 -1
- package/dist/plugins/retry.js +26 -2
- package/dist/plugins/scrape.d.ts +22 -0
- package/dist/plugins/scrape.d.ts.map +1 -0
- package/dist/plugins/scrape.js +87 -0
- package/dist/plugins/soap.d.ts +73 -0
- package/dist/plugins/soap.d.ts.map +1 -0
- package/dist/plugins/soap.js +347 -0
- package/dist/plugins/user-agent.d.ts +8 -0
- package/dist/plugins/user-agent.d.ts.map +1 -0
- package/dist/plugins/user-agent.js +46 -0
- package/dist/plugins/xml.d.ts +10 -0
- package/dist/plugins/xml.d.ts.map +1 -0
- package/dist/plugins/xml.js +194 -0
- package/dist/presets/anthropic.d.ts +7 -0
- package/dist/presets/anthropic.d.ts.map +1 -0
- package/dist/presets/anthropic.js +17 -0
- package/dist/presets/azure-openai.d.ts +9 -0
- package/dist/presets/azure-openai.d.ts.map +1 -0
- package/dist/presets/azure-openai.js +25 -0
- package/dist/presets/cloudflare.d.ts +13 -0
- package/dist/presets/cloudflare.d.ts.map +1 -0
- package/dist/presets/cloudflare.js +39 -0
- package/dist/presets/cohere.d.ts +6 -0
- package/dist/presets/cohere.d.ts.map +1 -0
- package/dist/presets/cohere.js +16 -0
- package/dist/presets/deepseek.d.ts +6 -0
- package/dist/presets/deepseek.d.ts.map +1 -0
- package/dist/presets/deepseek.js +16 -0
- package/dist/presets/digitalocean.d.ts +6 -0
- package/dist/presets/digitalocean.d.ts.map +1 -0
- package/dist/presets/digitalocean.js +16 -0
- package/dist/presets/discord.d.ts +7 -0
- package/dist/presets/discord.d.ts.map +1 -0
- package/dist/presets/discord.js +17 -0
- package/dist/presets/fireworks.d.ts +6 -0
- package/dist/presets/fireworks.d.ts.map +1 -0
- package/dist/presets/fireworks.js +16 -0
- package/dist/presets/gemini.d.ts +6 -0
- package/dist/presets/gemini.d.ts.map +1 -0
- package/dist/presets/gemini.js +16 -0
- package/dist/presets/github.d.ts +7 -0
- package/dist/presets/github.d.ts.map +1 -0
- package/dist/presets/github.js +17 -0
- package/dist/presets/gitlab.d.ts +7 -0
- package/dist/presets/gitlab.d.ts.map +1 -0
- package/dist/presets/gitlab.js +16 -0
- package/dist/presets/groq.d.ts +6 -0
- package/dist/presets/groq.d.ts.map +1 -0
- package/dist/presets/groq.js +16 -0
- package/dist/presets/huggingface.d.ts +6 -0
- package/dist/presets/huggingface.d.ts.map +1 -0
- package/dist/presets/huggingface.js +16 -0
- package/dist/presets/index.d.ts +28 -0
- package/dist/presets/index.d.ts.map +1 -0
- package/dist/presets/index.js +27 -0
- package/dist/presets/linear.d.ts +6 -0
- package/dist/presets/linear.d.ts.map +1 -0
- package/dist/presets/linear.js +16 -0
- package/dist/presets/mistral.d.ts +6 -0
- package/dist/presets/mistral.d.ts.map +1 -0
- package/dist/presets/mistral.js +16 -0
- package/dist/presets/notion.d.ts +7 -0
- package/dist/presets/notion.d.ts.map +1 -0
- package/dist/presets/notion.js +17 -0
- package/dist/presets/openai.d.ts +8 -0
- package/dist/presets/openai.d.ts.map +1 -0
- package/dist/presets/openai.js +23 -0
- package/dist/presets/perplexity.d.ts +6 -0
- package/dist/presets/perplexity.d.ts.map +1 -0
- package/dist/presets/perplexity.js +16 -0
- package/dist/presets/registry.d.ts +20 -0
- package/dist/presets/registry.d.ts.map +1 -0
- package/dist/presets/registry.js +311 -0
- package/dist/presets/replicate.d.ts +6 -0
- package/dist/presets/replicate.d.ts.map +1 -0
- package/dist/presets/replicate.js +16 -0
- package/dist/presets/slack.d.ts +6 -0
- package/dist/presets/slack.d.ts.map +1 -0
- package/dist/presets/slack.js +16 -0
- package/dist/presets/stripe.d.ts +8 -0
- package/dist/presets/stripe.d.ts.map +1 -0
- package/dist/presets/stripe.js +23 -0
- package/dist/presets/supabase.d.ts +7 -0
- package/dist/presets/supabase.d.ts.map +1 -0
- package/dist/presets/supabase.js +18 -0
- package/dist/presets/together.d.ts +6 -0
- package/dist/presets/together.d.ts.map +1 -0
- package/dist/presets/together.js +16 -0
- package/dist/presets/twilio.d.ts +7 -0
- package/dist/presets/twilio.d.ts.map +1 -0
- package/dist/presets/twilio.js +17 -0
- package/dist/presets/vercel.d.ts +7 -0
- package/dist/presets/vercel.d.ts.map +1 -0
- package/dist/presets/vercel.js +23 -0
- package/dist/presets/xai.d.ts +7 -0
- package/dist/presets/xai.d.ts.map +1 -0
- package/dist/presets/xai.js +17 -0
- package/dist/protocols/ftp.d.ts +63 -0
- package/dist/protocols/ftp.d.ts.map +1 -0
- package/dist/protocols/ftp.js +388 -0
- package/dist/protocols/index.d.ts +4 -0
- package/dist/protocols/index.d.ts.map +1 -0
- package/dist/protocols/index.js +3 -0
- package/dist/protocols/sftp.d.ts +65 -0
- package/dist/protocols/sftp.d.ts.map +1 -0
- package/dist/protocols/sftp.js +346 -0
- package/dist/protocols/telnet.d.ts +50 -0
- package/dist/protocols/telnet.d.ts.map +1 -0
- package/dist/protocols/telnet.js +139 -0
- package/dist/runner/request-runner.d.ts.map +1 -1
- package/dist/runner/request-runner.js +1 -0
- package/dist/scrape/document.d.ts +44 -0
- package/dist/scrape/document.d.ts.map +1 -0
- package/dist/scrape/document.js +198 -0
- package/dist/scrape/element.d.ts +50 -0
- package/dist/scrape/element.d.ts.map +1 -0
- package/dist/scrape/element.js +176 -0
- package/dist/scrape/extractors.d.ts +17 -0
- package/dist/scrape/extractors.d.ts.map +1 -0
- package/dist/scrape/extractors.js +356 -0
- package/dist/scrape/index.d.ts +5 -0
- package/dist/scrape/index.d.ts.map +1 -0
- package/dist/scrape/index.js +3 -0
- package/dist/scrape/types.d.ts +108 -0
- package/dist/scrape/types.d.ts.map +1 -0
- package/dist/scrape/types.js +1 -0
- package/dist/testing/index.d.ts +3 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +1 -0
- package/dist/testing/mock.d.ts +58 -0
- package/dist/testing/mock.d.ts.map +1 -0
- package/dist/testing/mock.js +252 -0
- package/dist/transport/fetch.d.ts.map +1 -1
- package/dist/transport/fetch.js +12 -4
- package/dist/transport/undici.d.ts +17 -1
- package/dist/transport/undici.d.ts.map +1 -1
- package/dist/transport/undici.js +708 -47
- package/dist/types/index.d.ts +111 -10
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -1
- package/dist/types/logger.d.ts +17 -0
- package/dist/types/logger.d.ts.map +1 -0
- package/dist/types/logger.js +66 -0
- package/dist/utils/agent-manager.d.ts.map +1 -1
- package/dist/utils/agent-manager.js +20 -4
- package/dist/utils/body.d.ts.map +1 -1
- package/dist/utils/body.js +14 -2
- package/dist/utils/charset.d.ts +16 -0
- package/dist/utils/charset.d.ts.map +1 -0
- package/dist/utils/charset.js +169 -0
- package/dist/utils/client-pool.d.ts +21 -0
- package/dist/utils/client-pool.d.ts.map +1 -0
- package/dist/utils/client-pool.js +49 -0
- package/dist/utils/concurrency.d.ts.map +1 -1
- package/dist/utils/concurrency.js +8 -4
- package/dist/utils/dns-toolkit.d.ts +13 -0
- package/dist/utils/dns-toolkit.d.ts.map +1 -0
- package/dist/utils/dns-toolkit.js +48 -0
- package/dist/utils/doh.d.ts.map +1 -1
- package/dist/utils/doh.js +16 -3
- package/dist/utils/download.d.ts +15 -0
- package/dist/utils/download.d.ts.map +1 -0
- package/dist/utils/download.js +44 -0
- package/dist/utils/env-proxy.d.ts +13 -0
- package/dist/utils/env-proxy.d.ts.map +1 -0
- package/dist/utils/env-proxy.js +105 -0
- package/dist/utils/header-parser.d.ts +15 -1
- package/dist/utils/header-parser.d.ts.map +1 -1
- package/dist/utils/header-parser.js +161 -1
- package/dist/utils/link-header.d.ts +70 -0
- package/dist/utils/link-header.d.ts.map +1 -0
- package/dist/utils/link-header.js +190 -0
- package/dist/utils/progress.d.ts +7 -2
- package/dist/utils/progress.d.ts.map +1 -1
- package/dist/utils/progress.js +48 -15
- package/dist/utils/rdap.d.ts +17 -0
- package/dist/utils/rdap.d.ts.map +1 -0
- package/dist/utils/rdap.js +32 -0
- package/dist/utils/request-pool.d.ts.map +1 -1
- package/dist/utils/request-pool.js +4 -3
- package/dist/utils/sse.d.ts.map +1 -1
- package/dist/utils/sse.js +8 -2
- package/dist/utils/status-codes.d.ts +84 -0
- package/dist/utils/status-codes.d.ts.map +1 -0
- package/dist/utils/status-codes.js +204 -0
- package/dist/utils/streaming.d.ts.map +1 -1
- package/dist/utils/streaming.js +1 -0
- package/dist/utils/tls-inspector.d.ts +21 -0
- package/dist/utils/tls-inspector.d.ts.map +1 -0
- package/dist/utils/tls-inspector.js +39 -0
- package/dist/utils/try-fn.d.ts.map +1 -1
- package/dist/utils/try-fn.js +11 -5
- package/dist/utils/upload.d.ts +1 -0
- package/dist/utils/upload.d.ts.map +1 -1
- package/dist/utils/upload.js +20 -3
- package/dist/utils/user-agent.d.ts +9 -9
- package/dist/utils/user-agent.js +9 -9
- package/dist/utils/whois.d.ts.map +1 -1
- package/dist/utils/whois.js +11 -2
- package/dist/websocket/client.d.ts +29 -1
- package/dist/websocket/client.d.ts.map +1 -1
- package/dist/websocket/client.js +145 -13
- package/package.json +45 -8
package/dist/core/client.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
+
import { consoleLogger } from '../types/index.js';
|
|
1
2
|
import { HttpRequest } from './request.js';
|
|
2
3
|
import { UndiciTransport } from '../transport/undici.js';
|
|
3
4
|
import { RequestPromise } from './request-promise.js';
|
|
4
|
-
import { HttpError } from '../core/errors.js';
|
|
5
|
-
import {
|
|
6
|
-
import { processBody } from '../utils/body.js';
|
|
5
|
+
import { HttpError, MaxSizeExceededError, ReckerError } from '../core/errors.js';
|
|
6
|
+
import { processBody, createFormData, isPlainObject } from '../utils/body.js';
|
|
7
7
|
import { AgentManager } from '../utils/agent-manager.js';
|
|
8
8
|
import { RequestPool } from '../utils/request-pool.js';
|
|
9
9
|
import { normalizeConcurrency } from '../utils/concurrency.js';
|
|
@@ -14,11 +14,14 @@ import { cache } from '../plugins/cache.js';
|
|
|
14
14
|
import { dedup } from '../plugins/dedup.js';
|
|
15
15
|
import { createXSRFMiddleware } from '../plugins/xsrf.js';
|
|
16
16
|
import { createCompressionMiddleware } from '../plugins/compression.js';
|
|
17
|
+
import { serializeXML } from '../plugins/xml.js';
|
|
17
18
|
import { MemoryStorage } from '../cache/memory-storage.js';
|
|
18
19
|
import { FileStorage } from '../cache/file-storage.js';
|
|
19
20
|
import { RequestRunner } from '../runner/request-runner.js';
|
|
20
21
|
import { ReckerWebSocket } from '../websocket/client.js';
|
|
21
22
|
import { whois as performWhois, isDomainAvailable } from '../utils/whois.js';
|
|
23
|
+
import { MemoryCookieJar } from '../cookies/memory-cookie-jar.js';
|
|
24
|
+
import { scrape as scrapeHelper } from '../plugins/scrape.js';
|
|
22
25
|
export class Client {
|
|
23
26
|
baseUrl;
|
|
24
27
|
middlewares;
|
|
@@ -33,6 +36,9 @@ export class Client {
|
|
|
33
36
|
agentManager;
|
|
34
37
|
concurrencyConfig;
|
|
35
38
|
requestPool;
|
|
39
|
+
maxResponseSize;
|
|
40
|
+
cookieJar;
|
|
41
|
+
cookieIgnoreInvalid = false;
|
|
36
42
|
constructor(options = {}) {
|
|
37
43
|
this.baseUrl = options.baseUrl || '';
|
|
38
44
|
this.middlewares = options.middlewares || [];
|
|
@@ -49,9 +55,13 @@ export class Client {
|
|
|
49
55
|
};
|
|
50
56
|
this.defaultParams = options.defaults?.params || {};
|
|
51
57
|
this.paginationConfig = options.pagination;
|
|
58
|
+
this.maxResponseSize = options.maxResponseSize;
|
|
52
59
|
this.debugEnabled = options.debug === true;
|
|
53
60
|
if (this.debugEnabled) {
|
|
54
|
-
this.logger =
|
|
61
|
+
this.logger = options.logger ?? consoleLogger;
|
|
62
|
+
}
|
|
63
|
+
else if (options.logger) {
|
|
64
|
+
this.logger = options.logger;
|
|
55
65
|
}
|
|
56
66
|
this.concurrencyConfig = normalizeConcurrency({
|
|
57
67
|
concurrency: options.concurrency,
|
|
@@ -75,11 +85,18 @@ export class Client {
|
|
|
75
85
|
proxy: options.proxy,
|
|
76
86
|
http2: http2Options,
|
|
77
87
|
dns: options.dns,
|
|
78
|
-
agent: this.agentManager
|
|
88
|
+
agent: this.agentManager,
|
|
89
|
+
socketPath: options.socketPath,
|
|
90
|
+
tls: options.tls,
|
|
91
|
+
observability: options.observability
|
|
79
92
|
});
|
|
80
93
|
}
|
|
81
94
|
else {
|
|
82
|
-
throw new
|
|
95
|
+
throw new ReckerError('baseUrl is required for default UndiciTransport, or provide a custom transport.', undefined, undefined, [
|
|
96
|
+
'Set baseUrl when using the built-in Undici transport.',
|
|
97
|
+
'Pass an absolute URL to each request.',
|
|
98
|
+
'Provide a custom transport if you need to handle relative paths differently.'
|
|
99
|
+
]);
|
|
83
100
|
}
|
|
84
101
|
if (options.retry) {
|
|
85
102
|
retry(options.retry)(this);
|
|
@@ -91,13 +108,13 @@ export class Client {
|
|
|
91
108
|
interval: this.concurrencyConfig.interval
|
|
92
109
|
});
|
|
93
110
|
this.middlewares.unshift(this.requestPool.asMiddleware());
|
|
94
|
-
if (this.debugEnabled) {
|
|
95
|
-
|
|
111
|
+
if (this.debugEnabled && this.logger) {
|
|
112
|
+
this.logger.debug(`Global concurrency limit: ${this.concurrencyConfig.max} concurrent requests`);
|
|
96
113
|
}
|
|
97
114
|
}
|
|
98
115
|
else {
|
|
99
|
-
if (this.debugEnabled) {
|
|
100
|
-
|
|
116
|
+
if (this.debugEnabled && this.logger) {
|
|
117
|
+
this.logger.debug('No global concurrency limit (allows unlimited parallel batches)');
|
|
101
118
|
}
|
|
102
119
|
}
|
|
103
120
|
if (options.dedup) {
|
|
@@ -134,6 +151,12 @@ export class Client {
|
|
|
134
151
|
this.middlewares.push(xsrfMiddleware);
|
|
135
152
|
}
|
|
136
153
|
}
|
|
154
|
+
if (options.cookies) {
|
|
155
|
+
this.setupCookieJar(options.cookies);
|
|
156
|
+
}
|
|
157
|
+
if (this.maxResponseSize !== undefined) {
|
|
158
|
+
this.middlewares.push(this.createMaxSizeMiddleware(this.maxResponseSize));
|
|
159
|
+
}
|
|
137
160
|
if (this.debugEnabled && this.logger) {
|
|
138
161
|
this.middlewares.unshift(this.createLoggingMiddleware(this.logger));
|
|
139
162
|
}
|
|
@@ -143,18 +166,107 @@ export class Client {
|
|
|
143
166
|
createLoggingMiddleware(logger) {
|
|
144
167
|
return async (req, next) => {
|
|
145
168
|
const startTime = Date.now();
|
|
146
|
-
logger.
|
|
169
|
+
logger.debug({ type: 'request', method: req.method, url: req.url }, `→ ${req.method} ${req.url}`);
|
|
147
170
|
try {
|
|
148
171
|
const response = await next(req);
|
|
149
|
-
|
|
172
|
+
const duration = Date.now() - startTime;
|
|
173
|
+
logger.debug({
|
|
174
|
+
type: 'response',
|
|
175
|
+
method: req.method,
|
|
176
|
+
url: req.url,
|
|
177
|
+
status: response.status,
|
|
178
|
+
duration,
|
|
179
|
+
timings: response.timings,
|
|
180
|
+
}, `← ${response.status} ${req.method} ${req.url} (${duration}ms)`);
|
|
150
181
|
return response;
|
|
151
182
|
}
|
|
152
183
|
catch (error) {
|
|
153
|
-
|
|
184
|
+
const duration = Date.now() - startTime;
|
|
185
|
+
const err = error;
|
|
186
|
+
logger.error({
|
|
187
|
+
type: 'error',
|
|
188
|
+
method: req.method,
|
|
189
|
+
url: req.url,
|
|
190
|
+
error: err.message,
|
|
191
|
+
errorName: err.name,
|
|
192
|
+
duration,
|
|
193
|
+
}, `✖ ${req.method} ${req.url} - ${err.message}`);
|
|
154
194
|
throw error;
|
|
155
195
|
}
|
|
156
196
|
};
|
|
157
197
|
}
|
|
198
|
+
createMaxSizeMiddleware(globalMaxSize) {
|
|
199
|
+
return async (req, next) => {
|
|
200
|
+
const response = await next(req);
|
|
201
|
+
const limit = req.maxResponseSize ?? globalMaxSize;
|
|
202
|
+
if (limit === undefined)
|
|
203
|
+
return response;
|
|
204
|
+
const contentLength = response.headers.get('Content-Length');
|
|
205
|
+
if (contentLength) {
|
|
206
|
+
const size = parseInt(contentLength, 10);
|
|
207
|
+
if (!isNaN(size) && size > limit) {
|
|
208
|
+
throw new MaxSizeExceededError(limit, size, req);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return response;
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
setupCookieJar(options) {
|
|
215
|
+
if (options === true) {
|
|
216
|
+
this.cookieJar = new MemoryCookieJar();
|
|
217
|
+
}
|
|
218
|
+
else if (typeof options === 'object') {
|
|
219
|
+
if (options.jar === true) {
|
|
220
|
+
this.cookieJar = new MemoryCookieJar();
|
|
221
|
+
}
|
|
222
|
+
else if (options.jar && typeof options.jar === 'object') {
|
|
223
|
+
this.cookieJar = options.jar;
|
|
224
|
+
}
|
|
225
|
+
this.cookieIgnoreInvalid = options.ignoreInvalid ?? false;
|
|
226
|
+
}
|
|
227
|
+
if (this.cookieJar) {
|
|
228
|
+
this.middlewares.push(this.createCookieMiddleware());
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
createCookieMiddleware() {
|
|
232
|
+
return async (req, next) => {
|
|
233
|
+
const jar = this.cookieJar;
|
|
234
|
+
try {
|
|
235
|
+
const cookieString = await jar.getCookieString(req.url);
|
|
236
|
+
if (cookieString) {
|
|
237
|
+
const existingCookie = req.headers.get('cookie');
|
|
238
|
+
const newCookie = existingCookie
|
|
239
|
+
? `${existingCookie}; ${cookieString}`
|
|
240
|
+
: cookieString;
|
|
241
|
+
req.headers.set('cookie', newCookie);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
catch (error) {
|
|
245
|
+
if (!this.cookieIgnoreInvalid) {
|
|
246
|
+
throw error;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
const response = await next(req);
|
|
250
|
+
const setCookieHeader = response.headers.get('set-cookie');
|
|
251
|
+
if (setCookieHeader) {
|
|
252
|
+
const cookies = this.splitSetCookieHeader(setCookieHeader);
|
|
253
|
+
for (const cookie of cookies) {
|
|
254
|
+
try {
|
|
255
|
+
await jar.setCookie(cookie, req.url);
|
|
256
|
+
}
|
|
257
|
+
catch (error) {
|
|
258
|
+
if (!this.cookieIgnoreInvalid) {
|
|
259
|
+
throw error;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
return response;
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
splitSetCookieHeader(header) {
|
|
268
|
+
return header.split(/,(?=\s*[a-zA-Z0-9_-]+=)/g).map(s => s.trim());
|
|
269
|
+
}
|
|
158
270
|
composeMiddlewares() {
|
|
159
271
|
const chain = [...this.middlewares];
|
|
160
272
|
const transportDispatch = this.transport.dispatch.bind(this.transport);
|
|
@@ -204,7 +316,7 @@ export class Client {
|
|
|
204
316
|
};
|
|
205
317
|
httpErrorMiddleware = async (req, next) => {
|
|
206
318
|
const response = await next(req);
|
|
207
|
-
if (req.throwHttpErrors !== false && !response.ok) {
|
|
319
|
+
if (req.throwHttpErrors !== false && !response.ok && response.status !== 304) {
|
|
208
320
|
throw new HttpError(response, req);
|
|
209
321
|
}
|
|
210
322
|
return response;
|
|
@@ -242,8 +354,13 @@ export class Client {
|
|
|
242
354
|
const hasRequestParams = requestParams && Object.keys(requestParams).length > 0;
|
|
243
355
|
const hasDefaultParams = Object.keys(this.defaultParams).length > 0;
|
|
244
356
|
if (!hasRequestParams && !hasDefaultParams) {
|
|
245
|
-
if (
|
|
246
|
-
return
|
|
357
|
+
if (path.startsWith('http://') || path.startsWith('https://')) {
|
|
358
|
+
return path;
|
|
359
|
+
}
|
|
360
|
+
if (this.baseUrl) {
|
|
361
|
+
const base = this.baseUrl.endsWith('/') ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
362
|
+
const p = path.startsWith('/') ? path : '/' + path;
|
|
363
|
+
return base + p;
|
|
247
364
|
}
|
|
248
365
|
return path;
|
|
249
366
|
}
|
|
@@ -254,60 +371,92 @@ export class Client {
|
|
|
254
371
|
finalPath = finalPath.replace(/:([a-zA-Z0-9_]+)/g, (match, paramName) => {
|
|
255
372
|
if (mergedParams && paramName in mergedParams) {
|
|
256
373
|
usedParams.add(paramName);
|
|
257
|
-
return String(mergedParams[paramName]);
|
|
374
|
+
return encodeURIComponent(String(mergedParams[paramName]));
|
|
258
375
|
}
|
|
259
|
-
throw new
|
|
376
|
+
throw new ReckerError(`Missing required path parameter: ${paramName}`, undefined, undefined, [
|
|
377
|
+
`Provide '${paramName}' in request params or defaults.`,
|
|
378
|
+
'Ensure the path template matches the provided params.',
|
|
379
|
+
'If optional, remove the placeholder from the path.'
|
|
380
|
+
]);
|
|
260
381
|
});
|
|
261
382
|
}
|
|
262
|
-
let finalUrl
|
|
263
|
-
if (
|
|
264
|
-
finalUrl =
|
|
383
|
+
let finalUrl;
|
|
384
|
+
if (finalPath.startsWith('http://') || finalPath.startsWith('https://')) {
|
|
385
|
+
finalUrl = finalPath;
|
|
265
386
|
}
|
|
266
|
-
else if (
|
|
267
|
-
|
|
387
|
+
else if (this.baseUrl) {
|
|
388
|
+
const base = this.baseUrl.endsWith('/') ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
|
389
|
+
const p = finalPath.startsWith('/') ? finalPath : '/' + finalPath;
|
|
390
|
+
finalUrl = base + p;
|
|
391
|
+
}
|
|
392
|
+
else {
|
|
393
|
+
throw new ReckerError('Relative path provided without a baseUrl or explicit transport.', undefined, undefined, [
|
|
394
|
+
'Set baseUrl when creating the client.',
|
|
395
|
+
'Use an absolute URL in request().',
|
|
396
|
+
'Provide a custom transport that resolves relative paths.'
|
|
397
|
+
]);
|
|
268
398
|
}
|
|
269
399
|
const remainingKeys = Object.keys(mergedParams).filter((k) => !usedParams.has(k));
|
|
270
400
|
if (remainingKeys.length > 0) {
|
|
271
|
-
const
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
});
|
|
275
|
-
return urlObj.toString();
|
|
401
|
+
const queryParts = remainingKeys.map(key => `${encodeURIComponent(key)}=${encodeURIComponent(String(mergedParams[key]))}`);
|
|
402
|
+
const separator = finalUrl.includes('?') ? '&' : '?';
|
|
403
|
+
return finalUrl + separator + queryParts.join('&');
|
|
276
404
|
}
|
|
277
405
|
return finalUrl;
|
|
278
406
|
}
|
|
279
407
|
request(path, options = {}) {
|
|
280
408
|
const url = this.buildUrl(path, options.params);
|
|
281
|
-
let mergedHeaders
|
|
409
|
+
let mergedHeaders;
|
|
282
410
|
if (options.headers) {
|
|
283
|
-
mergedHeaders =
|
|
284
|
-
|
|
411
|
+
mergedHeaders = this.defaultHeaders instanceof Headers
|
|
412
|
+
? new Headers(this.defaultHeaders)
|
|
413
|
+
: new Headers(this.defaultHeaders);
|
|
414
|
+
const optHeaders = options.headers instanceof Headers
|
|
415
|
+
? options.headers
|
|
416
|
+
: new Headers(options.headers);
|
|
417
|
+
optHeaders.forEach((value, key) => mergedHeaders.append(key, value));
|
|
285
418
|
}
|
|
286
|
-
else
|
|
287
|
-
mergedHeaders =
|
|
419
|
+
else {
|
|
420
|
+
mergedHeaders = this.defaultHeaders instanceof Headers
|
|
421
|
+
? this.defaultHeaders
|
|
422
|
+
: new Headers(this.defaultHeaders);
|
|
288
423
|
}
|
|
289
|
-
const
|
|
290
|
-
let
|
|
424
|
+
const needsController = options.timeout || options.signal;
|
|
425
|
+
let controller;
|
|
426
|
+
let signal = options.signal;
|
|
291
427
|
let timeoutId;
|
|
292
428
|
let externalAbortCleanup;
|
|
293
|
-
if (
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
if (
|
|
297
|
-
|
|
429
|
+
if (needsController) {
|
|
430
|
+
controller = new AbortController();
|
|
431
|
+
signal = controller.signal;
|
|
432
|
+
if (options.signal) {
|
|
433
|
+
const externalSignal = options.signal;
|
|
434
|
+
const abortHandler = () => controller.abort(externalSignal.reason);
|
|
435
|
+
if (externalSignal.aborted) {
|
|
436
|
+
abortHandler();
|
|
437
|
+
}
|
|
438
|
+
else {
|
|
439
|
+
externalSignal.addEventListener('abort', abortHandler, { once: true });
|
|
440
|
+
externalAbortCleanup = () => externalSignal.removeEventListener('abort', abortHandler);
|
|
441
|
+
}
|
|
298
442
|
}
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
443
|
+
const timeout = options.timeout;
|
|
444
|
+
if (timeout) {
|
|
445
|
+
const totalTimeout = typeof timeout === 'number' ? timeout : timeout.request;
|
|
446
|
+
if (totalTimeout) {
|
|
447
|
+
timeoutId = setTimeout(() => controller.abort(new ReckerError('Request timed out (total timeout reached)', req, undefined, [
|
|
448
|
+
'Increase the timeout value for long-running requests.',
|
|
449
|
+
'Check upstream performance or network latency.',
|
|
450
|
+
'Use per-phase timeouts to pinpoint where the delay occurs.'
|
|
451
|
+
])), totalTimeout);
|
|
452
|
+
}
|
|
302
453
|
}
|
|
303
454
|
}
|
|
304
|
-
if (options.timeout) {
|
|
305
|
-
timeoutId = setTimeout(() => controller.abort(new Error('Request timed out')), options.timeout);
|
|
306
|
-
}
|
|
307
455
|
const req = new HttpRequest(url, {
|
|
308
456
|
...options,
|
|
309
457
|
headers: mergedHeaders,
|
|
310
458
|
signal,
|
|
459
|
+
maxResponseSize: options.maxResponseSize ?? this.maxResponseSize
|
|
311
460
|
});
|
|
312
461
|
const responsePromise = this.handler(req);
|
|
313
462
|
if (timeoutId || externalAbortCleanup) {
|
|
@@ -315,6 +464,7 @@ export class Client {
|
|
|
315
464
|
if (timeoutId)
|
|
316
465
|
clearTimeout(timeoutId);
|
|
317
466
|
externalAbortCleanup?.();
|
|
467
|
+
}).catch(() => {
|
|
318
468
|
});
|
|
319
469
|
}
|
|
320
470
|
return new RequestPromise(responsePromise, controller);
|
|
@@ -339,13 +489,53 @@ export class Client {
|
|
|
339
489
|
multi(requests, options = {}) {
|
|
340
490
|
return this.batch(requests, options);
|
|
341
491
|
}
|
|
342
|
-
requestWithBody(method, path,
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
492
|
+
requestWithBody(method, path, bodyOrOptions, options) {
|
|
493
|
+
let actualBody = bodyOrOptions;
|
|
494
|
+
let actualOptions = options;
|
|
495
|
+
const isOptionsEmpty = actualOptions === undefined ||
|
|
496
|
+
(typeof actualOptions === 'object' && actualOptions !== null && Object.keys(actualOptions).length === 0);
|
|
497
|
+
if (isOptionsEmpty && isPlainObject(bodyOrOptions)) {
|
|
498
|
+
const potentialOptions = bodyOrOptions;
|
|
499
|
+
if (potentialOptions.json !== undefined ||
|
|
500
|
+
potentialOptions.form !== undefined ||
|
|
501
|
+
potentialOptions.xml !== undefined ||
|
|
502
|
+
potentialOptions.body !== undefined ||
|
|
503
|
+
potentialOptions.headers !== undefined ||
|
|
504
|
+
potentialOptions.timeout !== undefined ||
|
|
505
|
+
potentialOptions.retry !== undefined ||
|
|
506
|
+
potentialOptions.hooks !== undefined ||
|
|
507
|
+
potentialOptions.searchParams !== undefined ||
|
|
508
|
+
potentialOptions.params !== undefined) {
|
|
509
|
+
actualOptions = bodyOrOptions;
|
|
510
|
+
actualBody = undefined;
|
|
511
|
+
}
|
|
347
512
|
}
|
|
348
|
-
|
|
513
|
+
actualOptions = actualOptions || {};
|
|
514
|
+
const { json, form, xml, ...restOptions } = actualOptions;
|
|
515
|
+
let finalBody = actualBody;
|
|
516
|
+
let explicitContentType;
|
|
517
|
+
if (form !== undefined) {
|
|
518
|
+
finalBody = createFormData(form);
|
|
519
|
+
explicitContentType = undefined;
|
|
520
|
+
}
|
|
521
|
+
else if (json !== undefined) {
|
|
522
|
+
finalBody = JSON.stringify(json);
|
|
523
|
+
explicitContentType = 'application/json';
|
|
524
|
+
}
|
|
525
|
+
else if (xml !== undefined) {
|
|
526
|
+
finalBody = '<?xml version="1.0" encoding="UTF-8"?>\n' + serializeXML(xml);
|
|
527
|
+
explicitContentType = 'application/xml';
|
|
528
|
+
}
|
|
529
|
+
else if (restOptions.body !== undefined) {
|
|
530
|
+
finalBody = restOptions.body;
|
|
531
|
+
}
|
|
532
|
+
const { body: processedBody, contentType } = processBody(finalBody);
|
|
533
|
+
const headers = new Headers(restOptions.headers);
|
|
534
|
+
const finalContentType = explicitContentType ?? contentType;
|
|
535
|
+
if (finalContentType && !headers.has('Content-Type')) {
|
|
536
|
+
headers.set('Content-Type', finalContentType);
|
|
537
|
+
}
|
|
538
|
+
return this.request(path, { ...restOptions, method, body: processedBody, headers });
|
|
349
539
|
}
|
|
350
540
|
post(path, body, options = {}) {
|
|
351
541
|
return this.requestWithBody('POST', path, body, options);
|
|
@@ -401,6 +591,11 @@ export class Client {
|
|
|
401
591
|
unlink(path, body, options = {}) {
|
|
402
592
|
return this.requestWithBody('UNLINK', path, body, options);
|
|
403
593
|
}
|
|
594
|
+
scrape(path, options = {}) {
|
|
595
|
+
const method = options.method || 'GET';
|
|
596
|
+
const requestPromise = this.request(path, { ...options, method });
|
|
597
|
+
return scrapeHelper(requestPromise);
|
|
598
|
+
}
|
|
404
599
|
paginate(path, options = {}) {
|
|
405
600
|
const { getItems, getNextUrl, maxPages, pageParam, limitParam, resultsPath, nextCursorPath, ...reqOptions } = options;
|
|
406
601
|
const paginationOpts = {
|
|
@@ -448,7 +643,11 @@ export class Client {
|
|
|
448
643
|
wsUrl = new URL(path, base).toString();
|
|
449
644
|
}
|
|
450
645
|
else {
|
|
451
|
-
throw new
|
|
646
|
+
throw new ReckerError('WebSocket requires either a full ws:// URL or a baseUrl', undefined, undefined, [
|
|
647
|
+
'Pass a full ws:// or wss:// URL to websocket().',
|
|
648
|
+
'Configure baseUrl so relative websocket paths can be resolved.',
|
|
649
|
+
'Ensure the baseUrl uses http/https so it can be converted to ws/wss.'
|
|
650
|
+
]);
|
|
452
651
|
}
|
|
453
652
|
const headersObj = {};
|
|
454
653
|
if (this.defaultHeaders) {
|
package/dist/core/errors.d.ts
CHANGED
|
@@ -2,18 +2,34 @@ import { ReckerRequest, ReckerResponse } from '../types/index.js';
|
|
|
2
2
|
export declare class ReckerError extends Error {
|
|
3
3
|
request?: ReckerRequest;
|
|
4
4
|
response?: ReckerResponse;
|
|
5
|
-
|
|
5
|
+
suggestions: string[];
|
|
6
|
+
retriable: boolean;
|
|
7
|
+
constructor(message: string, request?: ReckerRequest, response?: ReckerResponse, suggestions?: string[], retriable?: boolean);
|
|
6
8
|
}
|
|
7
9
|
export declare class HttpError extends ReckerError {
|
|
8
10
|
status: number;
|
|
9
11
|
statusText: string;
|
|
10
12
|
constructor(response: ReckerResponse, request?: ReckerRequest);
|
|
11
13
|
}
|
|
14
|
+
export type TimeoutPhase = 'lookup' | 'connect' | 'secureConnect' | 'socket' | 'send' | 'response' | 'request';
|
|
12
15
|
export declare class TimeoutError extends ReckerError {
|
|
13
|
-
|
|
16
|
+
phase: TimeoutPhase;
|
|
17
|
+
timeout: number;
|
|
18
|
+
elapsed?: number;
|
|
19
|
+
event: string;
|
|
20
|
+
constructor(request?: ReckerRequest, options?: {
|
|
21
|
+
phase?: TimeoutPhase;
|
|
22
|
+
timeout?: number;
|
|
23
|
+
elapsed?: number;
|
|
24
|
+
});
|
|
14
25
|
}
|
|
15
26
|
export declare class NetworkError extends ReckerError {
|
|
16
27
|
code?: string;
|
|
17
28
|
constructor(message: string, code?: string, request?: ReckerRequest);
|
|
18
29
|
}
|
|
30
|
+
export declare class MaxSizeExceededError extends ReckerError {
|
|
31
|
+
maxSize: number;
|
|
32
|
+
actualSize?: number;
|
|
33
|
+
constructor(maxSize: number, actualSize?: number, request?: ReckerRequest);
|
|
34
|
+
}
|
|
19
35
|
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/core/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAElE,qBAAa,WAAY,SAAQ,KAAK;IACpC,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,QAAQ,CAAC,EAAE,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/core/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAElE,qBAAa,WAAY,SAAQ,KAAK;IACpC,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,EAAE,OAAO,CAAC;gBAGjB,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,aAAa,EACvB,QAAQ,CAAC,EAAE,cAAc,EACzB,WAAW,GAAE,MAAM,EAAO,EAC1B,SAAS,UAAQ;CASpB;AAED,qBAAa,SAAU,SAAQ,WAAW;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;gBAEP,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,aAAa;CAY9D;AAKD,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,SAAS,GACT,eAAe,GACf,QAAQ,GACR,MAAM,GACN,UAAU,GACV,SAAS,CAAC;AAMd,qBAAa,YAAa,SAAQ,WAAW;IAI3C,KAAK,EAAE,YAAY,CAAC;IAKpB,OAAO,EAAE,MAAM,CAAC;IAKhB,OAAO,CAAC,EAAE,MAAM,CAAC;IAKjB,KAAK,EAAE,MAAM,CAAC;gBAGZ,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB;CAqCJ;AAED,qBAAa,YAAa,SAAQ,WAAW;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;gBAEF,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa;CAUpE;AAED,qBAAa,oBAAqB,SAAQ,WAAW;IACnD,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;gBAER,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa;CAmB1E"}
|
package/dist/core/errors.js
CHANGED
|
@@ -1,34 +1,95 @@
|
|
|
1
1
|
export class ReckerError extends Error {
|
|
2
2
|
request;
|
|
3
3
|
response;
|
|
4
|
-
|
|
4
|
+
suggestions;
|
|
5
|
+
retriable;
|
|
6
|
+
constructor(message, request, response, suggestions = [], retriable = false) {
|
|
5
7
|
super(message);
|
|
6
8
|
this.name = 'ReckerError';
|
|
7
9
|
this.request = request;
|
|
8
10
|
this.response = response;
|
|
11
|
+
this.suggestions = suggestions;
|
|
12
|
+
this.retriable = retriable;
|
|
9
13
|
}
|
|
10
14
|
}
|
|
11
15
|
export class HttpError extends ReckerError {
|
|
12
16
|
status;
|
|
13
17
|
statusText;
|
|
14
18
|
constructor(response, request) {
|
|
15
|
-
super(`Request failed with status code ${response.status} ${response.statusText}`, request, response);
|
|
19
|
+
super(`Request failed with status code ${response.status} ${response.statusText}`, request, response, ['Check the upstream service response body for error details.', 'Inspect request headers/body to ensure they match the API contract.', 'Retry if this is a transient 5xx/429 error.'], isRetryableStatus(response.status));
|
|
16
20
|
this.name = 'HttpError';
|
|
17
21
|
this.status = response.status;
|
|
18
22
|
this.statusText = response.statusText;
|
|
19
23
|
}
|
|
20
24
|
}
|
|
21
25
|
export class TimeoutError extends ReckerError {
|
|
22
|
-
|
|
23
|
-
|
|
26
|
+
phase;
|
|
27
|
+
timeout;
|
|
28
|
+
elapsed;
|
|
29
|
+
event;
|
|
30
|
+
constructor(request, options) {
|
|
31
|
+
const phase = options?.phase || 'request';
|
|
32
|
+
const timeout = options?.timeout;
|
|
33
|
+
const elapsed = options?.elapsed;
|
|
34
|
+
const phaseMessages = {
|
|
35
|
+
lookup: 'DNS lookup timed out',
|
|
36
|
+
connect: 'TCP connection timed out',
|
|
37
|
+
secureConnect: 'TLS handshake timed out',
|
|
38
|
+
socket: 'Socket assignment timed out (connection pool exhausted)',
|
|
39
|
+
send: 'Request body upload timed out',
|
|
40
|
+
response: 'Waiting for response timed out (TTFB)',
|
|
41
|
+
request: 'Request timed out (total time exceeded)'
|
|
42
|
+
};
|
|
43
|
+
let message = phaseMessages[phase];
|
|
44
|
+
if (timeout !== undefined) {
|
|
45
|
+
message += ` after ${timeout}ms`;
|
|
46
|
+
}
|
|
47
|
+
if (elapsed !== undefined && elapsed !== timeout) {
|
|
48
|
+
message += ` (elapsed: ${Math.round(elapsed)}ms)`;
|
|
49
|
+
}
|
|
50
|
+
const suggestions = [
|
|
51
|
+
'Verify network connectivity and DNS resolution for the target host.',
|
|
52
|
+
'Increase the specific timeout phase or optimize the upstream response time.',
|
|
53
|
+
'Reduce concurrent requests if the connection pool is exhausted.'
|
|
54
|
+
];
|
|
55
|
+
super(message, request, undefined, suggestions, true);
|
|
24
56
|
this.name = 'TimeoutError';
|
|
57
|
+
this.phase = phase;
|
|
58
|
+
this.timeout = timeout ?? 0;
|
|
59
|
+
this.elapsed = elapsed;
|
|
60
|
+
this.event = `timeout:${phase}`;
|
|
25
61
|
}
|
|
26
62
|
}
|
|
27
63
|
export class NetworkError extends ReckerError {
|
|
28
64
|
code;
|
|
29
65
|
constructor(message, code, request) {
|
|
30
|
-
|
|
66
|
+
const suggestions = [
|
|
67
|
+
'Confirm the host and port are reachable from this environment.',
|
|
68
|
+
'Check proxy/VPN/firewall settings that might block the request.',
|
|
69
|
+
'Retry the request or switch transport if this is transient.'
|
|
70
|
+
];
|
|
71
|
+
super(message, request, undefined, suggestions, true);
|
|
31
72
|
this.name = 'NetworkError';
|
|
32
73
|
this.code = code;
|
|
33
74
|
}
|
|
34
75
|
}
|
|
76
|
+
export class MaxSizeExceededError extends ReckerError {
|
|
77
|
+
maxSize;
|
|
78
|
+
actualSize;
|
|
79
|
+
constructor(maxSize, actualSize, request) {
|
|
80
|
+
const sizeInfo = actualSize
|
|
81
|
+
? `${actualSize} bytes (max: ${maxSize} bytes)`
|
|
82
|
+
: `${maxSize} bytes`;
|
|
83
|
+
super(`Response size exceeded maximum allowed: ${sizeInfo}`, request, undefined, [
|
|
84
|
+
'Increase maxResponseSize if the larger payload is expected.',
|
|
85
|
+
'Add pagination/streaming to reduce payload size.',
|
|
86
|
+
'Ensure the upstream is not returning unexpected large responses.'
|
|
87
|
+
], false);
|
|
88
|
+
this.name = 'MaxSizeExceededError';
|
|
89
|
+
this.maxSize = maxSize;
|
|
90
|
+
this.actualSize = actualSize;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function isRetryableStatus(status) {
|
|
94
|
+
return [408, 425, 429, 500, 502, 503, 504].includes(status);
|
|
95
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,sBAAsB,CAAC;AACrC,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request-promise.d.ts","sourceRoot":"","sources":["../../src/core/request-promise.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"request-promise.d.ts","sourceRoot":"","sources":["../../src/core/request-promise.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AAOhC,qBAAa,cAAc,CAAC,CAAC,GAAG,OAAO,CAAE,YAAW,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAC5E,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,eAAe,CAAC,CAAkB;gBAE9B,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,eAAe;IAKlF,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,WAEvB;IAED,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,QAAQ,GAAG,KAAK,EACjD,WAAW,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,KAAK,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,EACrF,UAAU,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,KAAK,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,GACtE,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAI/B,KAAK,CAAC,OAAO,GAAG,KAAK,EACnB,UAAU,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,KAAK,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,IAAI,GACpE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;IAIvC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAKpE,MAAM,IAAI,IAAI;IAMR,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC;IAKzB,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAKvB,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;IAK5B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAKrB,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAKlD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBlC,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAK1C,IAAI,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,GAAG,IAAI,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;IAKtD,GAAG,IAAI,cAAc,CAAC,QAAQ,CAAC;IAK/B,QAAQ,IAAI,cAAc,CAAC,aAAa,CAAC;IAKzC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,UAAU,CAAC;CAI5D"}
|
|
@@ -2,6 +2,7 @@ import { createWriteStream } from 'node:fs';
|
|
|
2
2
|
import { pipeline } from 'node:stream/promises';
|
|
3
3
|
import { Readable } from 'node:stream';
|
|
4
4
|
import { tryFn } from '../utils/try-fn.js';
|
|
5
|
+
import { ReckerError } from './errors.js';
|
|
5
6
|
export class RequestPromise {
|
|
6
7
|
promise;
|
|
7
8
|
abortController;
|
|
@@ -49,8 +50,13 @@ export class RequestPromise {
|
|
|
49
50
|
async write(path) {
|
|
50
51
|
const response = await this.promise;
|
|
51
52
|
const body = response.read();
|
|
52
|
-
if (!body)
|
|
53
|
-
throw new
|
|
53
|
+
if (!body) {
|
|
54
|
+
throw new ReckerError('Response has no body to write', undefined, response, [
|
|
55
|
+
'Ensure the request returned a body (avoid HEAD/204).',
|
|
56
|
+
'Check if the request was aborted before the body streamed.',
|
|
57
|
+
'Verify upstream is not sending an empty response.'
|
|
58
|
+
]);
|
|
59
|
+
}
|
|
54
60
|
const nodeStream = Readable.fromWeb(body);
|
|
55
61
|
const fileStream = createWriteStream(path);
|
|
56
62
|
await pipeline(nodeStream, fileStream);
|