recker 1.0.26 → 1.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/dist/browser/browser/cache.d.ts +40 -0
  2. package/dist/browser/browser/cache.js +199 -0
  3. package/dist/browser/browser/crypto.d.ts +24 -0
  4. package/dist/browser/browser/crypto.js +80 -0
  5. package/dist/browser/browser/index.d.ts +31 -0
  6. package/dist/browser/browser/index.js +31 -0
  7. package/dist/browser/browser/recker.d.ts +26 -0
  8. package/dist/browser/browser/recker.js +61 -0
  9. package/dist/browser/cache/basic-file-storage.d.ts +12 -0
  10. package/dist/browser/cache/basic-file-storage.js +50 -0
  11. package/dist/browser/cache/memory-limits.d.ts +20 -0
  12. package/dist/browser/cache/memory-limits.js +96 -0
  13. package/dist/browser/cache/memory-storage.d.ts +132 -0
  14. package/dist/browser/cache/memory-storage.js +454 -0
  15. package/dist/browser/cache.d.ts +40 -0
  16. package/dist/browser/cache.js +199 -0
  17. package/dist/browser/constants/http-status.d.ts +73 -0
  18. package/dist/browser/constants/http-status.js +156 -0
  19. package/dist/browser/cookies/memory-cookie-jar.d.ts +30 -0
  20. package/dist/browser/cookies/memory-cookie-jar.js +210 -0
  21. package/dist/browser/core/client.d.ts +118 -0
  22. package/dist/browser/core/client.js +667 -0
  23. package/dist/browser/core/errors.d.ts +142 -0
  24. package/dist/browser/core/errors.js +308 -0
  25. package/dist/browser/core/index.d.ts +5 -0
  26. package/dist/browser/core/index.js +5 -0
  27. package/dist/browser/core/request-promise.d.ts +23 -0
  28. package/dist/browser/core/request-promise.js +82 -0
  29. package/dist/browser/core/request.d.ts +20 -0
  30. package/dist/browser/core/request.js +76 -0
  31. package/dist/browser/core/response.d.ts +34 -0
  32. package/dist/browser/core/response.js +178 -0
  33. package/dist/browser/crypto.d.ts +24 -0
  34. package/dist/browser/crypto.js +80 -0
  35. package/dist/browser/index.d.ts +31 -0
  36. package/dist/browser/index.js +31 -0
  37. package/dist/browser/plugins/auth/api-key.d.ts +8 -0
  38. package/dist/browser/plugins/auth/api-key.js +27 -0
  39. package/dist/browser/plugins/auth/auth0.d.ts +33 -0
  40. package/dist/browser/plugins/auth/auth0.js +94 -0
  41. package/dist/browser/plugins/auth/aws-sigv4.d.ts +10 -0
  42. package/dist/browser/plugins/auth/aws-sigv4.js +88 -0
  43. package/dist/browser/plugins/auth/azure-ad.d.ts +48 -0
  44. package/dist/browser/plugins/auth/azure-ad.js +152 -0
  45. package/dist/browser/plugins/auth/basic.d.ts +7 -0
  46. package/dist/browser/plugins/auth/basic.js +13 -0
  47. package/dist/browser/plugins/auth/bearer.d.ts +8 -0
  48. package/dist/browser/plugins/auth/bearer.js +17 -0
  49. package/dist/browser/plugins/auth/cognito.d.ts +45 -0
  50. package/dist/browser/plugins/auth/cognito.js +208 -0
  51. package/dist/browser/plugins/auth/digest.d.ts +8 -0
  52. package/dist/browser/plugins/auth/digest.js +100 -0
  53. package/dist/browser/plugins/auth/firebase.d.ts +32 -0
  54. package/dist/browser/plugins/auth/firebase.js +195 -0
  55. package/dist/browser/plugins/auth/github-app.d.ts +36 -0
  56. package/dist/browser/plugins/auth/github-app.js +170 -0
  57. package/dist/browser/plugins/auth/google-service-account.d.ts +49 -0
  58. package/dist/browser/plugins/auth/google-service-account.js +172 -0
  59. package/dist/browser/plugins/auth/index.d.ts +15 -0
  60. package/dist/browser/plugins/auth/index.js +15 -0
  61. package/dist/browser/plugins/auth/mtls.d.ts +37 -0
  62. package/dist/browser/plugins/auth/mtls.js +140 -0
  63. package/dist/browser/plugins/auth/oauth2.d.ts +8 -0
  64. package/dist/browser/plugins/auth/oauth2.js +26 -0
  65. package/dist/browser/plugins/auth/oidc.d.ts +55 -0
  66. package/dist/browser/plugins/auth/oidc.js +222 -0
  67. package/dist/browser/plugins/auth/okta.d.ts +47 -0
  68. package/dist/browser/plugins/auth/okta.js +157 -0
  69. package/dist/browser/plugins/auth.d.ts +1 -0
  70. package/dist/browser/plugins/auth.js +1 -0
  71. package/dist/browser/plugins/cache.d.ts +15 -0
  72. package/dist/browser/plugins/cache.js +486 -0
  73. package/dist/browser/plugins/circuit-breaker.d.ts +13 -0
  74. package/dist/browser/plugins/circuit-breaker.js +100 -0
  75. package/dist/browser/plugins/compression.d.ts +4 -0
  76. package/dist/browser/plugins/compression.js +130 -0
  77. package/dist/browser/plugins/cookie-jar.d.ts +5 -0
  78. package/dist/browser/plugins/cookie-jar.js +72 -0
  79. package/dist/browser/plugins/dedup.d.ts +5 -0
  80. package/dist/browser/plugins/dedup.js +35 -0
  81. package/dist/browser/plugins/graphql.d.ts +13 -0
  82. package/dist/browser/plugins/graphql.js +58 -0
  83. package/dist/browser/plugins/grpc-web.d.ts +79 -0
  84. package/dist/browser/plugins/grpc-web.js +261 -0
  85. package/dist/browser/plugins/hls.d.ts +105 -0
  86. package/dist/browser/plugins/hls.js +395 -0
  87. package/dist/browser/plugins/jsonrpc.d.ts +75 -0
  88. package/dist/browser/plugins/jsonrpc.js +143 -0
  89. package/dist/browser/plugins/logger.d.ts +13 -0
  90. package/dist/browser/plugins/logger.js +108 -0
  91. package/dist/browser/plugins/odata.d.ts +181 -0
  92. package/dist/browser/plugins/odata.js +564 -0
  93. package/dist/browser/plugins/pagination.d.ts +16 -0
  94. package/dist/browser/plugins/pagination.js +105 -0
  95. package/dist/browser/plugins/rate-limit.d.ts +15 -0
  96. package/dist/browser/plugins/rate-limit.js +162 -0
  97. package/dist/browser/plugins/retry.d.ts +14 -0
  98. package/dist/browser/plugins/retry.js +116 -0
  99. package/dist/browser/plugins/scrape.d.ts +21 -0
  100. package/dist/browser/plugins/scrape.js +82 -0
  101. package/dist/browser/plugins/server-timing.d.ts +7 -0
  102. package/dist/browser/plugins/server-timing.js +24 -0
  103. package/dist/browser/plugins/soap.d.ts +72 -0
  104. package/dist/browser/plugins/soap.js +347 -0
  105. package/dist/browser/plugins/xml.d.ts +9 -0
  106. package/dist/browser/plugins/xml.js +194 -0
  107. package/dist/browser/plugins/xsrf.d.ts +9 -0
  108. package/dist/browser/plugins/xsrf.js +48 -0
  109. package/dist/browser/recker.d.ts +26 -0
  110. package/dist/browser/recker.js +61 -0
  111. package/dist/browser/runner/request-runner.d.ts +46 -0
  112. package/dist/browser/runner/request-runner.js +89 -0
  113. package/dist/browser/scrape/document.d.ts +44 -0
  114. package/dist/browser/scrape/document.js +210 -0
  115. package/dist/browser/scrape/element.d.ts +49 -0
  116. package/dist/browser/scrape/element.js +176 -0
  117. package/dist/browser/scrape/extractors.d.ts +16 -0
  118. package/dist/browser/scrape/extractors.js +356 -0
  119. package/dist/browser/scrape/types.d.ts +107 -0
  120. package/dist/browser/scrape/types.js +1 -0
  121. package/dist/browser/transport/fetch.d.ts +11 -0
  122. package/dist/browser/transport/fetch.js +143 -0
  123. package/dist/browser/transport/undici.d.ts +38 -0
  124. package/dist/browser/transport/undici.js +897 -0
  125. package/dist/browser/types/ai.d.ts +267 -0
  126. package/dist/browser/types/ai.js +1 -0
  127. package/dist/browser/types/index.d.ts +351 -0
  128. package/dist/browser/types/index.js +1 -0
  129. package/dist/browser/types/logger.d.ts +16 -0
  130. package/dist/browser/types/logger.js +66 -0
  131. package/dist/browser/types/udp.d.ts +138 -0
  132. package/dist/browser/types/udp.js +1 -0
  133. package/dist/browser/utils/agent-manager.d.ts +29 -0
  134. package/dist/browser/utils/agent-manager.js +160 -0
  135. package/dist/browser/utils/body.d.ts +10 -0
  136. package/dist/browser/utils/body.js +148 -0
  137. package/dist/browser/utils/charset.d.ts +15 -0
  138. package/dist/browser/utils/charset.js +169 -0
  139. package/dist/browser/utils/concurrency.d.ts +20 -0
  140. package/dist/browser/utils/concurrency.js +120 -0
  141. package/dist/browser/utils/dns.d.ts +6 -0
  142. package/dist/browser/utils/dns.js +26 -0
  143. package/dist/browser/utils/header-parser.d.ts +94 -0
  144. package/dist/browser/utils/header-parser.js +617 -0
  145. package/dist/browser/utils/html-cleaner.d.ts +1 -0
  146. package/dist/browser/utils/html-cleaner.js +21 -0
  147. package/dist/browser/utils/link-header.d.ts +69 -0
  148. package/dist/browser/utils/link-header.js +190 -0
  149. package/dist/browser/utils/optional-require.d.ts +19 -0
  150. package/dist/browser/utils/optional-require.js +105 -0
  151. package/dist/browser/utils/progress.d.ts +8 -0
  152. package/dist/browser/utils/progress.js +82 -0
  153. package/dist/browser/utils/request-pool.d.ts +22 -0
  154. package/dist/browser/utils/request-pool.js +101 -0
  155. package/dist/browser/utils/sse.d.ts +7 -0
  156. package/dist/browser/utils/sse.js +67 -0
  157. package/dist/browser/utils/streaming.d.ts +17 -0
  158. package/dist/browser/utils/streaming.js +84 -0
  159. package/dist/browser/utils/try-fn.d.ts +3 -0
  160. package/dist/browser/utils/try-fn.js +59 -0
  161. package/dist/browser/utils/user-agent.d.ts +44 -0
  162. package/dist/browser/utils/user-agent.js +100 -0
  163. package/dist/browser/utils/whois.d.ts +32 -0
  164. package/dist/browser/utils/whois.js +246 -0
  165. package/dist/browser/websocket/client.d.ts +65 -0
  166. package/dist/browser/websocket/client.js +313 -0
  167. package/dist/cli/index.d.ts +1 -0
  168. package/dist/cli/index.js +1 -0
  169. package/dist/transport/fetch.d.ts +7 -1
  170. package/dist/transport/fetch.js +58 -76
  171. package/package.json +34 -2
@@ -0,0 +1,138 @@
1
+ import type { ReckerRequest, ReckerResponse, Transport } from './index.js';
2
+ export interface UDPTimings {
3
+ queued: number;
4
+ send: number;
5
+ receive: number;
6
+ retransmissions: number;
7
+ total: number;
8
+ }
9
+ export interface DTLSTimings extends UDPTimings {
10
+ handshake: number;
11
+ }
12
+ export interface QUICTimings extends UDPTimings {
13
+ quicHandshake: number;
14
+ streamOpen: number;
15
+ firstByte: number;
16
+ zeroRTT: boolean;
17
+ }
18
+ export interface UDPConnection {
19
+ protocol: 'udp' | 'dtls' | 'quic';
20
+ localAddress: string;
21
+ localPort: number;
22
+ remoteAddress: string;
23
+ remotePort: number;
24
+ }
25
+ export interface DTLSConnection extends UDPConnection {
26
+ protocol: 'dtls';
27
+ cipher: string;
28
+ version: string;
29
+ }
30
+ export interface QUICConnection extends UDPConnection {
31
+ protocol: 'quic';
32
+ alpn: string;
33
+ congestion: 'bbr' | 'cubic' | 'reno';
34
+ quicVersion: string;
35
+ migrated: boolean;
36
+ }
37
+ export interface BaseUDPTransportOptions {
38
+ timeout?: number;
39
+ retransmissions?: number;
40
+ maxPacketSize?: number;
41
+ observability?: boolean;
42
+ debug?: boolean;
43
+ localAddress?: string;
44
+ localPort?: number;
45
+ }
46
+ export interface UDPTransportOptions extends BaseUDPTransportOptions {
47
+ broadcast?: boolean;
48
+ multicastTTL?: number;
49
+ multicastGroups?: string[];
50
+ multicastLoopback?: boolean;
51
+ type?: 'udp4' | 'udp6';
52
+ recvBufferSize?: number;
53
+ sendBufferSize?: number;
54
+ }
55
+ export interface DTLSTransportOptions extends BaseUDPTransportOptions {
56
+ cert?: string | Buffer;
57
+ key?: string | Buffer;
58
+ ca?: string | Buffer | Array<string | Buffer>;
59
+ passphrase?: string;
60
+ ciphers?: string;
61
+ minVersion?: 'DTLSv1.0' | 'DTLSv1.2';
62
+ rejectUnauthorized?: boolean;
63
+ servername?: string;
64
+ handshakeTimeout?: number;
65
+ mtu?: number;
66
+ }
67
+ export interface QUICTransportOptions extends BaseUDPTransportOptions {
68
+ maxStreams?: number;
69
+ idleTimeout?: number;
70
+ congestionControl?: 'bbr' | 'cubic' | 'reno';
71
+ earlyData?: boolean;
72
+ sessionCache?: Map<string, Buffer>;
73
+ cert?: string | Buffer;
74
+ key?: string | Buffer;
75
+ ca?: string | Buffer | Array<string | Buffer>;
76
+ alpnProtocols?: string[];
77
+ initialRtt?: number;
78
+ connectionMigration?: boolean;
79
+ }
80
+ export interface UDPRequestOptions {
81
+ broadcast?: boolean;
82
+ multicast?: string;
83
+ ttl?: number;
84
+ port?: number;
85
+ }
86
+ export interface UDPRequest extends ReckerRequest {
87
+ udp?: UDPRequestOptions;
88
+ }
89
+ export interface UDPResponse<T = unknown> extends ReckerResponse<T> {
90
+ timings: UDPTimings;
91
+ connection: UDPConnection;
92
+ buffer(): Promise<Buffer>;
93
+ packets(): AsyncGenerator<Buffer>;
94
+ }
95
+ export interface UDPTransport extends Transport {
96
+ dispatch(req: UDPRequest): Promise<UDPResponse>;
97
+ send(host: string, port: number, data: Buffer): Promise<void>;
98
+ broadcast(port: number, data: Buffer): Promise<void>;
99
+ joinMulticast(group: string): void;
100
+ leaveMulticast(group: string): void;
101
+ close(): Promise<void>;
102
+ }
103
+ export interface DTLSTransportInterface extends Transport {
104
+ dispatch(req: UDPRequest): Promise<UDPResponse>;
105
+ handshake(): Promise<void>;
106
+ getSession(): Buffer | null;
107
+ close(): Promise<void>;
108
+ }
109
+ export interface QUICTransportInterface extends Transport {
110
+ dispatch(req: ReckerRequest): Promise<ReckerResponse>;
111
+ getSessionTicket(): Buffer | null;
112
+ migrate(newLocalAddress: string): Promise<void>;
113
+ close(): Promise<void>;
114
+ }
115
+ export interface UDPBatchResult<T> {
116
+ results: Array<T | Error>;
117
+ stats: {
118
+ total: number;
119
+ successful: number;
120
+ failed: number;
121
+ duration: number;
122
+ };
123
+ }
124
+ export interface UDPDiscoveryResult {
125
+ address: string;
126
+ port: number;
127
+ data: Buffer;
128
+ latency: number;
129
+ }
130
+ export interface SimpleUDPAPI {
131
+ send(address: string, data: Buffer, options?: UDPTransportOptions): Promise<UDPResponse>;
132
+ broadcast(port: number, data: Buffer, options?: UDPTransportOptions & {
133
+ timeout?: number;
134
+ }): Promise<UDPDiscoveryResult[]>;
135
+ discover(group: string, port: number, data: Buffer, options?: UDPTransportOptions & {
136
+ timeout?: number;
137
+ }): Promise<UDPDiscoveryResult[]>;
138
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,29 @@
1
+ import { Agent } from 'undici';
2
+ import type { AgentOptions } from '../types/index.js';
3
+ export interface AgentStats {
4
+ agentCount: number;
5
+ domains: string[];
6
+ totalConnections: number;
7
+ }
8
+ export declare class AgentManager {
9
+ private globalAgent?;
10
+ private domainAgents;
11
+ private options;
12
+ constructor(options?: AgentOptions);
13
+ getGlobalAgent(): Agent;
14
+ getAgentForDomain(domain: string, options?: Partial<AgentOptions>): Agent;
15
+ getAgentForUrl(url: string): Agent;
16
+ createBatchAgent(concurrency: number, requestCount: number, options?: Partial<AgentOptions>): Agent;
17
+ createStressTestAgent(concurrency: number): Agent;
18
+ private createAgent;
19
+ getStats(): AgentStats;
20
+ closeDomainAgent(domain: string): Promise<void>;
21
+ closeAll(): Promise<void>;
22
+ destroy(): Promise<void>;
23
+ }
24
+ export declare function createAgent(options?: AgentOptions): Agent;
25
+ export declare function extractDomain(url: string): string | null;
26
+ export declare function analyzeBatchDomains(urls: string[]): {
27
+ strategy: 'single' | 'multi';
28
+ domains: Set<string>;
29
+ };
@@ -0,0 +1,160 @@
1
+ import { Agent } from 'undici';
2
+ export class AgentManager {
3
+ globalAgent;
4
+ domainAgents;
5
+ options;
6
+ constructor(options = {}) {
7
+ this.domainAgents = new Map();
8
+ this.options = {
9
+ connections: options.connections ?? 10,
10
+ pipelining: options.pipelining ?? 1,
11
+ keepAlive: options.keepAlive ?? true,
12
+ keepAliveTimeout: options.keepAliveTimeout ?? 4 * 1000,
13
+ keepAliveMaxTimeout: options.keepAliveMaxTimeout ?? 10 * 60 * 1000,
14
+ keepAliveTimeoutThreshold: options.keepAliveTimeoutThreshold ?? 1 * 1000,
15
+ connectTimeout: options.connectTimeout ?? 10 * 1000,
16
+ perDomainPooling: options.perDomainPooling ?? true,
17
+ localAddress: options.localAddress,
18
+ maxRequestsPerClient: options.maxRequestsPerClient ?? 0,
19
+ maxCachedSessions: options.maxCachedSessions ?? 100,
20
+ maxHeaderSize: options.maxHeaderSize,
21
+ clientTtl: options.clientTtl ?? null,
22
+ };
23
+ }
24
+ getGlobalAgent() {
25
+ if (!this.globalAgent) {
26
+ this.globalAgent = this.createAgent(this.options);
27
+ }
28
+ return this.globalAgent;
29
+ }
30
+ getAgentForDomain(domain, options) {
31
+ if (!this.options.perDomainPooling) {
32
+ return this.getGlobalAgent();
33
+ }
34
+ let agent = this.domainAgents.get(domain);
35
+ if (!agent) {
36
+ const agentOptions = { ...this.options, ...options };
37
+ agent = this.createAgent(agentOptions);
38
+ this.domainAgents.set(domain, agent);
39
+ }
40
+ return agent;
41
+ }
42
+ getAgentForUrl(url) {
43
+ try {
44
+ const parsedUrl = new URL(url);
45
+ return this.getAgentForDomain(parsedUrl.hostname);
46
+ }
47
+ catch {
48
+ return this.getGlobalAgent();
49
+ }
50
+ }
51
+ createBatchAgent(concurrency, requestCount, options) {
52
+ const smartConnections = Math.max(1, Math.min(Math.ceil(concurrency / 2), 50, requestCount));
53
+ const smartPipelining = requestCount > 20 && concurrency > 5
54
+ ? 2
55
+ : 1;
56
+ const batchOptions = {
57
+ ...this.options,
58
+ connections: options?.connections ?? smartConnections,
59
+ pipelining: options?.pipelining ?? smartPipelining,
60
+ ...options,
61
+ };
62
+ return this.createAgent(batchOptions);
63
+ }
64
+ createStressTestAgent(concurrency) {
65
+ const pipelining = 10;
66
+ const connections = Math.min(Math.ceil(concurrency / pipelining), 100);
67
+ return this.createAgent({
68
+ connections,
69
+ pipelining,
70
+ keepAliveTimeout: 30000,
71
+ keepAliveMaxTimeout: 120000,
72
+ connectTimeout: 5000,
73
+ });
74
+ }
75
+ createAgent(options) {
76
+ return new Agent({
77
+ connections: options.connections,
78
+ pipelining: options.pipelining,
79
+ maxHeaderSize: options.maxHeaderSize,
80
+ maxRequestsPerClient: options.maxRequestsPerClient,
81
+ maxCachedSessions: options.maxCachedSessions,
82
+ clientTtl: options.clientTtl ?? null,
83
+ keepAliveTimeout: options.keepAliveTimeout,
84
+ keepAliveMaxTimeout: options.keepAliveMaxTimeout,
85
+ keepAliveTimeoutThreshold: options.keepAliveTimeoutThreshold,
86
+ connectTimeout: options.connectTimeout,
87
+ socketPath: undefined,
88
+ connect: {
89
+ timeout: options.connectTimeout,
90
+ keepAlive: options.keepAlive,
91
+ keepAliveInitialDelay: options.keepAliveTimeout,
92
+ localAddress: options.localAddress,
93
+ },
94
+ });
95
+ }
96
+ getStats() {
97
+ return {
98
+ agentCount: 1 + this.domainAgents.size,
99
+ domains: Array.from(this.domainAgents.keys()),
100
+ totalConnections: (1 + this.domainAgents.size) * this.options.connections,
101
+ };
102
+ }
103
+ async closeDomainAgent(domain) {
104
+ const agent = this.domainAgents.get(domain);
105
+ if (agent) {
106
+ await agent.close();
107
+ this.domainAgents.delete(domain);
108
+ }
109
+ }
110
+ async closeAll() {
111
+ const closePromises = [];
112
+ if (this.globalAgent) {
113
+ closePromises.push(this.globalAgent.close());
114
+ this.globalAgent = undefined;
115
+ }
116
+ for (const agent of this.domainAgents.values()) {
117
+ closePromises.push(agent.close());
118
+ }
119
+ this.domainAgents.clear();
120
+ await Promise.all(closePromises);
121
+ }
122
+ async destroy() {
123
+ const destroyPromises = [];
124
+ if (this.globalAgent) {
125
+ destroyPromises.push(this.globalAgent.destroy());
126
+ this.globalAgent = undefined;
127
+ }
128
+ for (const agent of this.domainAgents.values()) {
129
+ destroyPromises.push(agent.destroy());
130
+ }
131
+ this.domainAgents.clear();
132
+ await Promise.all(destroyPromises);
133
+ }
134
+ }
135
+ export function createAgent(options = {}) {
136
+ const manager = new AgentManager(options);
137
+ return manager.getGlobalAgent();
138
+ }
139
+ export function extractDomain(url) {
140
+ try {
141
+ const parsed = new URL(url);
142
+ return parsed.hostname;
143
+ }
144
+ catch {
145
+ return null;
146
+ }
147
+ }
148
+ export function analyzeBatchDomains(urls) {
149
+ const domains = new Set();
150
+ for (const url of urls) {
151
+ const domain = extractDomain(url);
152
+ if (domain) {
153
+ domains.add(domain);
154
+ }
155
+ }
156
+ return {
157
+ strategy: domains.size <= 1 ? 'single' : 'multi',
158
+ domains,
159
+ };
160
+ }
@@ -0,0 +1,10 @@
1
+ export type BodyInput = string | Blob | ArrayBuffer | FormData | URLSearchParams | ReadableStream | Record<string, any> | null | undefined;
2
+ export interface ProcessedBody {
3
+ body: any;
4
+ contentType?: string;
5
+ }
6
+ export declare function isPlainObject(value: any): boolean;
7
+ export declare function processBody(body: BodyInput): ProcessedBody;
8
+ export declare function createFormData(data: Record<string, any>): FormData;
9
+ export declare function createMultipart(data: Record<string, any>): FormData;
10
+ export declare function isFileUpload(body: any): boolean;
@@ -0,0 +1,148 @@
1
+ export function isPlainObject(value) {
2
+ if (typeof value !== 'object' || value === null) {
3
+ return false;
4
+ }
5
+ if (value instanceof Blob ||
6
+ value instanceof FormData ||
7
+ value instanceof URLSearchParams ||
8
+ value instanceof ArrayBuffer ||
9
+ value instanceof ReadableStream) {
10
+ return false;
11
+ }
12
+ if (typeof File !== 'undefined') {
13
+ try {
14
+ if (value instanceof File) {
15
+ return false;
16
+ }
17
+ }
18
+ catch {
19
+ }
20
+ }
21
+ return true;
22
+ }
23
+ export function processBody(body) {
24
+ if (body === null || body === undefined) {
25
+ return { body: undefined };
26
+ }
27
+ if (typeof body === 'string') {
28
+ return {
29
+ body,
30
+ contentType: 'text/plain; charset=utf-8'
31
+ };
32
+ }
33
+ if (body instanceof FormData) {
34
+ return {
35
+ body,
36
+ contentType: undefined
37
+ };
38
+ }
39
+ if (body instanceof URLSearchParams) {
40
+ return {
41
+ body: body.toString(),
42
+ contentType: 'application/x-www-form-urlencoded'
43
+ };
44
+ }
45
+ if (body instanceof Blob || isFile(body)) {
46
+ return {
47
+ body,
48
+ contentType: body.type || 'application/octet-stream'
49
+ };
50
+ }
51
+ if (body instanceof ArrayBuffer) {
52
+ return {
53
+ body,
54
+ contentType: 'application/octet-stream'
55
+ };
56
+ }
57
+ if (body instanceof ReadableStream) {
58
+ return {
59
+ body,
60
+ contentType: 'application/octet-stream'
61
+ };
62
+ }
63
+ if (isPlainObject(body)) {
64
+ if (isFileUpload(body)) {
65
+ return {
66
+ body: createFormData(body),
67
+ contentType: undefined
68
+ };
69
+ }
70
+ return {
71
+ body: JSON.stringify(body),
72
+ contentType: 'application/json'
73
+ };
74
+ }
75
+ if (Array.isArray(body)) {
76
+ return {
77
+ body: JSON.stringify(body),
78
+ contentType: 'application/json'
79
+ };
80
+ }
81
+ return {
82
+ body: String(body),
83
+ contentType: 'text/plain; charset=utf-8'
84
+ };
85
+ }
86
+ export function createFormData(data) {
87
+ const formData = new FormData();
88
+ for (const [key, value] of Object.entries(data)) {
89
+ if (value === null || value === undefined) {
90
+ continue;
91
+ }
92
+ if (Array.isArray(value)) {
93
+ value.forEach((item) => {
94
+ if (item instanceof Blob || isFile(item)) {
95
+ formData.append(key, item);
96
+ }
97
+ else {
98
+ formData.append(key, String(item));
99
+ }
100
+ });
101
+ continue;
102
+ }
103
+ if (value instanceof Blob || isFile(value)) {
104
+ formData.append(key, value);
105
+ continue;
106
+ }
107
+ if (typeof value === 'object') {
108
+ formData.append(key, JSON.stringify(value));
109
+ continue;
110
+ }
111
+ formData.append(key, String(value));
112
+ }
113
+ return formData;
114
+ }
115
+ export function createMultipart(data) {
116
+ return createFormData(data);
117
+ }
118
+ function isFile(value) {
119
+ if (typeof File === 'undefined') {
120
+ return false;
121
+ }
122
+ return value instanceof File;
123
+ }
124
+ export function isFileUpload(body) {
125
+ if (!body)
126
+ return false;
127
+ if (body instanceof Blob || isFile(body)) {
128
+ return true;
129
+ }
130
+ if (body instanceof FormData) {
131
+ for (const value of body.values()) {
132
+ if (value instanceof Blob || isFile(value)) {
133
+ return true;
134
+ }
135
+ }
136
+ }
137
+ if (isPlainObject(body)) {
138
+ for (const value of Object.values(body)) {
139
+ if (value instanceof Blob || isFile(value)) {
140
+ return true;
141
+ }
142
+ if (Array.isArray(value) && value.some(v => v instanceof Blob || isFile(v))) {
143
+ return true;
144
+ }
145
+ }
146
+ }
147
+ return false;
148
+ }
@@ -0,0 +1,15 @@
1
+ export interface CharsetInfo {
2
+ charset: string;
3
+ source: 'header' | 'bom' | 'xml' | 'html-meta' | 'default';
4
+ confidence: 'high' | 'medium' | 'low';
5
+ }
6
+ export declare function normalizeCharset(charset: string): string;
7
+ export declare function detectFromContentType(contentType: string | null): string | null;
8
+ export declare function detectFromBOM(buffer: Uint8Array): CharsetInfo | null;
9
+ export declare function stripBOM(buffer: Uint8Array): Uint8Array;
10
+ export declare function detectFromXMLDeclaration(content: string): string | null;
11
+ export declare function detectFromHTMLMeta(html: string): string | null;
12
+ export declare function detectCharset(buffer: Uint8Array, contentType?: string | null): CharsetInfo;
13
+ export declare function decodeText(buffer: Uint8Array, charsetOrInfo?: string | CharsetInfo): string;
14
+ export declare function getSupportedEncodings(): string[];
15
+ export declare function isCharsetSupported(charset: string): boolean;
@@ -0,0 +1,169 @@
1
+ const charsetAliases = {
2
+ 'utf8': 'utf-8',
3
+ 'utf_8': 'utf-8',
4
+ 'utf16': 'utf-16',
5
+ 'utf_16': 'utf-16',
6
+ 'utf16le': 'utf-16le',
7
+ 'utf16be': 'utf-16be',
8
+ 'latin1': 'iso-8859-1',
9
+ 'latin-1': 'iso-8859-1',
10
+ 'iso8859-1': 'iso-8859-1',
11
+ 'iso_8859-1': 'iso-8859-1',
12
+ 'cp1252': 'windows-1252',
13
+ 'win1252': 'windows-1252',
14
+ 'us-ascii': 'ascii',
15
+ 'usascii': 'ascii',
16
+ 'gbk': 'gbk',
17
+ 'gb2312': 'gb2312',
18
+ 'gb18030': 'gb18030',
19
+ 'shift-jis': 'shift_jis',
20
+ 'shiftjis': 'shift_jis',
21
+ 'sjis': 'shift_jis',
22
+ 'euc-jp': 'euc-jp',
23
+ 'eucjp': 'euc-jp',
24
+ 'iso-2022-jp': 'iso-2022-jp',
25
+ 'euc-kr': 'euc-kr',
26
+ 'euckr': 'euc-kr',
27
+ };
28
+ export function normalizeCharset(charset) {
29
+ const lower = charset.toLowerCase().trim();
30
+ return charsetAliases[lower] || lower;
31
+ }
32
+ export function detectFromContentType(contentType) {
33
+ if (!contentType)
34
+ return null;
35
+ const charsetMatch = contentType.match(/charset=["']?([^"';\s]+)["']?/i);
36
+ if (charsetMatch) {
37
+ return normalizeCharset(charsetMatch[1]);
38
+ }
39
+ return null;
40
+ }
41
+ export function detectFromBOM(buffer) {
42
+ if (buffer.length < 2)
43
+ return null;
44
+ if (buffer.length >= 4 && buffer[0] === 0x00 && buffer[1] === 0x00 &&
45
+ buffer[2] === 0xFE && buffer[3] === 0xFF) {
46
+ return { charset: 'utf-32be', source: 'bom', confidence: 'high' };
47
+ }
48
+ if (buffer.length >= 4 && buffer[0] === 0xFF && buffer[1] === 0xFE &&
49
+ buffer[2] === 0x00 && buffer[3] === 0x00) {
50
+ return { charset: 'utf-32le', source: 'bom', confidence: 'high' };
51
+ }
52
+ if (buffer.length >= 3 && buffer[0] === 0xEF && buffer[1] === 0xBB && buffer[2] === 0xBF) {
53
+ return { charset: 'utf-8', source: 'bom', confidence: 'high' };
54
+ }
55
+ if (buffer[0] === 0xFE && buffer[1] === 0xFF) {
56
+ return { charset: 'utf-16be', source: 'bom', confidence: 'high' };
57
+ }
58
+ if (buffer[0] === 0xFF && buffer[1] === 0xFE) {
59
+ return { charset: 'utf-16le', source: 'bom', confidence: 'high' };
60
+ }
61
+ return null;
62
+ }
63
+ export function stripBOM(buffer) {
64
+ const bomInfo = detectFromBOM(buffer);
65
+ if (!bomInfo)
66
+ return buffer;
67
+ switch (bomInfo.charset) {
68
+ case 'utf-8':
69
+ return buffer.slice(3);
70
+ case 'utf-16le':
71
+ case 'utf-16be':
72
+ return buffer.slice(2);
73
+ case 'utf-32le':
74
+ case 'utf-32be':
75
+ return buffer.slice(4);
76
+ default:
77
+ return buffer;
78
+ }
79
+ }
80
+ export function detectFromXMLDeclaration(content) {
81
+ const xmlMatch = content.match(/<\?xml[^?]*encoding=["']([^"']+)["'][^?]*\?>/i);
82
+ if (xmlMatch) {
83
+ return normalizeCharset(xmlMatch[1]);
84
+ }
85
+ return null;
86
+ }
87
+ export function detectFromHTMLMeta(html) {
88
+ const head = html.slice(0, 1024);
89
+ const charsetMeta = head.match(/<meta[^>]+charset=["']?([^"'>\s]+)["']?/i);
90
+ if (charsetMeta) {
91
+ return normalizeCharset(charsetMeta[1]);
92
+ }
93
+ const httpEquiv = head.match(/<meta[^>]+http-equiv=["']?Content-Type["']?[^>]+content=["']?[^"'>]*charset=([^"'>\s;]+)/i);
94
+ if (httpEquiv) {
95
+ return normalizeCharset(httpEquiv[1]);
96
+ }
97
+ const httpEquivReverse = head.match(/<meta[^>]+content=["']?[^"'>]*charset=([^"'>\s;]+)[^>]+http-equiv=["']?Content-Type["']?/i);
98
+ if (httpEquivReverse) {
99
+ return normalizeCharset(httpEquivReverse[1]);
100
+ }
101
+ return null;
102
+ }
103
+ export function detectCharset(buffer, contentType) {
104
+ const headerCharset = detectFromContentType(contentType || null);
105
+ if (headerCharset) {
106
+ return { charset: headerCharset, source: 'header', confidence: 'high' };
107
+ }
108
+ const bomResult = detectFromBOM(buffer);
109
+ if (bomResult) {
110
+ return bomResult;
111
+ }
112
+ const textDecoder = new TextDecoder('utf-8', { fatal: false });
113
+ const text = textDecoder.decode(buffer.slice(0, 1024));
114
+ if (text.trimStart().startsWith('<?xml')) {
115
+ const xmlCharset = detectFromXMLDeclaration(text);
116
+ if (xmlCharset) {
117
+ return { charset: xmlCharset, source: 'xml', confidence: 'medium' };
118
+ }
119
+ }
120
+ if (text.toLowerCase().includes('<html') || text.toLowerCase().includes('<!doctype html')) {
121
+ const htmlCharset = detectFromHTMLMeta(text);
122
+ if (htmlCharset) {
123
+ return { charset: htmlCharset, source: 'html-meta', confidence: 'medium' };
124
+ }
125
+ }
126
+ return { charset: 'utf-8', source: 'default', confidence: 'low' };
127
+ }
128
+ export function decodeText(buffer, charsetOrInfo) {
129
+ let charset = 'utf-8';
130
+ if (typeof charsetOrInfo === 'string') {
131
+ charset = normalizeCharset(charsetOrInfo);
132
+ }
133
+ else if (charsetOrInfo) {
134
+ charset = charsetOrInfo.charset;
135
+ }
136
+ const cleanBuffer = stripBOM(buffer);
137
+ try {
138
+ const decoder = new TextDecoder(charset);
139
+ return decoder.decode(cleanBuffer);
140
+ }
141
+ catch {
142
+ const decoder = new TextDecoder('utf-8', { fatal: false });
143
+ return decoder.decode(cleanBuffer);
144
+ }
145
+ }
146
+ export function getSupportedEncodings() {
147
+ return [
148
+ 'utf-8', 'utf-16le', 'utf-16be',
149
+ 'iso-8859-1', 'iso-8859-2', 'iso-8859-3', 'iso-8859-4', 'iso-8859-5',
150
+ 'iso-8859-6', 'iso-8859-7', 'iso-8859-8', 'iso-8859-9', 'iso-8859-10',
151
+ 'iso-8859-11', 'iso-8859-13', 'iso-8859-14', 'iso-8859-15', 'iso-8859-16',
152
+ 'windows-1250', 'windows-1251', 'windows-1252', 'windows-1253',
153
+ 'windows-1254', 'windows-1255', 'windows-1256', 'windows-1257', 'windows-1258',
154
+ 'koi8-r', 'koi8-u',
155
+ 'gbk', 'gb2312', 'gb18030',
156
+ 'big5', 'big5-hkscs',
157
+ 'shift_jis', 'euc-jp', 'iso-2022-jp',
158
+ 'euc-kr',
159
+ ];
160
+ }
161
+ export function isCharsetSupported(charset) {
162
+ try {
163
+ new TextDecoder(normalizeCharset(charset));
164
+ return true;
165
+ }
166
+ catch {
167
+ return false;
168
+ }
169
+ }
@@ -0,0 +1,20 @@
1
+ import type { AgentOptions, ClientOptions } from '../types/index.js';
2
+ export interface NormalizedConcurrencyConfig {
3
+ max: number;
4
+ requestsPerInterval: number;
5
+ interval: number;
6
+ runner: {
7
+ concurrency: number;
8
+ retries?: number;
9
+ retryDelay?: number;
10
+ };
11
+ agent: AgentOptions & {
12
+ connections: number;
13
+ };
14
+ http2: {
15
+ maxConcurrentStreams: number;
16
+ };
17
+ }
18
+ export declare function normalizeConcurrency(options: Pick<ClientOptions, 'concurrency' | 'http2'>): NormalizedConcurrencyConfig;
19
+ export declare function calculateOptimalConnections(maxConcurrent: number, http2Enabled: boolean, http2MaxStreams?: number, pipelining?: number): number;
20
+ export declare function createBatchConfig(baseConcurrency: NormalizedConcurrencyConfig, batchConcurrency: number, requestCount: number): NormalizedConcurrencyConfig;