recker 1.0.2-0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +109 -0
  3. package/dist/cache/file-storage.d.ts +13 -0
  4. package/dist/cache/file-storage.d.ts.map +1 -0
  5. package/dist/cache/file-storage.js +50 -0
  6. package/dist/cache/memory-storage.d.ts +10 -0
  7. package/dist/cache/memory-storage.d.ts.map +1 -0
  8. package/dist/cache/memory-storage.js +29 -0
  9. package/dist/cache/redis-storage.d.ts +16 -0
  10. package/dist/cache/redis-storage.d.ts.map +1 -0
  11. package/dist/cache/redis-storage.js +25 -0
  12. package/dist/constants.d.ts +19 -0
  13. package/dist/constants.d.ts.map +1 -0
  14. package/dist/constants.js +18 -0
  15. package/dist/contract/index.d.ts +32 -0
  16. package/dist/contract/index.d.ts.map +1 -0
  17. package/dist/contract/index.js +67 -0
  18. package/dist/core/client.d.ts +107 -0
  19. package/dist/core/client.d.ts.map +1 -0
  20. package/dist/core/client.js +475 -0
  21. package/dist/core/errors.d.ts +19 -0
  22. package/dist/core/errors.d.ts.map +1 -0
  23. package/dist/core/errors.js +34 -0
  24. package/dist/core/request-promise.d.ts +24 -0
  25. package/dist/core/request-promise.d.ts.map +1 -0
  26. package/dist/core/request-promise.js +77 -0
  27. package/dist/core/request.d.ts +15 -0
  28. package/dist/core/request.d.ts.map +1 -0
  29. package/dist/core/request.js +44 -0
  30. package/dist/core/response.d.ts +33 -0
  31. package/dist/core/response.d.ts.map +1 -0
  32. package/dist/core/response.js +154 -0
  33. package/dist/index.d.ts +40 -0
  34. package/dist/index.d.ts.map +1 -0
  35. package/dist/index.js +39 -0
  36. package/dist/mcp/client.d.ts +59 -0
  37. package/dist/mcp/client.d.ts.map +1 -0
  38. package/dist/mcp/client.js +195 -0
  39. package/dist/mcp/index.d.ts +3 -0
  40. package/dist/mcp/index.d.ts.map +1 -0
  41. package/dist/mcp/index.js +2 -0
  42. package/dist/mcp/types.d.ts +151 -0
  43. package/dist/mcp/types.d.ts.map +1 -0
  44. package/dist/mcp/types.js +1 -0
  45. package/dist/plugins/cache.d.ts +10 -0
  46. package/dist/plugins/cache.d.ts.map +1 -0
  47. package/dist/plugins/cache.js +72 -0
  48. package/dist/plugins/circuit-breaker.d.ts +14 -0
  49. package/dist/plugins/circuit-breaker.d.ts.map +1 -0
  50. package/dist/plugins/circuit-breaker.js +100 -0
  51. package/dist/plugins/compression.d.ts +5 -0
  52. package/dist/plugins/compression.d.ts.map +1 -0
  53. package/dist/plugins/compression.js +128 -0
  54. package/dist/plugins/cookie-jar.d.ts +6 -0
  55. package/dist/plugins/cookie-jar.d.ts.map +1 -0
  56. package/dist/plugins/cookie-jar.js +72 -0
  57. package/dist/plugins/dedup.d.ts +6 -0
  58. package/dist/plugins/dedup.d.ts.map +1 -0
  59. package/dist/plugins/dedup.js +34 -0
  60. package/dist/plugins/graphql.d.ts +13 -0
  61. package/dist/plugins/graphql.d.ts.map +1 -0
  62. package/dist/plugins/graphql.js +39 -0
  63. package/dist/plugins/har-player.d.ts +7 -0
  64. package/dist/plugins/har-player.d.ts.map +1 -0
  65. package/dist/plugins/har-player.js +53 -0
  66. package/dist/plugins/har-recorder.d.ts +7 -0
  67. package/dist/plugins/har-recorder.d.ts.map +1 -0
  68. package/dist/plugins/har-recorder.js +67 -0
  69. package/dist/plugins/logger.d.ts +11 -0
  70. package/dist/plugins/logger.d.ts.map +1 -0
  71. package/dist/plugins/logger.js +72 -0
  72. package/dist/plugins/pagination.d.ts +17 -0
  73. package/dist/plugins/pagination.d.ts.map +1 -0
  74. package/dist/plugins/pagination.js +105 -0
  75. package/dist/plugins/proxy-rotator.d.ts +8 -0
  76. package/dist/plugins/proxy-rotator.d.ts.map +1 -0
  77. package/dist/plugins/proxy-rotator.js +35 -0
  78. package/dist/plugins/rate-limit.d.ts +8 -0
  79. package/dist/plugins/rate-limit.d.ts.map +1 -0
  80. package/dist/plugins/rate-limit.js +57 -0
  81. package/dist/plugins/retry.d.ts +14 -0
  82. package/dist/plugins/retry.d.ts.map +1 -0
  83. package/dist/plugins/retry.js +92 -0
  84. package/dist/plugins/server-timing.d.ts +8 -0
  85. package/dist/plugins/server-timing.d.ts.map +1 -0
  86. package/dist/plugins/server-timing.js +24 -0
  87. package/dist/plugins/xsrf.d.ts +10 -0
  88. package/dist/plugins/xsrf.d.ts.map +1 -0
  89. package/dist/plugins/xsrf.js +48 -0
  90. package/dist/runner/request-runner.d.ts +47 -0
  91. package/dist/runner/request-runner.d.ts.map +1 -0
  92. package/dist/runner/request-runner.js +89 -0
  93. package/dist/transport/fetch.d.ts +6 -0
  94. package/dist/transport/fetch.d.ts.map +1 -0
  95. package/dist/transport/fetch.js +153 -0
  96. package/dist/transport/undici.d.ts +23 -0
  97. package/dist/transport/undici.d.ts.map +1 -0
  98. package/dist/transport/undici.js +218 -0
  99. package/dist/types/index.d.ts +251 -0
  100. package/dist/types/index.d.ts.map +1 -0
  101. package/dist/types/index.js +1 -0
  102. package/dist/utils/agent-manager.d.ts +29 -0
  103. package/dist/utils/agent-manager.d.ts.map +1 -0
  104. package/dist/utils/agent-manager.js +133 -0
  105. package/dist/utils/body.d.ts +11 -0
  106. package/dist/utils/body.d.ts.map +1 -0
  107. package/dist/utils/body.js +136 -0
  108. package/dist/utils/cert.d.ts +12 -0
  109. package/dist/utils/cert.d.ts.map +1 -0
  110. package/dist/utils/cert.js +32 -0
  111. package/dist/utils/concurrency.d.ts +21 -0
  112. package/dist/utils/concurrency.d.ts.map +1 -0
  113. package/dist/utils/concurrency.js +116 -0
  114. package/dist/utils/dns.d.ts +7 -0
  115. package/dist/utils/dns.d.ts.map +1 -0
  116. package/dist/utils/dns.js +26 -0
  117. package/dist/utils/doh.d.ts +3 -0
  118. package/dist/utils/doh.d.ts.map +1 -0
  119. package/dist/utils/doh.js +35 -0
  120. package/dist/utils/header-parser.d.ts +81 -0
  121. package/dist/utils/header-parser.d.ts.map +1 -0
  122. package/dist/utils/header-parser.js +457 -0
  123. package/dist/utils/html-cleaner.d.ts +2 -0
  124. package/dist/utils/html-cleaner.d.ts.map +1 -0
  125. package/dist/utils/html-cleaner.js +21 -0
  126. package/dist/utils/logger.d.ts +33 -0
  127. package/dist/utils/logger.d.ts.map +1 -0
  128. package/dist/utils/logger.js +160 -0
  129. package/dist/utils/progress.d.ts +4 -0
  130. package/dist/utils/progress.d.ts.map +1 -0
  131. package/dist/utils/progress.js +49 -0
  132. package/dist/utils/request-pool.d.ts +23 -0
  133. package/dist/utils/request-pool.d.ts.map +1 -0
  134. package/dist/utils/request-pool.js +100 -0
  135. package/dist/utils/sse.d.ts +8 -0
  136. package/dist/utils/sse.d.ts.map +1 -0
  137. package/dist/utils/sse.js +62 -0
  138. package/dist/utils/streaming.d.ts +18 -0
  139. package/dist/utils/streaming.d.ts.map +1 -0
  140. package/dist/utils/streaming.js +83 -0
  141. package/dist/utils/task-pool.d.ts +38 -0
  142. package/dist/utils/task-pool.js +104 -0
  143. package/dist/utils/try-fn.d.ts +4 -0
  144. package/dist/utils/try-fn.d.ts.map +1 -0
  145. package/dist/utils/try-fn.js +53 -0
  146. package/dist/utils/upload.d.ts +10 -0
  147. package/dist/utils/upload.d.ts.map +1 -0
  148. package/dist/utils/upload.js +45 -0
  149. package/dist/utils/user-agent.d.ts +45 -0
  150. package/dist/utils/user-agent.d.ts.map +1 -0
  151. package/dist/utils/user-agent.js +100 -0
  152. package/dist/utils/whois.d.ts +15 -0
  153. package/dist/utils/whois.d.ts.map +1 -0
  154. package/dist/utils/whois.js +159 -0
  155. package/dist/websocket/client.d.ts +38 -0
  156. package/dist/websocket/client.d.ts.map +1 -0
  157. package/dist/websocket/client.js +184 -0
  158. package/package.json +100 -0
@@ -0,0 +1,39 @@
1
+ export class GraphQLError extends Error {
2
+ errors;
3
+ response;
4
+ constructor(errors, response) {
5
+ super(errors[0].message);
6
+ this.errors = errors;
7
+ this.response = response;
8
+ this.name = 'GraphQLError';
9
+ }
10
+ }
11
+ export function graphqlPlugin(options = {}) {
12
+ const throwOnErrors = options.throwOnErrors !== false;
13
+ const middleware = async (req, next) => {
14
+ const res = await next(req);
15
+ const contentType = res.headers.get('content-type');
16
+ if (throwOnErrors && contentType && contentType.includes('application/json')) {
17
+ const clone = res.clone();
18
+ try {
19
+ const body = await clone.json();
20
+ if (body && Array.isArray(body.errors) && body.errors.length > 0) {
21
+ throw new GraphQLError(body.errors, res);
22
+ }
23
+ }
24
+ catch (err) {
25
+ if (err instanceof GraphQLError)
26
+ throw err;
27
+ }
28
+ }
29
+ return res;
30
+ };
31
+ return (client) => {
32
+ client.use(middleware);
33
+ };
34
+ }
35
+ export async function graphql(client, query, variables = {}, options = {}) {
36
+ const body = { query, variables };
37
+ const res = await client.post('', body, options).json();
38
+ return res.data;
39
+ }
@@ -0,0 +1,7 @@
1
+ import { Plugin } from '../types/index.js';
2
+ export interface HarPlayerOptions {
3
+ path: string;
4
+ strict?: boolean;
5
+ }
6
+ export declare function harPlayer(options: HarPlayerOptions): Plugin;
7
+ //# sourceMappingURL=har-player.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"har-player.d.ts","sourceRoot":"","sources":["../../src/plugins/har-player.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAA6C,MAAM,mBAAmB,CAAC;AAItF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IAEb,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAgBD,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,MAAM,CAqE3D"}
@@ -0,0 +1,53 @@
1
+ import { readFileSync } from 'node:fs';
2
+ import { HttpResponse } from '../core/response.js';
3
+ export function harPlayer(options) {
4
+ let entries = [];
5
+ try {
6
+ const content = readFileSync(options.path, 'utf-8');
7
+ const har = JSON.parse(content);
8
+ entries = har.log.entries;
9
+ }
10
+ catch (err) {
11
+ console.warn(`[Recker] Failed to load HAR file: ${options.path}`, err);
12
+ }
13
+ const matchEntry = (req, entry) => {
14
+ if (req.method !== entry.request.method)
15
+ return false;
16
+ if (req.url !== entry.request.url)
17
+ return false;
18
+ if (entry.request.postData?.text && req.body) {
19
+ if (String(req.body) !== entry.request.postData.text) {
20
+ try {
21
+ const reqJson = JSON.parse(String(req.body));
22
+ const entryJson = JSON.parse(entry.request.postData.text);
23
+ if (JSON.stringify(reqJson) !== JSON.stringify(entryJson))
24
+ return false;
25
+ }
26
+ catch {
27
+ return false;
28
+ }
29
+ }
30
+ }
31
+ return true;
32
+ };
33
+ const middleware = async (req, next) => {
34
+ const entry = entries.find(e => matchEntry(req, e));
35
+ if (entry) {
36
+ const headers = new Headers();
37
+ entry.response.headers.forEach(h => headers.append(h.name, h.value));
38
+ const nativeRes = new Response(entry.response.content.text, {
39
+ status: entry.response.status,
40
+ statusText: entry.response.statusText,
41
+ headers: headers
42
+ });
43
+ return new HttpResponse(nativeRes);
44
+ }
45
+ if (options.strict) {
46
+ throw new Error(`[Recker HAR Player] No matching recording found for ${req.method} ${req.url}`);
47
+ }
48
+ return next(req);
49
+ };
50
+ return (client) => {
51
+ client.use(middleware);
52
+ };
53
+ }
@@ -0,0 +1,7 @@
1
+ import { Plugin } from '../types/index.js';
2
+ export interface HarOptions {
3
+ path?: string;
4
+ onEntry?: (entry: any) => void;
5
+ }
6
+ export declare function harRecorder(options?: HarOptions): Plugin;
7
+ //# sourceMappingURL=har-recorder.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"har-recorder.d.ts","sourceRoot":"","sources":["../../src/plugins/har-recorder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAiC,MAAM,mBAAmB,CAAC;AAG1E,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC;CAChC;AAED,wBAAgB,WAAW,CAAC,OAAO,GAAE,UAAe,GAAG,MAAM,CA2E5D"}
@@ -0,0 +1,67 @@
1
+ import { writeFileSync } from 'node:fs';
2
+ export function harRecorder(options = {}) {
3
+ const entries = [];
4
+ const startTime = new Date().toISOString();
5
+ return (client) => {
6
+ const requestMap = new WeakMap();
7
+ client.beforeRequest((req) => {
8
+ requestMap.set(req, { start: Date.now(), req });
9
+ });
10
+ client.afterResponse(async (req, res) => {
11
+ const meta = requestMap.get(req);
12
+ if (!meta)
13
+ return;
14
+ const time = Date.now() - meta.start;
15
+ const entry = {
16
+ startedDateTime: new Date(meta.start).toISOString(),
17
+ time: time,
18
+ request: {
19
+ method: req.method,
20
+ url: req.url,
21
+ httpVersion: "HTTP/1.1",
22
+ cookies: [],
23
+ headers: [...req.headers].map(([name, value]) => ({ name, value })),
24
+ queryString: [],
25
+ headersSize: -1,
26
+ bodySize: -1
27
+ },
28
+ response: {
29
+ status: res.status,
30
+ statusText: res.statusText,
31
+ httpVersion: "HTTP/1.1",
32
+ cookies: [],
33
+ headers: [...res.headers].map(([name, value]) => ({ name, value })),
34
+ content: {
35
+ size: -1,
36
+ mimeType: res.headers.get('content-type') || '',
37
+ text: ''
38
+ },
39
+ redirectURL: "",
40
+ headersSize: -1,
41
+ bodySize: -1
42
+ },
43
+ cache: {},
44
+ timings: {
45
+ send: 0,
46
+ wait: res.timings?.firstByte || 0,
47
+ receive: 0
48
+ }
49
+ };
50
+ entries.push(entry);
51
+ if (options.onEntry) {
52
+ options.onEntry(entry);
53
+ }
54
+ if (options.path) {
55
+ const har = {
56
+ log: {
57
+ version: "1.2",
58
+ creator: { name: "Recker", version: "1.0.0" },
59
+ pages: [],
60
+ entries: entries
61
+ }
62
+ };
63
+ writeFileSync(options.path, JSON.stringify(har, null, 2));
64
+ }
65
+ });
66
+ };
67
+ }
@@ -0,0 +1,11 @@
1
+ import { Plugin, ReckerRequest } from '../types/index.js';
2
+ export interface LoggerOptions {
3
+ log?: (message: string) => void;
4
+ logError?: (message: string, error: any) => void;
5
+ showHeaders?: boolean;
6
+ showBody?: boolean;
7
+ colors?: boolean;
8
+ }
9
+ export declare function logger(options?: LoggerOptions): Plugin;
10
+ export declare function toCurl(req: ReckerRequest): string;
11
+ //# sourceMappingURL=logger.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/plugins/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,aAAa,EAAkB,MAAM,mBAAmB,CAAC;AAE1E,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC;IACjD,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAcD,wBAAgB,MAAM,CAAC,OAAO,GAAE,aAAkB,GAAG,MAAM,CAkD1D;AAED,wBAAgB,MAAM,CAAC,GAAG,EAAE,aAAa,GAAG,MAAM,CA4BjD"}
@@ -0,0 +1,72 @@
1
+ const ANSI = {
2
+ RESET: "\x1b[0m",
3
+ BOLD: "\x1b[1m",
4
+ DIM: "\x1b[2m",
5
+ RED: "\x1b[31m",
6
+ GREEN: "\x1b[32m",
7
+ YELLOW: "\x1b[33m",
8
+ BLUE: "\x1b[34m",
9
+ MAGENTA: "\x1b[35m",
10
+ CYAN: "\x1b[36m",
11
+ };
12
+ export function logger(options = {}) {
13
+ const log = options.log || console.log;
14
+ const logError = options.logError || console.error;
15
+ const showHeaders = options.showHeaders || false;
16
+ const showBody = options.showBody || false;
17
+ const useColors = options.colors !== false;
18
+ const c = (color, text) => useColors ? `${ANSI[color]}${text}${ANSI.RESET}` : text;
19
+ const timers = new WeakMap();
20
+ return (client) => {
21
+ client.beforeRequest((req) => {
22
+ timers.set(req, performance.now());
23
+ const method = c('BLUE', req.method.padEnd(7));
24
+ const url = c('CYAN', req.url);
25
+ log(`${c('BOLD', '-->')} ${method} ${url}`);
26
+ if (showHeaders) {
27
+ req.headers.forEach((v, k) => log(` ${c('DIM', k)}: ${v}`));
28
+ }
29
+ if (showBody && req.body) {
30
+ log(` ${c('DIM', 'Body')}: ${typeof req.body === 'string' ? req.body : '[Object/Stream]'}`);
31
+ }
32
+ });
33
+ client.afterResponse((req, res) => {
34
+ const start = timers.get(req);
35
+ const duration = start ? (performance.now() - start).toFixed(0) + 'ms' : '?';
36
+ const statusColor = res.ok ? 'GREEN' : 'RED';
37
+ const status = c(statusColor, String(res.status));
38
+ const statusText = c(statusColor, res.statusText);
39
+ log(`${c('BOLD', '<--')} ${c('BLUE', req.method.padEnd(7))} ${c('CYAN', req.url)} ${status} ${statusText} ${c('YELLOW', duration)}`);
40
+ if (showHeaders) {
41
+ res.headers.forEach((v, k) => log(` ${c('DIM', k)}: ${v}`));
42
+ }
43
+ });
44
+ client.onError((err, req) => {
45
+ const start = timers.get(req);
46
+ const duration = start ? (performance.now() - start).toFixed(0) + 'ms' : '?';
47
+ logError(`${c('RED', '!!!')} ${c('BLUE', req.method)} ${c('CYAN', req.url)} ${c('RED', err.message)} ${c('YELLOW', duration)}`, err);
48
+ });
49
+ };
50
+ }
51
+ export function toCurl(req) {
52
+ const parts = ['curl'];
53
+ if (req.method !== 'GET') {
54
+ parts.push(`-X ${req.method}`);
55
+ }
56
+ parts.push(`'${req.url}'`);
57
+ req.headers.forEach((value, key) => {
58
+ parts.push(`-H '${key}: ${value}'`);
59
+ });
60
+ if (req.body) {
61
+ if (typeof req.body === 'string') {
62
+ parts.push(`-d '${req.body}'`);
63
+ }
64
+ else if (req.body instanceof URLSearchParams) {
65
+ parts.push(`-d '${req.body.toString()}'`);
66
+ }
67
+ else {
68
+ parts.push(`-d '[Body]'`);
69
+ }
70
+ }
71
+ return parts.join(' ');
72
+ }
@@ -0,0 +1,17 @@
1
+ import { ReckerResponse, PageResult } from '../types/index.js';
2
+ interface IClient {
3
+ request(url: string, options?: any): Promise<ReckerResponse>;
4
+ }
5
+ export interface PaginationOptions<T = any> {
6
+ getItems?: (data: any) => T[];
7
+ getNextUrl?: (response: ReckerResponse, data: any, currentUrl: string) => string | null;
8
+ maxPages?: number;
9
+ pageParam?: string;
10
+ limitParam?: string;
11
+ resultsPath?: string;
12
+ nextCursorPath?: string;
13
+ }
14
+ export declare function streamPages<T = any>(client: IClient, url: string, requestOptions?: any, paginationOptions?: PaginationOptions): AsyncGenerator<PageResult<T>>;
15
+ export declare function paginate<T>(client: IClient, url: string, requestOptions?: any, paginationOptions?: PaginationOptions<T>): AsyncGenerator<T>;
16
+ export {};
17
+ //# sourceMappingURL=pagination.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pagination.d.ts","sourceRoot":"","sources":["../../src/plugins/pagination.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,cAAc,EAAiB,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG7F,UAAU,OAAO;IACb,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CAChE;AAED,MAAM,WAAW,iBAAiB,CAAC,CAAC,GAAG,GAAG;IACxC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC;IAC9B,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACxF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAKD,wBAAuB,WAAW,CAAC,CAAC,GAAG,GAAG,EACxC,MAAM,EAAE,OAAO,EACf,GAAG,EAAE,MAAM,EACX,cAAc,GAAE,GAAQ,EACxB,iBAAiB,GAAE,iBAAsB,GACxC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CA8F/B;AAED,wBAAuB,QAAQ,CAAC,CAAC,EAC/B,MAAM,EAAE,OAAO,EACf,GAAG,EAAE,MAAM,EACX,cAAc,GAAE,GAAQ,EACxB,iBAAiB,GAAE,iBAAiB,CAAC,CAAC,CAAM,GAC3C,cAAc,CAAC,CAAC,CAAC,CAuBnB"}
@@ -0,0 +1,105 @@
1
+ export async function* streamPages(client, url, requestOptions = {}, paginationOptions = {}) {
2
+ let currentUrl = url;
3
+ let pageCount = 1;
4
+ const maxPages = paginationOptions.maxPages || Infinity;
5
+ while (currentUrl && pageCount <= maxPages) {
6
+ const response = await client.request(currentUrl, { ...requestOptions, method: 'GET' });
7
+ const data = await response.json();
8
+ yield {
9
+ data,
10
+ response,
11
+ pageNumber: pageCount
12
+ };
13
+ pageCount++;
14
+ if (paginationOptions.getNextUrl) {
15
+ currentUrl = paginationOptions.getNextUrl(response, data, currentUrl);
16
+ continue;
17
+ }
18
+ if (paginationOptions.nextCursorPath) {
19
+ const parts = paginationOptions.nextCursorPath.split('.');
20
+ let cursor = data;
21
+ for (const part of parts) {
22
+ if (cursor && typeof cursor === 'object') {
23
+ cursor = cursor[part];
24
+ }
25
+ else {
26
+ cursor = undefined;
27
+ break;
28
+ }
29
+ }
30
+ if (cursor) {
31
+ const cursorParamName = paginationOptions.pageParam || 'cursor';
32
+ const isAbsolute = currentUrl.startsWith('http');
33
+ const urlObj = new URL(isAbsolute ? currentUrl : `http://dummy-base${currentUrl.startsWith('/') ? '' : '/'}${currentUrl}`);
34
+ urlObj.searchParams.set(cursorParamName, String(cursor));
35
+ if (isAbsolute) {
36
+ currentUrl = urlObj.toString();
37
+ }
38
+ else {
39
+ currentUrl = urlObj.pathname + urlObj.search;
40
+ }
41
+ }
42
+ else {
43
+ currentUrl = null;
44
+ }
45
+ continue;
46
+ }
47
+ if (paginationOptions.pageParam) {
48
+ const isAbsolute = currentUrl.startsWith('http');
49
+ const urlObj = new URL(isAbsolute ? currentUrl : `http://dummy-base${currentUrl.startsWith('/') ? '' : '/'}${currentUrl}`);
50
+ const paramName = paginationOptions.pageParam;
51
+ const currentVal = parseInt(urlObj.searchParams.get(paramName) || String(pageCount - 1), 10);
52
+ let isEmpty = false;
53
+ if (Array.isArray(data) && data.length === 0)
54
+ isEmpty = true;
55
+ if (data && Array.isArray(data.data) && data.data.length === 0)
56
+ isEmpty = true;
57
+ if (data && Array.isArray(data.items) && data.items.length === 0)
58
+ isEmpty = true;
59
+ if (isEmpty) {
60
+ currentUrl = null;
61
+ }
62
+ else {
63
+ const nextPage = (isNaN(currentVal) ? 1 : currentVal) + 1;
64
+ urlObj.searchParams.set(paramName, String(nextPage));
65
+ currentUrl = isAbsolute ? urlObj.toString() : (urlObj.pathname + urlObj.search);
66
+ }
67
+ continue;
68
+ }
69
+ const linkHeader = response.headers.get('link');
70
+ if (linkHeader) {
71
+ const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/);
72
+ currentUrl = match ? match[1] : null;
73
+ }
74
+ else {
75
+ currentUrl = null;
76
+ }
77
+ }
78
+ }
79
+ export async function* paginate(client, url, requestOptions = {}, paginationOptions = {}) {
80
+ for await (const page of streamPages(client, url, requestOptions, paginationOptions)) {
81
+ const data = page.data;
82
+ let items = [];
83
+ if (paginationOptions.getItems) {
84
+ items = paginationOptions.getItems(data);
85
+ }
86
+ else if (paginationOptions.resultsPath) {
87
+ items = paginationOptions.resultsPath.split('.').reduce((o, i) => o?.[i], data) || [];
88
+ }
89
+ else if (Array.isArray(data)) {
90
+ items = data;
91
+ }
92
+ else if (data && Array.isArray(data.data)) {
93
+ items = data.data;
94
+ }
95
+ else if (data && Array.isArray(data.items)) {
96
+ items = data.items;
97
+ }
98
+ else {
99
+ items = [data];
100
+ }
101
+ for (const item of items) {
102
+ yield item;
103
+ }
104
+ }
105
+ }
@@ -0,0 +1,8 @@
1
+ import { Plugin } from '../types/index.js';
2
+ export interface ProxyRotatorOptions {
3
+ proxies: string[];
4
+ strategy?: 'round-robin' | 'random';
5
+ failover?: boolean;
6
+ }
7
+ export declare function proxyRotator(options: ProxyRotatorOptions): Plugin;
8
+ //# sourceMappingURL=proxy-rotator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"proxy-rotator.d.ts","sourceRoot":"","sources":["../../src/plugins/proxy-rotator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAiB,MAAM,mBAAmB,CAAC;AAG1D,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;IAEpC,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,MAAM,CAoDjE"}
@@ -0,0 +1,35 @@
1
+ import { ProxyAgent } from 'undici';
2
+ export function proxyRotator(options) {
3
+ const proxies = options.proxies.map(url => ({
4
+ url,
5
+ agent: new ProxyAgent(url),
6
+ failures: 0
7
+ }));
8
+ let index = 0;
9
+ const getNextProxy = () => {
10
+ if (proxies.length === 0)
11
+ return null;
12
+ let selected;
13
+ if (options.strategy === 'random') {
14
+ selected = proxies[Math.floor(Math.random() * proxies.length)];
15
+ }
16
+ else {
17
+ selected = proxies[index];
18
+ index = (index + 1) % proxies.length;
19
+ }
20
+ return selected;
21
+ };
22
+ return (client) => {
23
+ client.beforeRequest((req) => {
24
+ const proxy = getNextProxy();
25
+ if (proxy) {
26
+ req._dispatcher = proxy.agent;
27
+ req._proxyUrl = proxy.url;
28
+ }
29
+ });
30
+ client.onError((err, req) => {
31
+ if (options.failover && req._proxyUrl) {
32
+ }
33
+ });
34
+ };
35
+ }
@@ -0,0 +1,8 @@
1
+ import { Plugin } from '../types/index.js';
2
+ export interface RateLimitOptions {
3
+ concurrency?: number;
4
+ requestsPerInterval?: number;
5
+ interval?: number;
6
+ }
7
+ export declare function rateLimit(options: RateLimitOptions): Plugin;
8
+ //# sourceMappingURL=rate-limit.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rate-limit.d.ts","sourceRoot":"","sources":["../../src/plugins/rate-limit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAOvD,MAAM,WAAW,gBAAgB;IAK/B,WAAW,CAAC,EAAE,MAAM,CAAC;IAIrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAI7B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAiBD,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,MAAM,CAqE3D"}
@@ -0,0 +1,57 @@
1
+ export function rateLimit(options) {
2
+ const concurrency = options.concurrency || Infinity;
3
+ const limit = options.requestsPerInterval || Infinity;
4
+ const interval = options.interval || 1000;
5
+ let activeCount = 0;
6
+ const queue = [];
7
+ let tokens = limit;
8
+ let lastRefill = Date.now();
9
+ const refillTokens = () => {
10
+ const now = Date.now();
11
+ const timePassed = now - lastRefill;
12
+ if (timePassed >= interval) {
13
+ const intervalsPassed = Math.floor(timePassed / interval);
14
+ tokens = Math.min(limit, tokens + (intervalsPassed * limit));
15
+ lastRefill = now;
16
+ }
17
+ };
18
+ const processQueue = () => {
19
+ if (queue.length === 0)
20
+ return;
21
+ refillTokens();
22
+ while (queue.length > 0 && activeCount < concurrency && tokens > 0) {
23
+ const next = queue.shift();
24
+ if (next) {
25
+ activeCount++;
26
+ tokens--;
27
+ next();
28
+ }
29
+ }
30
+ if (queue.length > 0 && tokens <= 0 && limit !== Infinity) {
31
+ const timeToWait = interval - (Date.now() - lastRefill);
32
+ setTimeout(processQueue, Math.max(0, timeToWait));
33
+ }
34
+ };
35
+ const rateLimitMiddleware = (req, next) => {
36
+ return new Promise((resolve, reject) => {
37
+ const run = async () => {
38
+ try {
39
+ const res = await next(req);
40
+ resolve(res);
41
+ }
42
+ catch (err) {
43
+ reject(err);
44
+ }
45
+ finally {
46
+ activeCount--;
47
+ processQueue();
48
+ }
49
+ };
50
+ queue.push(run);
51
+ processQueue();
52
+ });
53
+ };
54
+ return (client) => {
55
+ client.use(rateLimitMiddleware);
56
+ };
57
+ }
@@ -0,0 +1,14 @@
1
+ import { Plugin } from '../types/index.js';
2
+ export type BackoffStrategy = 'linear' | 'exponential' | 'decorrelated';
3
+ export interface RetryOptions {
4
+ maxAttempts?: number;
5
+ delay?: number;
6
+ maxDelay?: number;
7
+ backoff?: BackoffStrategy;
8
+ jitter?: boolean;
9
+ statusCodes?: number[];
10
+ shouldRetry?: (error: unknown) => boolean;
11
+ onRetry?: (attempt: number, error: unknown, delay: number) => void;
12
+ }
13
+ export declare function retry(options?: RetryOptions): Plugin;
14
+ //# sourceMappingURL=retry.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../../src/plugins/retry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAGvD,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,aAAa,GAAG,cAAc,CAAC;AAExE,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IAC1C,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACpE;AAgDD,wBAAgB,KAAK,CAAC,OAAO,GAAE,YAAiB,GAAG,MAAM,CAgFxD"}
@@ -0,0 +1,92 @@
1
+ import { HttpError, NetworkError, TimeoutError } from '../core/errors.js';
2
+ function calculateDelay(attempt, baseDelay, maxDelay, strategy, useJitter) {
3
+ let calculatedDelay;
4
+ switch (strategy) {
5
+ case 'linear':
6
+ calculatedDelay = baseDelay * attempt;
7
+ break;
8
+ case 'exponential':
9
+ calculatedDelay = Math.pow(2, attempt - 1) * baseDelay;
10
+ break;
11
+ case 'decorrelated':
12
+ const prevDelay = attempt === 1 ? baseDelay : Math.pow(2, attempt - 2) * baseDelay;
13
+ calculatedDelay = Math.random() * (prevDelay * 3 - baseDelay) + baseDelay;
14
+ break;
15
+ default:
16
+ calculatedDelay = baseDelay * attempt;
17
+ }
18
+ calculatedDelay = Math.min(calculatedDelay, maxDelay);
19
+ if (useJitter) {
20
+ const jitterRange = calculatedDelay * 0.25;
21
+ const jitterAmount = (Math.random() * jitterRange * 2) - jitterRange;
22
+ calculatedDelay += jitterAmount;
23
+ }
24
+ return Math.max(0, Math.floor(calculatedDelay));
25
+ }
26
+ export function retry(options = {}) {
27
+ const maxAttempts = options.maxAttempts || 3;
28
+ const baseDelay = options.delay || 1000;
29
+ const maxDelay = options.maxDelay || 30_000;
30
+ const backoffStrategy = options.backoff || 'exponential';
31
+ const useJitter = options.jitter !== false;
32
+ const statusCodes = options.statusCodes || [408, 429, 500, 502, 503, 504];
33
+ const onRetry = options.onRetry;
34
+ const defaultShouldRetry = (error) => {
35
+ if (error instanceof NetworkError)
36
+ return true;
37
+ if (error instanceof TimeoutError)
38
+ return true;
39
+ if (error instanceof HttpError) {
40
+ return statusCodes.includes(error.status);
41
+ }
42
+ if (error && typeof error === 'object' && 'code' in error) {
43
+ const code = error.code;
44
+ return code === 'ECONNRESET' || code === 'ETIMEDOUT' || code === 'ENOTFOUND';
45
+ }
46
+ return false;
47
+ };
48
+ const shouldRetry = options.shouldRetry || defaultShouldRetry;
49
+ return (client) => {
50
+ const middleware = async (req, next) => {
51
+ let attempt = 0;
52
+ while (true) {
53
+ try {
54
+ attempt++;
55
+ const res = await next(req);
56
+ if (attempt < maxAttempts && !res.ok && statusCodes.includes(res.status)) {
57
+ const delayMs = calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
58
+ const err = new HttpError(res, req);
59
+ if (onRetry) {
60
+ onRetry(attempt, err, delayMs);
61
+ }
62
+ if (client.hooks && client.hooks.onRetry) {
63
+ for (const hook of client.hooks.onRetry) {
64
+ await hook(err, attempt, delayMs, req);
65
+ }
66
+ }
67
+ await new Promise(resolve => setTimeout(resolve, delayMs));
68
+ continue;
69
+ }
70
+ return res;
71
+ }
72
+ catch (error) {
73
+ if (attempt < maxAttempts && shouldRetry(error)) {
74
+ const delayMs = calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
75
+ if (onRetry) {
76
+ onRetry(attempt, error, delayMs);
77
+ }
78
+ if (client.hooks && client.hooks.onRetry) {
79
+ for (const hook of client.hooks.onRetry) {
80
+ await hook(error, attempt, delayMs, req);
81
+ }
82
+ }
83
+ await new Promise(resolve => setTimeout(resolve, delayMs));
84
+ continue;
85
+ }
86
+ throw error;
87
+ }
88
+ }
89
+ };
90
+ client.use(middleware);
91
+ };
92
+ }
@@ -0,0 +1,8 @@
1
+ import { Plugin } from '../types/index.js';
2
+ export interface ServerTiming {
3
+ name: string;
4
+ duration?: number;
5
+ description?: string;
6
+ }
7
+ export declare function serverTiming(): Plugin;
8
+ //# sourceMappingURL=server-timing.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"server-timing.d.ts","sourceRoot":"","sources":["../../src/plugins/server-timing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAiC,MAAM,mBAAmB,CAAC;AAE1E,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,YAAY,IAAI,MAAM,CAyBrC"}