recker 1.0.2-0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +109 -0
- package/dist/cache/file-storage.d.ts +13 -0
- package/dist/cache/file-storage.d.ts.map +1 -0
- package/dist/cache/file-storage.js +50 -0
- package/dist/cache/memory-storage.d.ts +10 -0
- package/dist/cache/memory-storage.d.ts.map +1 -0
- package/dist/cache/memory-storage.js +29 -0
- package/dist/cache/redis-storage.d.ts +16 -0
- package/dist/cache/redis-storage.d.ts.map +1 -0
- package/dist/cache/redis-storage.js +25 -0
- package/dist/constants.d.ts +19 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +18 -0
- package/dist/contract/index.d.ts +32 -0
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +67 -0
- package/dist/core/client.d.ts +107 -0
- package/dist/core/client.d.ts.map +1 -0
- package/dist/core/client.js +475 -0
- package/dist/core/errors.d.ts +19 -0
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/errors.js +34 -0
- package/dist/core/request-promise.d.ts +24 -0
- package/dist/core/request-promise.d.ts.map +1 -0
- package/dist/core/request-promise.js +77 -0
- package/dist/core/request.d.ts +15 -0
- package/dist/core/request.d.ts.map +1 -0
- package/dist/core/request.js +44 -0
- package/dist/core/response.d.ts +33 -0
- package/dist/core/response.d.ts.map +1 -0
- package/dist/core/response.js +154 -0
- package/dist/index.d.ts +40 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +39 -0
- package/dist/mcp/client.d.ts +59 -0
- package/dist/mcp/client.d.ts.map +1 -0
- package/dist/mcp/client.js +195 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +2 -0
- package/dist/mcp/types.d.ts +151 -0
- package/dist/mcp/types.d.ts.map +1 -0
- package/dist/mcp/types.js +1 -0
- package/dist/plugins/cache.d.ts +10 -0
- package/dist/plugins/cache.d.ts.map +1 -0
- package/dist/plugins/cache.js +72 -0
- package/dist/plugins/circuit-breaker.d.ts +14 -0
- package/dist/plugins/circuit-breaker.d.ts.map +1 -0
- package/dist/plugins/circuit-breaker.js +100 -0
- package/dist/plugins/compression.d.ts +5 -0
- package/dist/plugins/compression.d.ts.map +1 -0
- package/dist/plugins/compression.js +128 -0
- package/dist/plugins/cookie-jar.d.ts +6 -0
- package/dist/plugins/cookie-jar.d.ts.map +1 -0
- package/dist/plugins/cookie-jar.js +72 -0
- package/dist/plugins/dedup.d.ts +6 -0
- package/dist/plugins/dedup.d.ts.map +1 -0
- package/dist/plugins/dedup.js +34 -0
- package/dist/plugins/graphql.d.ts +13 -0
- package/dist/plugins/graphql.d.ts.map +1 -0
- package/dist/plugins/graphql.js +39 -0
- package/dist/plugins/har-player.d.ts +7 -0
- package/dist/plugins/har-player.d.ts.map +1 -0
- package/dist/plugins/har-player.js +53 -0
- package/dist/plugins/har-recorder.d.ts +7 -0
- package/dist/plugins/har-recorder.d.ts.map +1 -0
- package/dist/plugins/har-recorder.js +67 -0
- package/dist/plugins/logger.d.ts +11 -0
- package/dist/plugins/logger.d.ts.map +1 -0
- package/dist/plugins/logger.js +72 -0
- package/dist/plugins/pagination.d.ts +17 -0
- package/dist/plugins/pagination.d.ts.map +1 -0
- package/dist/plugins/pagination.js +105 -0
- package/dist/plugins/proxy-rotator.d.ts +8 -0
- package/dist/plugins/proxy-rotator.d.ts.map +1 -0
- package/dist/plugins/proxy-rotator.js +35 -0
- package/dist/plugins/rate-limit.d.ts +8 -0
- package/dist/plugins/rate-limit.d.ts.map +1 -0
- package/dist/plugins/rate-limit.js +57 -0
- package/dist/plugins/retry.d.ts +14 -0
- package/dist/plugins/retry.d.ts.map +1 -0
- package/dist/plugins/retry.js +92 -0
- package/dist/plugins/server-timing.d.ts +8 -0
- package/dist/plugins/server-timing.d.ts.map +1 -0
- package/dist/plugins/server-timing.js +24 -0
- package/dist/plugins/xsrf.d.ts +10 -0
- package/dist/plugins/xsrf.d.ts.map +1 -0
- package/dist/plugins/xsrf.js +48 -0
- package/dist/runner/request-runner.d.ts +47 -0
- package/dist/runner/request-runner.d.ts.map +1 -0
- package/dist/runner/request-runner.js +89 -0
- package/dist/transport/fetch.d.ts +6 -0
- package/dist/transport/fetch.d.ts.map +1 -0
- package/dist/transport/fetch.js +153 -0
- package/dist/transport/undici.d.ts +23 -0
- package/dist/transport/undici.d.ts.map +1 -0
- package/dist/transport/undici.js +218 -0
- package/dist/types/index.d.ts +251 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +1 -0
- package/dist/utils/agent-manager.d.ts +29 -0
- package/dist/utils/agent-manager.d.ts.map +1 -0
- package/dist/utils/agent-manager.js +133 -0
- package/dist/utils/body.d.ts +11 -0
- package/dist/utils/body.d.ts.map +1 -0
- package/dist/utils/body.js +136 -0
- package/dist/utils/cert.d.ts +12 -0
- package/dist/utils/cert.d.ts.map +1 -0
- package/dist/utils/cert.js +32 -0
- package/dist/utils/concurrency.d.ts +21 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +116 -0
- package/dist/utils/dns.d.ts +7 -0
- package/dist/utils/dns.d.ts.map +1 -0
- package/dist/utils/dns.js +26 -0
- package/dist/utils/doh.d.ts +3 -0
- package/dist/utils/doh.d.ts.map +1 -0
- package/dist/utils/doh.js +35 -0
- package/dist/utils/header-parser.d.ts +81 -0
- package/dist/utils/header-parser.d.ts.map +1 -0
- package/dist/utils/header-parser.js +457 -0
- package/dist/utils/html-cleaner.d.ts +2 -0
- package/dist/utils/html-cleaner.d.ts.map +1 -0
- package/dist/utils/html-cleaner.js +21 -0
- package/dist/utils/logger.d.ts +33 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +160 -0
- package/dist/utils/progress.d.ts +4 -0
- package/dist/utils/progress.d.ts.map +1 -0
- package/dist/utils/progress.js +49 -0
- package/dist/utils/request-pool.d.ts +23 -0
- package/dist/utils/request-pool.d.ts.map +1 -0
- package/dist/utils/request-pool.js +100 -0
- package/dist/utils/sse.d.ts +8 -0
- package/dist/utils/sse.d.ts.map +1 -0
- package/dist/utils/sse.js +62 -0
- package/dist/utils/streaming.d.ts +18 -0
- package/dist/utils/streaming.d.ts.map +1 -0
- package/dist/utils/streaming.js +83 -0
- package/dist/utils/task-pool.d.ts +38 -0
- package/dist/utils/task-pool.js +104 -0
- package/dist/utils/try-fn.d.ts +4 -0
- package/dist/utils/try-fn.d.ts.map +1 -0
- package/dist/utils/try-fn.js +53 -0
- package/dist/utils/upload.d.ts +10 -0
- package/dist/utils/upload.d.ts.map +1 -0
- package/dist/utils/upload.js +45 -0
- package/dist/utils/user-agent.d.ts +45 -0
- package/dist/utils/user-agent.d.ts.map +1 -0
- package/dist/utils/user-agent.js +100 -0
- package/dist/utils/whois.d.ts +15 -0
- package/dist/utils/whois.d.ts.map +1 -0
- package/dist/utils/whois.js +159 -0
- package/dist/websocket/client.d.ts +38 -0
- package/dist/websocket/client.d.ts.map +1 -0
- package/dist/websocket/client.js +184 -0
- package/package.json +100 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export function serverTiming() {
|
|
2
|
+
return (client) => {
|
|
3
|
+
client.afterResponse((req, res) => {
|
|
4
|
+
const header = res.headers.get('server-timing');
|
|
5
|
+
if (!header)
|
|
6
|
+
return;
|
|
7
|
+
const timings = header.split(',').map(entry => {
|
|
8
|
+
const parts = entry.split(';');
|
|
9
|
+
const name = parts[0].trim();
|
|
10
|
+
let duration;
|
|
11
|
+
let description;
|
|
12
|
+
for (let i = 1; i < parts.length; i++) {
|
|
13
|
+
const [key, val] = parts[i].split('=').map(s => s.trim());
|
|
14
|
+
if (key === 'dur')
|
|
15
|
+
duration = parseFloat(val);
|
|
16
|
+
if (key === 'desc')
|
|
17
|
+
description = val?.replace(/"/g, '');
|
|
18
|
+
}
|
|
19
|
+
return { name, duration, description };
|
|
20
|
+
});
|
|
21
|
+
res.serverTimings = timings;
|
|
22
|
+
});
|
|
23
|
+
};
|
|
24
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Middleware } from '../types/index.js';
|
|
2
|
+
export interface XSRFPluginOptions {
|
|
3
|
+
cookieName?: string;
|
|
4
|
+
headerName?: string;
|
|
5
|
+
token?: string;
|
|
6
|
+
cookies?: string;
|
|
7
|
+
}
|
|
8
|
+
export declare function xsrf(options?: XSRFPluginOptions): Middleware;
|
|
9
|
+
export declare function createXSRFMiddleware(config: boolean | XSRFPluginOptions): Middleware | null;
|
|
10
|
+
//# sourceMappingURL=xsrf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xsrf.d.ts","sourceRoot":"","sources":["../../src/plugins/xsrf.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,MAAM,WAAW,iBAAiB;IAKhC,UAAU,CAAC,EAAE,MAAM,CAAC;IAKpB,UAAU,CAAC,EAAE,MAAM,CAAC;IAKpB,KAAK,CAAC,EAAE,MAAM,CAAC;IAKf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAuED,wBAAgB,IAAI,CAAC,OAAO,GAAE,iBAAsB,GAAG,UAAU,CAsBhE;AAMD,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,OAAO,GAAG,iBAAiB,GAAG,UAAU,GAAG,IAAI,CAU3F"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
function parseCookies(cookieString) {
|
|
2
|
+
const cookies = {};
|
|
3
|
+
if (!cookieString) {
|
|
4
|
+
return cookies;
|
|
5
|
+
}
|
|
6
|
+
const pairs = cookieString.split(';');
|
|
7
|
+
for (const pair of pairs) {
|
|
8
|
+
const [key, ...valueParts] = pair.split('=');
|
|
9
|
+
const trimmedKey = key?.trim();
|
|
10
|
+
const value = valueParts.join('=').trim();
|
|
11
|
+
if (trimmedKey) {
|
|
12
|
+
cookies[trimmedKey] = decodeURIComponent(value || '');
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
return cookies;
|
|
16
|
+
}
|
|
17
|
+
function getXSRFToken(cookieName, cookies) {
|
|
18
|
+
if (cookies) {
|
|
19
|
+
const parsed = parseCookies(cookies);
|
|
20
|
+
return parsed[cookieName] || null;
|
|
21
|
+
}
|
|
22
|
+
if (typeof document !== 'undefined' && document.cookie) {
|
|
23
|
+
const parsed = parseCookies(document.cookie);
|
|
24
|
+
return parsed[cookieName] || null;
|
|
25
|
+
}
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
export function xsrf(options = {}) {
|
|
29
|
+
const { cookieName = 'XSRF-TOKEN', headerName = 'X-XSRF-TOKEN', token: manualToken, cookies } = options;
|
|
30
|
+
return async (req, next) => {
|
|
31
|
+
const token = manualToken || getXSRFToken(cookieName, cookies);
|
|
32
|
+
if (token) {
|
|
33
|
+
if (!req.headers.has(headerName)) {
|
|
34
|
+
req.headers.set(headerName, token);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return next(req);
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
export function createXSRFMiddleware(config) {
|
|
41
|
+
if (!config) {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
if (config === true) {
|
|
45
|
+
return xsrf();
|
|
46
|
+
}
|
|
47
|
+
return xsrf(config);
|
|
48
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { EventEmitter } from 'events';
|
|
2
|
+
export interface RunnerOptions {
|
|
3
|
+
concurrency?: number;
|
|
4
|
+
retries?: number;
|
|
5
|
+
retryDelay?: number;
|
|
6
|
+
}
|
|
7
|
+
export interface RequestTask<T = any> {
|
|
8
|
+
id: string;
|
|
9
|
+
fn: () => Promise<T>;
|
|
10
|
+
priority: number;
|
|
11
|
+
retries?: number;
|
|
12
|
+
}
|
|
13
|
+
export interface RunnerResult<T = any> {
|
|
14
|
+
results: (T | Error)[];
|
|
15
|
+
stats: {
|
|
16
|
+
total: number;
|
|
17
|
+
successful: number;
|
|
18
|
+
failed: number;
|
|
19
|
+
duration: number;
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
export declare class RequestRunner extends EventEmitter {
|
|
23
|
+
private concurrency;
|
|
24
|
+
private queue;
|
|
25
|
+
private activeCount;
|
|
26
|
+
private paused;
|
|
27
|
+
private results;
|
|
28
|
+
private stats;
|
|
29
|
+
private startTime;
|
|
30
|
+
constructor(options?: RunnerOptions);
|
|
31
|
+
add<T>(fn: () => Promise<T>, options?: {
|
|
32
|
+
priority?: number;
|
|
33
|
+
id?: string;
|
|
34
|
+
}): void;
|
|
35
|
+
run<T>(items: any[], processor: (item: any, index: number) => Promise<T>, options?: {
|
|
36
|
+
priority?: number;
|
|
37
|
+
}): Promise<RunnerResult<T>>;
|
|
38
|
+
private processNext;
|
|
39
|
+
getProgress(): {
|
|
40
|
+
total: number;
|
|
41
|
+
completed: number;
|
|
42
|
+
pending: number;
|
|
43
|
+
active: number;
|
|
44
|
+
percent: number;
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=request-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"request-runner.d.ts","sourceRoot":"","sources":["../../src/runner/request-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAItC,MAAM,WAAW,aAAa;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW,CAAC,CAAC,GAAG,GAAG;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,GAAG;IACnC,OAAO,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC;IACvB,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED,qBAAa,aAAc,SAAQ,YAAY;IAC7C,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,KAAK,CAA0C;IACvD,OAAO,CAAC,SAAS,CAAa;gBAElB,OAAO,GAAE,aAAkB;IAKhC,GAAG,CAAC,CAAC,EACV,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,OAAO,GAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAO,GAC/C,IAAI;IAWM,GAAG,CAAC,CAAC,EAChB,KAAK,EAAE,GAAG,EAAE,EACZ,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACnD,OAAO,GAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAO,GAClC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YA+Bb,WAAW;IA8BlB,WAAW;;;;;;;CAUnB"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { EventEmitter } from 'events';
|
|
2
|
+
export class RequestRunner extends EventEmitter {
|
|
3
|
+
concurrency;
|
|
4
|
+
queue = [];
|
|
5
|
+
activeCount = 0;
|
|
6
|
+
paused = false;
|
|
7
|
+
results = new Map();
|
|
8
|
+
stats = { total: 0, successful: 0, failed: 0 };
|
|
9
|
+
startTime = 0;
|
|
10
|
+
constructor(options = {}) {
|
|
11
|
+
super();
|
|
12
|
+
this.concurrency = options.concurrency || 5;
|
|
13
|
+
}
|
|
14
|
+
add(fn, options = {}) {
|
|
15
|
+
this.queue.push({
|
|
16
|
+
id: options.id || Math.random().toString(36).slice(2),
|
|
17
|
+
fn,
|
|
18
|
+
priority: options.priority || 0,
|
|
19
|
+
});
|
|
20
|
+
this.queue.sort((a, b) => b.priority - a.priority);
|
|
21
|
+
this.stats.total++;
|
|
22
|
+
this.processNext();
|
|
23
|
+
}
|
|
24
|
+
async run(items, processor, options = {}) {
|
|
25
|
+
this.startTime = Date.now();
|
|
26
|
+
this.stats = { total: items.length, successful: 0, failed: 0 };
|
|
27
|
+
this.results.clear();
|
|
28
|
+
const promises = items.map((item, index) => {
|
|
29
|
+
return new Promise((resolve) => {
|
|
30
|
+
this.add(async () => {
|
|
31
|
+
try {
|
|
32
|
+
const res = await processor(item, index);
|
|
33
|
+
resolve(res);
|
|
34
|
+
return res;
|
|
35
|
+
}
|
|
36
|
+
catch (err) {
|
|
37
|
+
resolve(err);
|
|
38
|
+
throw err;
|
|
39
|
+
}
|
|
40
|
+
}, { priority: options.priority, id: String(index) });
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
const results = await Promise.all(promises);
|
|
44
|
+
return {
|
|
45
|
+
results,
|
|
46
|
+
stats: {
|
|
47
|
+
...this.stats,
|
|
48
|
+
duration: Date.now() - this.startTime
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
async processNext() {
|
|
53
|
+
if (this.paused || this.activeCount >= this.concurrency || this.queue.length === 0) {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
const task = this.queue.shift();
|
|
57
|
+
if (!task)
|
|
58
|
+
return;
|
|
59
|
+
this.activeCount++;
|
|
60
|
+
this.emit('taskStart', task);
|
|
61
|
+
try {
|
|
62
|
+
const result = await task.fn();
|
|
63
|
+
this.stats.successful++;
|
|
64
|
+
this.emit('taskComplete', { task, result });
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
this.stats.failed++;
|
|
68
|
+
this.emit('taskError', { task, error });
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
this.activeCount--;
|
|
72
|
+
this.emit('progress', this.getProgress());
|
|
73
|
+
if (this.activeCount === 0 && this.queue.length === 0) {
|
|
74
|
+
this.emit('drained');
|
|
75
|
+
}
|
|
76
|
+
this.processNext();
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
getProgress() {
|
|
80
|
+
const completed = this.stats.successful + this.stats.failed;
|
|
81
|
+
return {
|
|
82
|
+
total: this.stats.total,
|
|
83
|
+
completed,
|
|
84
|
+
pending: this.queue.length,
|
|
85
|
+
active: this.activeCount,
|
|
86
|
+
percent: this.stats.total > 0 ? (completed / this.stats.total) * 100 : 0
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/transport/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,SAAS,EAAW,MAAM,mBAAmB,CAAC;AAEtF,qBAAa,cAAe,YAAW,SAAS;;IAGxC,QAAQ,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC;CA6G5D"}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
export class FetchTransport {
|
|
2
|
+
constructor() { }
|
|
3
|
+
async dispatch(req) {
|
|
4
|
+
const start = performance.now();
|
|
5
|
+
const requestInit = {
|
|
6
|
+
method: req.method,
|
|
7
|
+
headers: req.headers,
|
|
8
|
+
body: req.body,
|
|
9
|
+
signal: req.signal,
|
|
10
|
+
duplex: req.body ? 'half' : undefined
|
|
11
|
+
};
|
|
12
|
+
try {
|
|
13
|
+
const response = await globalThis.fetch(req.url, requestInit);
|
|
14
|
+
const totalTime = performance.now() - start;
|
|
15
|
+
const timings = {
|
|
16
|
+
total: totalTime,
|
|
17
|
+
firstByte: totalTime,
|
|
18
|
+
};
|
|
19
|
+
const reckerResponse = {
|
|
20
|
+
status: response.status,
|
|
21
|
+
statusText: response.statusText,
|
|
22
|
+
headers: response.headers,
|
|
23
|
+
ok: response.ok,
|
|
24
|
+
url: response.url,
|
|
25
|
+
timings,
|
|
26
|
+
raw: response,
|
|
27
|
+
json: () => response.json(),
|
|
28
|
+
text: () => response.text(),
|
|
29
|
+
blob: () => response.blob(),
|
|
30
|
+
cleanText: async () => {
|
|
31
|
+
const text = await response.text();
|
|
32
|
+
return text.replace(/<[^>]*>?/gm, '');
|
|
33
|
+
},
|
|
34
|
+
read: () => response.body,
|
|
35
|
+
clone: () => {
|
|
36
|
+
const cloned = response.clone();
|
|
37
|
+
return createReckerResponseWrapper(cloned, timings);
|
|
38
|
+
},
|
|
39
|
+
async *sse() {
|
|
40
|
+
if (!response.body)
|
|
41
|
+
return;
|
|
42
|
+
const reader = response.body.pipeThrough(new TextDecoderStream()).getReader();
|
|
43
|
+
while (true) {
|
|
44
|
+
const { done, value } = await reader.read();
|
|
45
|
+
if (done)
|
|
46
|
+
break;
|
|
47
|
+
const lines = value.split('\n');
|
|
48
|
+
for (const line of lines) {
|
|
49
|
+
if (line.startsWith('data: ')) {
|
|
50
|
+
yield { data: line.slice(6) };
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
async *download() {
|
|
56
|
+
if (!response.body)
|
|
57
|
+
return;
|
|
58
|
+
const reader = response.body.getReader();
|
|
59
|
+
let loaded = 0;
|
|
60
|
+
const total = Number(response.headers.get('content-length')) || 0;
|
|
61
|
+
while (true) {
|
|
62
|
+
const { done, value } = await reader.read();
|
|
63
|
+
if (done)
|
|
64
|
+
break;
|
|
65
|
+
loaded += value.length;
|
|
66
|
+
yield {
|
|
67
|
+
loaded,
|
|
68
|
+
total,
|
|
69
|
+
percent: total ? (loaded / total) * 100 : undefined
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
},
|
|
73
|
+
async *[Symbol.asyncIterator]() {
|
|
74
|
+
if (!response.body)
|
|
75
|
+
return;
|
|
76
|
+
const reader = response.body.getReader();
|
|
77
|
+
while (true) {
|
|
78
|
+
const { done, value } = await reader.read();
|
|
79
|
+
if (done)
|
|
80
|
+
break;
|
|
81
|
+
yield value;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
return reckerResponse;
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
throw error;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function createReckerResponseWrapper(response, timings) {
|
|
93
|
+
return new FetchResponseWrapper(response, timings);
|
|
94
|
+
}
|
|
95
|
+
class FetchResponseWrapper {
|
|
96
|
+
raw;
|
|
97
|
+
timings;
|
|
98
|
+
constructor(raw, timings) {
|
|
99
|
+
this.raw = raw;
|
|
100
|
+
this.timings = timings;
|
|
101
|
+
}
|
|
102
|
+
get status() { return this.raw.status; }
|
|
103
|
+
get statusText() { return this.raw.statusText; }
|
|
104
|
+
get headers() { return this.raw.headers; }
|
|
105
|
+
get ok() { return this.raw.ok; }
|
|
106
|
+
get url() { return this.raw.url; }
|
|
107
|
+
get connection() { return {}; }
|
|
108
|
+
json() { return this.raw.json(); }
|
|
109
|
+
text() { return this.raw.text(); }
|
|
110
|
+
blob() { return this.raw.blob(); }
|
|
111
|
+
async cleanText() { return (await this.text()).replace(/<[^>]*>?/gm, ''); }
|
|
112
|
+
read() { return this.raw.body; }
|
|
113
|
+
clone() { return new FetchResponseWrapper(this.raw.clone(), this.timings); }
|
|
114
|
+
async *sse() {
|
|
115
|
+
if (!this.raw.body)
|
|
116
|
+
return;
|
|
117
|
+
const stream = this.raw.body.pipeThrough(new TextDecoderStream());
|
|
118
|
+
const reader = stream.getReader();
|
|
119
|
+
while (true) {
|
|
120
|
+
const { done, value } = await reader.read();
|
|
121
|
+
if (done)
|
|
122
|
+
break;
|
|
123
|
+
if (value.startsWith('data: ')) {
|
|
124
|
+
yield { data: value.slice(6) };
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
async *download() {
|
|
129
|
+
if (!this.raw.body)
|
|
130
|
+
return;
|
|
131
|
+
const reader = this.raw.body.getReader();
|
|
132
|
+
let loaded = 0;
|
|
133
|
+
const total = Number(this.raw.headers.get('content-length')) || 0;
|
|
134
|
+
while (true) {
|
|
135
|
+
const { done, value } = await reader.read();
|
|
136
|
+
if (done)
|
|
137
|
+
break;
|
|
138
|
+
loaded += value.length;
|
|
139
|
+
yield { loaded, total, percent: total ? (loaded / total) * 100 : 0 };
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
async *[Symbol.asyncIterator]() {
|
|
143
|
+
if (!this.raw.body)
|
|
144
|
+
return;
|
|
145
|
+
const reader = this.raw.body.getReader();
|
|
146
|
+
while (true) {
|
|
147
|
+
const { done, value } = await reader.read();
|
|
148
|
+
if (done)
|
|
149
|
+
break;
|
|
150
|
+
yield value;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { ReckerRequest, ReckerResponse, Transport, ProxyOptions, HTTP2Options, DNSOptions } from '../types/index.js';
|
|
2
|
+
import { AgentManager } from '../utils/agent-manager.js';
|
|
3
|
+
interface UndiciTransportOptions {
|
|
4
|
+
connectTimeout?: number;
|
|
5
|
+
headersTimeout?: number;
|
|
6
|
+
bodyTimeout?: number;
|
|
7
|
+
maxRedirections?: number;
|
|
8
|
+
proxy?: ProxyOptions | string;
|
|
9
|
+
http2?: HTTP2Options;
|
|
10
|
+
dns?: DNSOptions;
|
|
11
|
+
agent?: AgentManager;
|
|
12
|
+
}
|
|
13
|
+
export declare class UndiciTransport implements Transport {
|
|
14
|
+
private baseUrl;
|
|
15
|
+
private options;
|
|
16
|
+
private proxyAgent?;
|
|
17
|
+
private dnsAgent?;
|
|
18
|
+
private agentManager?;
|
|
19
|
+
constructor(baseUrl: string, options?: UndiciTransportOptions);
|
|
20
|
+
dispatch(req: ReckerRequest): Promise<ReckerResponse>;
|
|
21
|
+
}
|
|
22
|
+
export {};
|
|
23
|
+
//# sourceMappingURL=undici.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"undici.d.ts","sourceRoot":"","sources":["../../src/transport/undici.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,aAAa,EAAE,cAAc,EAAW,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAgB,MAAM,mBAAmB,CAAC;AAO5J,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAsNzD,UAAU,sBAAsB;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,YAAY,GAAG,MAAM,CAAC;IAC9B,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,YAAY,CAAC;CACtB;AAED,qBAAa,eAAgB,YAAW,SAAS;IAC/C,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAyB;IACxC,OAAO,CAAC,UAAU,CAAC,CAAa;IAChC,OAAO,CAAC,QAAQ,CAAC,CAAQ;IACzB,OAAO,CAAC,YAAY,CAAC,CAAe;gBAExB,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,sBAA2B;IAiC3D,QAAQ,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC;CAmF5D"}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import { request as undiciRequest, errors as undiciErrors, ProxyAgent, Agent } from 'undici';
|
|
2
|
+
import { HttpResponse } from '../core/response.js';
|
|
3
|
+
import { NetworkError, TimeoutError } from '../core/errors.js';
|
|
4
|
+
import { performance } from 'perf_hooks';
|
|
5
|
+
import { AsyncLocalStorage } from 'async_hooks';
|
|
6
|
+
import { channel } from 'node:diagnostics_channel';
|
|
7
|
+
import { createLookupFunction } from '../utils/dns.js';
|
|
8
|
+
const undiciRequestChannel = channel('undici:request:create');
|
|
9
|
+
const undiciBodySentChannel = channel('undici:request:bodySent');
|
|
10
|
+
const undiciHeadersChannel = channel('undici:request:headers');
|
|
11
|
+
const undiciConnectChannel = channel('undici:client:connect');
|
|
12
|
+
const requestStorage = new AsyncLocalStorage();
|
|
13
|
+
undiciRequestChannel.subscribe((message) => {
|
|
14
|
+
const store = requestStorage.getStore();
|
|
15
|
+
if (store) {
|
|
16
|
+
store.requestStartTime = performance.now();
|
|
17
|
+
store.timings = { queuing: 0, dns: 0, tcp: 0, tls: 0, firstByte: 0, content: 0, total: 0 };
|
|
18
|
+
store.connection = {};
|
|
19
|
+
}
|
|
20
|
+
});
|
|
21
|
+
undiciBodySentChannel.subscribe((message) => {
|
|
22
|
+
const store = requestStorage.getStore();
|
|
23
|
+
if (store && store.hooks && store.hooks.onRequestSent) {
|
|
24
|
+
store.hooks.onRequestSent();
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
undiciHeadersChannel.subscribe((message) => {
|
|
28
|
+
const payload = message;
|
|
29
|
+
const store = requestStorage.getStore();
|
|
30
|
+
if (store && payload && payload.timing) {
|
|
31
|
+
const { timing } = payload;
|
|
32
|
+
store.timings.queuing = timing.queuing;
|
|
33
|
+
store.timings.dns = timing.dns;
|
|
34
|
+
store.timings.tcp = timing.tcp;
|
|
35
|
+
store.timings.tls = timing.tls;
|
|
36
|
+
store.timings.firstByte = timing.response;
|
|
37
|
+
store.timings.content = timing.body;
|
|
38
|
+
store.timings.total = timing.ended;
|
|
39
|
+
if (store.hooks) {
|
|
40
|
+
if (store.hooks.onDnsLookup && timing.dns > 0) {
|
|
41
|
+
store.hooks.onDnsLookup({ domain: payload.request.origin, duration: timing.dns });
|
|
42
|
+
}
|
|
43
|
+
if (store.hooks.onTcpConnect && timing.tcp > 0) {
|
|
44
|
+
store.hooks.onTcpConnect({ remoteAddress: '', duration: timing.tcp });
|
|
45
|
+
}
|
|
46
|
+
if (store.hooks.onTlsHandshake && timing.tls > 0) {
|
|
47
|
+
store.hooks.onTlsHandshake({ protocol: '', cipher: '', duration: timing.tls });
|
|
48
|
+
}
|
|
49
|
+
if (store.hooks.onResponseStart) {
|
|
50
|
+
const headers = new Headers();
|
|
51
|
+
for (let i = 0; i < payload.response.headers.length; i += 2) {
|
|
52
|
+
headers.append(payload.response.headers[i], payload.response.headers[i + 1]);
|
|
53
|
+
}
|
|
54
|
+
store.hooks.onResponseStart({ status: payload.response.statusCode, headers });
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
undiciConnectChannel.subscribe((message) => {
|
|
60
|
+
const payload = message;
|
|
61
|
+
const store = requestStorage.getStore();
|
|
62
|
+
if (store && payload && payload.socket) {
|
|
63
|
+
const { socket } = payload;
|
|
64
|
+
store.connection.remoteAddress = socket.remoteAddress;
|
|
65
|
+
store.connection.remotePort = socket.remotePort;
|
|
66
|
+
store.connection.localAddress = socket.localAddress;
|
|
67
|
+
store.connection.localPort = socket.localPort;
|
|
68
|
+
const protocol = socket.alpnProtocol || (socket.tlsSocket ? socket.tlsSocket.getProtocol() : undefined);
|
|
69
|
+
store.connection.protocol = protocol;
|
|
70
|
+
store.connection.cipher = socket.tlsSocket ? socket.tlsSocket.getCipher()?.name : undefined;
|
|
71
|
+
if (store.hooks) {
|
|
72
|
+
if (store.hooks.onTcpConnect) {
|
|
73
|
+
store.hooks.onTcpConnect({ remoteAddress: socket.remoteAddress, duration: 0 });
|
|
74
|
+
}
|
|
75
|
+
if (store.hooks.onTlsHandshake && socket.tlsSocket) {
|
|
76
|
+
store.hooks.onTlsHandshake({
|
|
77
|
+
protocol: protocol || 'unknown',
|
|
78
|
+
cipher: socket.tlsSocket.getCipher()?.name || 'unknown',
|
|
79
|
+
duration: 0
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
if (protocol === 'h2') {
|
|
84
|
+
const http2Session = socket.session || socket;
|
|
85
|
+
const http2State = http2Session.state;
|
|
86
|
+
const remoteSettings = http2Session.remoteSettings;
|
|
87
|
+
const localSettings = http2Session.localSettings;
|
|
88
|
+
store.connection.http2 = {
|
|
89
|
+
streamId: socket.streamId,
|
|
90
|
+
streamWeight: socket.weight,
|
|
91
|
+
streamDependency: socket.dependency,
|
|
92
|
+
serverPush: Boolean(socket.serverPush),
|
|
93
|
+
settingsReceived: Boolean(remoteSettings),
|
|
94
|
+
maxConcurrentStreams: remoteSettings?.maxConcurrentStreams,
|
|
95
|
+
currentStreams: http2State?.streamCount,
|
|
96
|
+
pendingStreams: http2State?.pendingStreamCount,
|
|
97
|
+
localWindowSize: http2State?.localWindowSize ?? http2State?.effectiveLocalWindowSize,
|
|
98
|
+
remoteWindowSize: http2State?.effectiveRecvDataLength,
|
|
99
|
+
localSettings: localSettings ? { ...localSettings } : undefined,
|
|
100
|
+
remoteSettings: remoteSettings ? { ...remoteSettings } : undefined
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
if (protocol === 'h3' || protocol?.startsWith('h3-')) {
|
|
104
|
+
const quicStats = socket.stats;
|
|
105
|
+
const handshakeConfirmed = socket.handshakeConfirmed;
|
|
106
|
+
store.connection.http3 = {
|
|
107
|
+
quicVersion: protocol,
|
|
108
|
+
zeroRTT: socket.zeroRTT || false,
|
|
109
|
+
maxStreams: socket.maxStreams,
|
|
110
|
+
handshakeConfirmed: typeof handshakeConfirmed === 'boolean' ? handshakeConfirmed : undefined
|
|
111
|
+
};
|
|
112
|
+
if (quicStats && typeof quicStats.rtt === 'number') {
|
|
113
|
+
store.connection.rtt = quicStats.rtt;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
store.connection.reused = Boolean(socket.reused);
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
export class UndiciTransport {
|
|
120
|
+
baseUrl;
|
|
121
|
+
options;
|
|
122
|
+
proxyAgent;
|
|
123
|
+
dnsAgent;
|
|
124
|
+
agentManager;
|
|
125
|
+
constructor(baseUrl, options = {}) {
|
|
126
|
+
this.baseUrl = baseUrl;
|
|
127
|
+
this.options = options;
|
|
128
|
+
if (options.proxy) {
|
|
129
|
+
const proxyUrl = typeof options.proxy === 'string' ? options.proxy : options.proxy.url;
|
|
130
|
+
const proxyAuth = typeof options.proxy === 'object' && options.proxy.auth
|
|
131
|
+
? `${options.proxy.auth.username}:${options.proxy.auth.password}`
|
|
132
|
+
: undefined;
|
|
133
|
+
const finalProxyUrl = proxyAuth
|
|
134
|
+
? proxyUrl.replace('://', `://${proxyAuth}@`)
|
|
135
|
+
: proxyUrl;
|
|
136
|
+
this.proxyAgent = new ProxyAgent(finalProxyUrl);
|
|
137
|
+
}
|
|
138
|
+
this.agentManager = options.agent;
|
|
139
|
+
if (options.dns && !this.agentManager) {
|
|
140
|
+
const lookupFn = createLookupFunction(options.dns);
|
|
141
|
+
this.dnsAgent = new Agent({
|
|
142
|
+
connect: {
|
|
143
|
+
lookup: lookupFn,
|
|
144
|
+
},
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
async dispatch(req) {
|
|
149
|
+
const headers = {};
|
|
150
|
+
req.headers.forEach((value, key) => {
|
|
151
|
+
headers[key] = value;
|
|
152
|
+
});
|
|
153
|
+
const path = req.url.startsWith(this.baseUrl) ? req.url.substring(this.baseUrl.length) : req.url;
|
|
154
|
+
const fullUrl = new URL(path, this.baseUrl).toString();
|
|
155
|
+
const requestContext = {
|
|
156
|
+
timings: {},
|
|
157
|
+
connection: {},
|
|
158
|
+
requestStartTime: 0,
|
|
159
|
+
requestCorrelationId: Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15),
|
|
160
|
+
hooks: req._hooks
|
|
161
|
+
};
|
|
162
|
+
return requestStorage.run(requestContext, async () => {
|
|
163
|
+
try {
|
|
164
|
+
const startTime = performance.now();
|
|
165
|
+
if (requestContext.requestStartTime === 0) {
|
|
166
|
+
requestContext.requestStartTime = startTime;
|
|
167
|
+
}
|
|
168
|
+
const dispatcher = req._dispatcher || this.proxyAgent || this.dnsAgent;
|
|
169
|
+
const undiciOptions = {
|
|
170
|
+
method: req.method,
|
|
171
|
+
headers: headers,
|
|
172
|
+
body: req.body,
|
|
173
|
+
signal: req.signal,
|
|
174
|
+
dispatcher: dispatcher,
|
|
175
|
+
connectTimeout: this.options.connectTimeout,
|
|
176
|
+
headersTimeout: this.options.headersTimeout,
|
|
177
|
+
bodyTimeout: this.options.bodyTimeout,
|
|
178
|
+
maxRedirections: this.options.maxRedirections,
|
|
179
|
+
};
|
|
180
|
+
if (this.options.http2) {
|
|
181
|
+
if (this.options.http2.enabled) {
|
|
182
|
+
undiciOptions.allowH2 = true;
|
|
183
|
+
}
|
|
184
|
+
if (this.options.http2.maxConcurrentStreams !== undefined) {
|
|
185
|
+
undiciOptions.maxConcurrentStreams = this.options.http2.maxConcurrentStreams;
|
|
186
|
+
}
|
|
187
|
+
if (this.options.http2.pipelining !== undefined) {
|
|
188
|
+
undiciOptions.pipelining = this.options.http2.pipelining;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
const undiciResponse = await undiciRequest(fullUrl, undiciOptions);
|
|
192
|
+
const ttfb = performance.now() - startTime;
|
|
193
|
+
const totalTime = performance.now() - requestContext.requestStartTime;
|
|
194
|
+
if (!requestContext.timings.firstByte) {
|
|
195
|
+
requestContext.timings.firstByte = ttfb;
|
|
196
|
+
}
|
|
197
|
+
if (!requestContext.timings.total) {
|
|
198
|
+
requestContext.timings.total = totalTime;
|
|
199
|
+
}
|
|
200
|
+
return new HttpResponse(undiciResponse, {
|
|
201
|
+
timings: requestContext.timings,
|
|
202
|
+
connection: requestContext.connection
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
catch (error) {
|
|
206
|
+
if (error instanceof undiciErrors.ConnectTimeoutError ||
|
|
207
|
+
error instanceof undiciErrors.HeadersTimeoutError ||
|
|
208
|
+
error instanceof undiciErrors.BodyTimeoutError) {
|
|
209
|
+
throw new TimeoutError(req);
|
|
210
|
+
}
|
|
211
|
+
if (error.code === 'UND_ERR_CONNECT_TIMEOUT' || error.code === 'UND_ERR_HEADERS_TIMEOUT') {
|
|
212
|
+
throw new TimeoutError(req);
|
|
213
|
+
}
|
|
214
|
+
throw new NetworkError(error.message, error.code, req);
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
}
|