recker 1.0.72 → 1.0.75-next.2e5a94f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.md +5 -18
  2. package/dist/browser/core/client.d.ts +14 -8
  3. package/dist/browser/core/client.js +199 -17
  4. package/dist/browser/core/errors.d.ts +15 -1
  5. package/dist/browser/core/errors.js +140 -9
  6. package/dist/browser/core/request.d.ts +5 -0
  7. package/dist/browser/core/request.js +33 -2
  8. package/dist/browser/core-runtime/plugin-manifest.d.ts +24 -0
  9. package/dist/browser/core-runtime/plugin-manifest.js +159 -0
  10. package/dist/browser/core-runtime/request-context.d.ts +13 -0
  11. package/dist/browser/core-runtime/request-context.js +24 -0
  12. package/dist/browser/core-runtime/typed-events.d.ts +89 -0
  13. package/dist/browser/core-runtime/typed-events.js +34 -0
  14. package/dist/browser/index.iife.min.js +79 -79
  15. package/dist/browser/index.min.js +79 -79
  16. package/dist/browser/index.mini.iife.js +913 -97
  17. package/dist/browser/index.mini.iife.min.js +46 -46
  18. package/dist/browser/index.mini.min.js +46 -46
  19. package/dist/browser/index.mini.umd.js +913 -97
  20. package/dist/browser/index.mini.umd.min.js +46 -46
  21. package/dist/browser/index.umd.min.js +79 -79
  22. package/dist/browser/plugins/auth/aws-sigv4.d.ts +1 -0
  23. package/dist/browser/plugins/auth/aws-sigv4.js +19 -2
  24. package/dist/browser/plugins/retry.js +29 -1
  25. package/dist/browser/presets/aws.d.ts +1 -0
  26. package/dist/browser/presets/aws.js +62 -1
  27. package/dist/browser/runner/request-runner.d.ts +15 -5
  28. package/dist/browser/runner/request-runner.js +164 -30
  29. package/dist/browser/scrape/parser/nodes/html.d.ts +6 -0
  30. package/dist/browser/scrape/parser/nodes/html.js +70 -18
  31. package/dist/browser/scrape/parser/nodes/node.d.ts +1 -0
  32. package/dist/browser/scrape/parser/nodes/node.js +5 -0
  33. package/dist/browser/scrape/spider.d.ts +1 -0
  34. package/dist/browser/scrape/spider.js +39 -26
  35. package/dist/browser/seo/analyzer.d.ts +1 -1
  36. package/dist/browser/seo/analyzer.js +73 -42
  37. package/dist/browser/seo/index.d.ts +1 -1
  38. package/dist/browser/seo/rules/types.d.ts +2 -0
  39. package/dist/browser/seo/seo-spider.d.ts +2 -3
  40. package/dist/browser/seo/seo-spider.js +26 -202
  41. package/dist/browser/seo/types.d.ts +4 -0
  42. package/dist/browser/seo/validators/sitemap.js +9 -2
  43. package/dist/browser/transport/fetch.js +38 -5
  44. package/dist/browser/transport/undici.js +73 -11
  45. package/dist/browser/transport/worker.d.ts +0 -1
  46. package/dist/browser/transport/worker.js +1 -3
  47. package/dist/browser/types/index.d.ts +24 -0
  48. package/dist/cli/commands/mcp.js +5 -3
  49. package/dist/core/client.d.ts +14 -8
  50. package/dist/core/client.js +199 -17
  51. package/dist/core/errors.d.ts +15 -1
  52. package/dist/core/errors.js +140 -9
  53. package/dist/core/request.d.ts +5 -0
  54. package/dist/core/request.js +33 -2
  55. package/dist/core-runtime/plugin-manifest.d.ts +24 -0
  56. package/dist/core-runtime/plugin-manifest.js +159 -0
  57. package/dist/core-runtime/request-context.d.ts +13 -0
  58. package/dist/core-runtime/request-context.js +24 -0
  59. package/dist/core-runtime/typed-events.d.ts +89 -0
  60. package/dist/core-runtime/typed-events.js +34 -0
  61. package/dist/index.d.ts +2 -1
  62. package/dist/index.js +2 -1
  63. package/dist/mcp/cli.js +10 -8
  64. package/dist/mcp/profiles.d.ts +1 -1
  65. package/dist/mcp/profiles.js +31 -6
  66. package/dist/mcp/tools/categories.js +0 -1
  67. package/dist/mcp/tools/seo.js +320 -4
  68. package/dist/plugins/auth/aws-sigv4.d.ts +1 -0
  69. package/dist/plugins/auth/aws-sigv4.js +19 -2
  70. package/dist/plugins/retry.js +29 -1
  71. package/dist/presets/aws.d.ts +1 -0
  72. package/dist/presets/aws.js +62 -1
  73. package/dist/recker.d.ts +3 -0
  74. package/dist/recker.js +5 -0
  75. package/dist/runner/request-runner.d.ts +15 -5
  76. package/dist/runner/request-runner.js +164 -30
  77. package/dist/scrape/parser/nodes/html.d.ts +6 -0
  78. package/dist/scrape/parser/nodes/html.js +70 -18
  79. package/dist/scrape/parser/nodes/node.d.ts +1 -0
  80. package/dist/scrape/parser/nodes/node.js +5 -0
  81. package/dist/scrape/spider.d.ts +1 -0
  82. package/dist/scrape/spider.js +39 -26
  83. package/dist/search/google.d.ts +67 -0
  84. package/dist/search/google.js +480 -0
  85. package/dist/search/index.d.ts +3 -0
  86. package/dist/search/index.js +1 -0
  87. package/dist/seo/analyzer.d.ts +1 -1
  88. package/dist/seo/analyzer.js +73 -42
  89. package/dist/seo/index.d.ts +1 -1
  90. package/dist/seo/rules/types.d.ts +2 -0
  91. package/dist/seo/seo-spider.d.ts +2 -3
  92. package/dist/seo/seo-spider.js +26 -202
  93. package/dist/seo/types.d.ts +4 -0
  94. package/dist/seo/validators/sitemap.js +9 -2
  95. package/dist/transport/fetch.js +38 -5
  96. package/dist/transport/undici.js +73 -11
  97. package/dist/transport/worker.d.ts +0 -1
  98. package/dist/transport/worker.js +1 -3
  99. package/dist/types/index.d.ts +24 -0
  100. package/dist/version.js +1 -1
  101. package/package.json +9 -1
@@ -1,4 +1,5 @@
1
1
  import { createProgressStream } from '../utils/progress.js';
2
+ import { TimeoutError } from '../core/errors.js';
2
3
  function parseContentLength(headers) {
3
4
  const raw = headers.get('content-length');
4
5
  if (!raw)
@@ -6,6 +7,19 @@ function parseContentLength(headers) {
6
7
  const parsed = parseInt(raw, 10);
7
8
  return Number.isFinite(parsed) ? parsed : undefined;
8
9
  }
10
+ function getAbortReason(error) {
11
+ return error?.cause ?? error?.reason;
12
+ }
13
+ function isTimeoutReason(reason) {
14
+ if (reason instanceof TimeoutError) {
15
+ return true;
16
+ }
17
+ if (!reason || typeof reason !== 'object') {
18
+ return false;
19
+ }
20
+ const timeoutReason = reason;
21
+ return timeoutReason.name === 'TimeoutError';
22
+ }
9
23
  function wrapDownloadResponse(response, onProgress) {
10
24
  if (!onProgress || !response.body)
11
25
  return response;
@@ -66,10 +80,16 @@ export class FetchTransport {
66
80
  ? req.timeout
67
81
  : req.timeout?.request;
68
82
  let signal = req.signal;
83
+ const requestTimeoutError = timeoutMs ? new TimeoutError(req, {
84
+ phase: 'request',
85
+ timeout: timeoutMs
86
+ }) : undefined;
69
87
  if (timeoutMs && !signal) {
70
88
  abortController = new AbortController();
71
89
  signal = abortController.signal;
72
- timeoutId = setTimeout(() => abortController.abort(), timeoutMs);
90
+ timeoutId = setTimeout(() => {
91
+ timeoutControllerAbort(abortController, requestTimeoutError);
92
+ }, timeoutMs);
73
93
  }
74
94
  const followRedirects = req.followRedirects !== false;
75
95
  const maxRedirects = req.maxRedirects ?? 20;
@@ -148,10 +168,13 @@ export class FetchTransport {
148
168
  }
149
169
  }
150
170
  catch (error) {
151
- if (error.name === 'AbortError' && abortController) {
152
- const timeoutError = new Error(`Request timeout after ${timeoutMs}ms`);
153
- timeoutError.name = 'TimeoutError';
154
- throw timeoutError;
171
+ if (timeoutMs && (error.name === 'AbortError' || error.name === 'TimeoutError')) {
172
+ const timeoutReason = getAbortReason(error);
173
+ if (isTimeoutReason(timeoutReason) ||
174
+ isTimeoutReason(getAbortReason(signal)) ||
175
+ (abortController && isTimeoutReason(requestTimeoutError))) {
176
+ throw timeoutReason instanceof TimeoutError ? timeoutReason : requestTimeoutError ?? error;
177
+ }
155
178
  }
156
179
  throw error;
157
180
  }
@@ -161,6 +184,16 @@ export class FetchTransport {
161
184
  }
162
185
  }
163
186
  }
187
+ function timeoutControllerAbort(controller, reason) {
188
+ if (!controller.signal.aborted) {
189
+ if (reason) {
190
+ controller.abort(reason);
191
+ }
192
+ else {
193
+ controller.abort();
194
+ }
195
+ }
196
+ }
164
197
  class FetchResponseWrapper {
165
198
  raw;
166
199
  timings;
@@ -15,6 +15,40 @@ const undiciBodySentChannel = channel('undici:request:bodySent');
15
15
  const undiciHeadersChannel = channel('undici:request:headers');
16
16
  const undiciConnectChannel = channel('undici:client:connect');
17
17
  const requestStorage = new AsyncLocalStorage();
18
+ function getAbortReason(error) {
19
+ return error?.cause ?? error?.reason;
20
+ }
21
+ function isTimeoutAbortReason(reason) {
22
+ if (reason instanceof TimeoutError) {
23
+ return true;
24
+ }
25
+ if (!reason || typeof reason !== 'object') {
26
+ return false;
27
+ }
28
+ const timeoutReason = reason;
29
+ return timeoutReason.name === 'TimeoutError';
30
+ }
31
+ function isTotalRequestTimeoutError(error, timeoutReason, timeoutMs) {
32
+ if (!timeoutMs) {
33
+ return false;
34
+ }
35
+ if (timeoutReason !== undefined) {
36
+ return true;
37
+ }
38
+ const causeOrReason = getAbortReason(error);
39
+ const message = error?.message;
40
+ const hasTimeoutMessage = typeof message === 'string' && (message.includes('Request timed out (total time exceeded)') ||
41
+ message.includes('timed out (total time exceeded)'));
42
+ const hasRequestAbortMessage = typeof message === 'string' && message.toLowerCase() === 'request was aborted';
43
+ return (error instanceof TimeoutError ||
44
+ isTimeoutAbortReason(causeOrReason) ||
45
+ isTimeoutAbortReason(error) ||
46
+ error instanceof undiciErrors.RequestAbortedError ||
47
+ hasTimeoutMessage ||
48
+ hasRequestAbortMessage ||
49
+ error?.code === 'ABORT_ERR' ||
50
+ error?.code === 'UND_ERR_REQUEST_TIMEOUT');
51
+ }
18
52
  undiciRequestChannel.subscribe((message) => {
19
53
  const store = requestStorage.getStore();
20
54
  if (store) {
@@ -36,7 +70,7 @@ undiciRequestChannel.subscribe((message) => {
36
70
  }
37
71
  }
38
72
  });
39
- undiciBodySentChannel.subscribe((message) => {
73
+ undiciBodySentChannel.subscribe((_message) => {
40
74
  const store = requestStorage.getStore();
41
75
  if (store?.hooks && store.hooks.onRequestSent) {
42
76
  store.hooks.onRequestSent();
@@ -317,6 +351,9 @@ export class UndiciTransport {
317
351
  });
318
352
  let timeoutController;
319
353
  let timeoutId;
354
+ let timeoutReason;
355
+ let timeoutError;
356
+ let cancelOriginalAbortListener;
320
357
  if (!this.observability) {
321
358
  return this.dispatchFast(req, headers, currentUrl, timeouts, handleRedirectsManually, maxRedirects, followRedirects, uploadTotal);
322
359
  }
@@ -340,24 +377,32 @@ export class UndiciTransport {
340
377
  let effectiveSignal = req.signal;
341
378
  if (timeouts.totalTimeout) {
342
379
  timeoutController = new AbortController();
380
+ timeoutError = new TimeoutError(req, {
381
+ phase: 'request',
382
+ timeout: timeouts.totalTimeout
383
+ });
343
384
  if (req.signal) {
344
385
  const originalSignal = req.signal;
345
386
  effectiveSignal = timeoutController.signal;
346
387
  const onOriginalAbort = () => {
347
- timeoutController.abort();
388
+ timeoutReason = originalSignal.reason ?? new Error('Request aborted by external signal');
389
+ timeoutController.abort(timeoutReason);
348
390
  };
349
391
  if (originalSignal.aborted) {
350
- timeoutController.abort();
392
+ timeoutReason = originalSignal.reason ?? new Error('Request aborted by external signal');
393
+ timeoutController.abort(timeoutReason);
351
394
  }
352
395
  else {
353
396
  originalSignal.addEventListener('abort', onOriginalAbort, { once: true });
397
+ cancelOriginalAbortListener = () => originalSignal.removeEventListener('abort', onOriginalAbort);
354
398
  }
355
399
  }
356
400
  else {
357
401
  effectiveSignal = timeoutController.signal;
358
402
  }
359
403
  timeoutId = setTimeout(() => {
360
- timeoutController.abort();
404
+ timeoutReason = timeoutError;
405
+ timeoutController.abort(timeoutReason);
361
406
  }, timeouts.totalTimeout);
362
407
  }
363
408
  while (true) {
@@ -593,8 +638,8 @@ export class UndiciTransport {
593
638
  timeout: timeouts.bodyTimeout
594
639
  });
595
640
  }
596
- if (error.name === 'AbortError' || error.code === 'ABORT_ERR') {
597
- throw new TimeoutError(req, {
641
+ if (isTotalRequestTimeoutError(error, timeoutReason, timeouts.totalTimeout)) {
642
+ throw timeoutError ?? new TimeoutError(req, {
598
643
  phase: 'request',
599
644
  timeout: timeouts.totalTimeout
600
645
  });
@@ -611,6 +656,9 @@ export class UndiciTransport {
611
656
  throw new NetworkError(error.message, code, req);
612
657
  }
613
658
  finally {
659
+ if (typeof cancelOriginalAbortListener === 'function') {
660
+ cancelOriginalAbortListener();
661
+ }
614
662
  if (timeoutId) {
615
663
  clearTimeout(timeoutId);
616
664
  }
@@ -620,6 +668,9 @@ export class UndiciTransport {
620
668
  async dispatchFast(req, headers, currentUrl, timeouts, handleRedirectsManually, maxRedirects, followRedirects, uploadTotal) {
621
669
  let timeoutController;
622
670
  let timeoutId;
671
+ let timeoutReason;
672
+ let timeoutError;
673
+ let cancelOriginalAbortListener;
623
674
  try {
624
675
  let redirectCount = 0;
625
676
  let currentMethod = req.method;
@@ -628,24 +679,32 @@ export class UndiciTransport {
628
679
  let effectiveSignal = req.signal;
629
680
  if (timeouts.totalTimeout) {
630
681
  timeoutController = new AbortController();
682
+ timeoutError = new TimeoutError(req, {
683
+ phase: 'request',
684
+ timeout: timeouts.totalTimeout
685
+ });
631
686
  if (req.signal) {
632
687
  const originalSignal = req.signal;
633
688
  effectiveSignal = timeoutController.signal;
634
689
  const onOriginalAbort = () => {
635
- timeoutController.abort();
690
+ timeoutReason = originalSignal.reason ?? new Error('Request aborted by external signal');
691
+ timeoutController.abort(timeoutReason);
636
692
  };
637
693
  if (originalSignal.aborted) {
638
- timeoutController.abort();
694
+ timeoutReason = originalSignal.reason ?? new Error('Request aborted by external signal');
695
+ timeoutController.abort(timeoutReason);
639
696
  }
640
697
  else {
641
698
  originalSignal.addEventListener('abort', onOriginalAbort, { once: true });
699
+ cancelOriginalAbortListener = () => originalSignal.removeEventListener('abort', onOriginalAbort);
642
700
  }
643
701
  }
644
702
  else {
645
703
  effectiveSignal = timeoutController.signal;
646
704
  }
647
705
  timeoutId = setTimeout(() => {
648
- timeoutController.abort();
706
+ timeoutReason = timeoutError;
707
+ timeoutController.abort(timeoutReason);
649
708
  }, timeouts.totalTimeout);
650
709
  }
651
710
  while (true) {
@@ -822,8 +881,8 @@ export class UndiciTransport {
822
881
  timeout: timeouts.bodyTimeout
823
882
  });
824
883
  }
825
- if (error.name === 'AbortError' || error.code === 'ABORT_ERR') {
826
- throw new TimeoutError(req, {
884
+ if (isTotalRequestTimeoutError(error, timeoutReason, timeouts.totalTimeout)) {
885
+ throw timeoutError ?? new TimeoutError(req, {
827
886
  phase: 'request',
828
887
  timeout: timeouts.totalTimeout
829
888
  });
@@ -840,6 +899,9 @@ export class UndiciTransport {
840
899
  throw new NetworkError(error.message, code, req);
841
900
  }
842
901
  finally {
902
+ if (typeof cancelOriginalAbortListener === 'function') {
903
+ cancelOriginalAbortListener();
904
+ }
843
905
  if (timeoutId) {
844
906
  clearTimeout(timeoutId);
845
907
  }
@@ -3,7 +3,6 @@ export interface WorkerTransportOptions {
3
3
  poolSize?: number;
4
4
  }
5
5
  export declare class WorkerTransport implements Transport {
6
- private options;
7
6
  private workers;
8
7
  private workerIndex;
9
8
  private pendingRequests;
@@ -58,13 +58,11 @@ self.onmessage = async (event) => {
58
58
  };
59
59
  `;
60
60
  export class WorkerTransport {
61
- options;
62
61
  workers = [];
63
62
  workerIndex = 0;
64
63
  pendingRequests = new Map();
65
64
  workerUrl;
66
65
  constructor(options = {}) {
67
- this.options = options;
68
66
  const poolSize = options.poolSize ?? (typeof navigator !== 'undefined' ? navigator.hardwareConcurrency : 4) ?? 4;
69
67
  const blob = new Blob([WORKER_SCRIPT], { type: 'application/javascript' });
70
68
  this.workerUrl = URL.createObjectURL(blob);
@@ -166,7 +164,7 @@ export class WorkerTransport {
166
164
  }
167
165
  this.workers = [];
168
166
  URL.revokeObjectURL(this.workerUrl);
169
- for (const [id, pending] of this.pendingRequests) {
167
+ for (const pending of this.pendingRequests.values()) {
170
168
  pending.reject(new Error('Transport terminated'));
171
169
  }
172
170
  this.pendingRequests.clear();
@@ -24,6 +24,10 @@ export interface RedirectInfo {
24
24
  status: number;
25
25
  headers: Headers;
26
26
  }
27
+ export interface ReckerRuntimeEventBus {
28
+ on<K extends string>(name: K, handler: (event: unknown) => void): () => void;
29
+ emit<K extends string>(name: K, event: unknown): void;
30
+ }
27
31
  export interface RequestOptions {
28
32
  method?: Method;
29
33
  headers?: HeadersInit;
@@ -40,6 +44,11 @@ export interface RequestOptions {
40
44
  onUploadProgress?: ProgressCallback;
41
45
  onDownloadProgress?: ProgressCallback;
42
46
  maxResponseSize?: number;
47
+ correlationId?: string;
48
+ tenant?: string;
49
+ policyTags?: string[];
50
+ policySource?: string;
51
+ traceId?: string;
43
52
  beforeRedirect?: (info: RedirectInfo) => void | false | string | Promise<void | false | string>;
44
53
  maxRedirects?: number;
45
54
  followRedirects?: boolean;
@@ -57,6 +66,11 @@ export interface ReckerRequest {
57
66
  onUploadProgress?: ProgressCallback;
58
67
  onDownloadProgress?: ProgressCallback;
59
68
  maxResponseSize?: number;
69
+ correlationId?: string;
70
+ tenant?: string;
71
+ policyTags?: string[];
72
+ policySource?: string;
73
+ traceId?: string;
60
74
  beforeRedirect?: (info: RedirectInfo) => void | false | string | Promise<void | false | string>;
61
75
  maxRedirects?: number;
62
76
  followRedirects?: boolean;
@@ -74,6 +88,15 @@ export interface ReckerRequest {
74
88
  onHttp2Stream?: (info: any) => void;
75
89
  onHttp2FlowControl?: (info: any) => void;
76
90
  };
91
+ _runtime?: {
92
+ requestId: string;
93
+ correlationId: string;
94
+ tenant?: string;
95
+ policyTags: string[];
96
+ policySource?: string;
97
+ createdAt: number;
98
+ traceId?: string;
99
+ };
77
100
  }
78
101
  export interface Timings {
79
102
  queuing?: number;
@@ -378,6 +401,7 @@ export interface ClientOptions {
378
401
  cookies?: boolean | CookieOptions;
379
402
  timeout?: number | TimeoutOptions;
380
403
  observability?: boolean;
404
+ runtimeEventBus?: ReckerRuntimeEventBus;
381
405
  retry?: {
382
406
  maxAttempts?: number;
383
407
  delay?: number;
package/dist/version.js CHANGED
@@ -1,4 +1,4 @@
1
- const VERSION = '1.0.72';
1
+ const VERSION = '1.0.75-next.2e5a94f';
2
2
  let _version = null;
3
3
  export async function getVersion() {
4
4
  if (_version)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "recker",
3
- "version": "1.0.72",
3
+ "version": "1.0.75-next.2e5a94f",
4
4
  "description": "Multi-Protocol SDK for the AI Era - HTTP, WebSocket, DNS, FTP, SFTP, Telnet, HLS unified with AI providers and MCP tools",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",
@@ -127,6 +127,14 @@
127
127
  "types": "./dist/plugins/*.d.ts",
128
128
  "import": "./dist/plugins/*.js"
129
129
  },
130
+ "./search": {
131
+ "types": "./dist/search/index.d.ts",
132
+ "import": "./dist/search/index.js"
133
+ },
134
+ "./search/*": {
135
+ "types": "./dist/search/*.d.ts",
136
+ "import": "./dist/search/*.js"
137
+ },
130
138
  "./scrape": {
131
139
  "types": "./dist/scrape/index.d.ts",
132
140
  "import": "./dist/scrape/index.js"