recker 1.0.85 → 1.0.86-next.a24fa13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/browser/browser/index.d.ts +2 -0
  2. package/dist/browser/browser/index.js +1 -0
  3. package/dist/browser/browser/recker.d.ts +2 -0
  4. package/dist/browser/browser/recker.js +2 -0
  5. package/dist/browser/core/client.d.ts +2 -0
  6. package/dist/browser/core/client.js +8 -0
  7. package/dist/browser/core/request.d.ts +3 -0
  8. package/dist/browser/core/request.js +6 -2
  9. package/dist/browser/index.d.ts +2 -0
  10. package/dist/browser/index.iife.min.js +79 -79
  11. package/dist/browser/index.js +1 -0
  12. package/dist/browser/index.min.js +79 -79
  13. package/dist/browser/index.mini.iife.js +312 -15
  14. package/dist/browser/index.mini.iife.min.js +38 -38
  15. package/dist/browser/index.mini.min.js +42 -42
  16. package/dist/browser/index.mini.umd.js +312 -15
  17. package/dist/browser/index.mini.umd.min.js +38 -38
  18. package/dist/browser/index.umd.min.js +79 -79
  19. package/dist/browser/plugins/queue.d.ts +41 -0
  20. package/dist/browser/plugins/queue.js +184 -0
  21. package/dist/browser/recker.d.ts +2 -0
  22. package/dist/browser/recker.js +2 -0
  23. package/dist/browser/scrape/crawl-queue.d.ts +31 -0
  24. package/dist/browser/scrape/crawl-queue.js +40 -0
  25. package/dist/browser/scrape/crawl-storage.d.ts +33 -0
  26. package/dist/browser/scrape/crawl-storage.js +26 -0
  27. package/dist/browser/scrape/index.d.ts +6 -0
  28. package/dist/browser/scrape/index.js +3 -0
  29. package/dist/browser/scrape/proxy-adapter.d.ts +12 -0
  30. package/dist/browser/scrape/proxy-adapter.js +17 -0
  31. package/dist/browser/scrape/spider.d.ts +14 -4
  32. package/dist/browser/scrape/spider.js +119 -45
  33. package/dist/browser/transport/curl.js +53 -9
  34. package/dist/browser/transport/undici.js +4 -0
  35. package/dist/browser/types/index.d.ts +53 -2
  36. package/dist/core/client.d.ts +2 -0
  37. package/dist/core/client.js +8 -0
  38. package/dist/core/request.d.ts +3 -0
  39. package/dist/core/request.js +6 -2
  40. package/dist/index.d.ts +2 -0
  41. package/dist/index.js +2 -0
  42. package/dist/plugins/queue.d.ts +41 -0
  43. package/dist/plugins/queue.js +184 -0
  44. package/dist/queue/consumer.d.ts +17 -0
  45. package/dist/queue/consumer.js +48 -0
  46. package/dist/scrape/crawl-queue.d.ts +31 -0
  47. package/dist/scrape/crawl-queue.js +40 -0
  48. package/dist/scrape/crawl-storage.d.ts +33 -0
  49. package/dist/scrape/crawl-storage.js +26 -0
  50. package/dist/scrape/index.d.ts +6 -0
  51. package/dist/scrape/index.js +3 -0
  52. package/dist/scrape/proxy-adapter.d.ts +12 -0
  53. package/dist/scrape/proxy-adapter.js +17 -0
  54. package/dist/scrape/spider.d.ts +14 -4
  55. package/dist/scrape/spider.js +119 -45
  56. package/dist/transport/curl.js +53 -9
  57. package/dist/transport/undici.js +4 -0
  58. package/dist/types/index.d.ts +53 -2
  59. package/dist/version.js +1 -1
  60. package/package.json +2 -2
@@ -32,5 +32,7 @@ export * from '../constants/http-status.js';
32
32
  export * from './crypto.js';
33
33
  export * from './cache.js';
34
34
  export * from './compression.js';
35
+ export { createRaffelClient } from 'raffel/client/browser';
36
+ export type { RaffelClientOptions, RaffelClient, ClientStream as RaffelClientStream, CallOptions as RaffelCallOptions, ClientChannel as RaffelClientChannel, ClientChannelMember as RaffelClientChannelMember, } from 'raffel/client/browser';
35
37
  export * from './recker.js';
36
38
  export { Client as Recker } from '../core/client.js';
@@ -32,5 +32,6 @@ export * from '../constants/http-status.js';
32
32
  export * from './crypto.js';
33
33
  export * from './cache.js';
34
34
  export * from './compression.js';
35
+ export { createRaffelClient } from 'raffel/client/browser';
35
36
  export * from './recker.js';
36
37
  export { Client as Recker } from '../core/client.js';
@@ -4,6 +4,7 @@ import type { RequestOptions } from '../types/index.js';
4
4
  import { analyzeSeo } from '../seo/analyzer.js';
5
5
  import { createAI } from '../ai/index.js';
6
6
  import { simulateNetwork } from '../plugins/network-simulation.js';
7
+ import type { RaffelClientOptions, RaffelClient } from 'raffel/client/browser';
7
8
  export declare function get<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
8
9
  export declare function post<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
9
10
  export declare function post<T = unknown>(url: string, body?: unknown, options?: RequestOptions): RequestPromise<T>;
@@ -26,6 +27,7 @@ export declare const recker: {
26
27
  options: typeof options;
27
28
  purge: typeof purge;
28
29
  ws: typeof ws;
30
+ raffel: (url: string, options?: Omit<RaffelClientOptions, "url">) => RaffelClient;
29
31
  seo: typeof analyzeSeo;
30
32
  ai: typeof createAI;
31
33
  har: import("../plugins/har-recorder.js").HarRecorder;
@@ -4,6 +4,7 @@ import { analyzeSeo } from '../seo/analyzer.js';
4
4
  import { createAI } from '../ai/index.js';
5
5
  import { harRecorder } from '../plugins/har-recorder.js';
6
6
  import { simulateNetwork } from '../plugins/network-simulation.js';
7
+ import { createRaffelClient } from 'raffel/client/browser';
7
8
  let _defaultClient = null;
8
9
  const REQUEST_OPTIONS_HINTS = new Set([
9
10
  'method',
@@ -98,6 +99,7 @@ export const recker = {
98
99
  options,
99
100
  purge,
100
101
  ws,
102
+ raffel: (url, options) => createRaffelClient({ ...options, url }),
101
103
  seo: analyzeSeo,
102
104
  ai: createAI,
103
105
  har: harRecorder,
@@ -1,4 +1,5 @@
1
1
  import { ClientOptions, Middleware, ReckerRequest, ReckerResponse, RequestOptions, CacheStorage, PageResult } from '../types/index.js';
2
+ import type { QueueOptions } from '../types/index.js';
2
3
  import type { ClientAI, ClientOptionsWithAI } from '../types/ai-client.js';
3
4
  import { RequestPromise } from './request-promise.js';
4
5
  import { type RuntimeEventName, type RuntimeEventPayloads, type TypedEventBus } from '../core-runtime/typed-events.js';
@@ -27,6 +28,7 @@ export interface ExtendedClientOptions extends ClientOptions {
27
28
  retry?: RetryOptions;
28
29
  cache?: ClientCacheConfig;
29
30
  dedup?: DedupOptions;
31
+ queue?: QueueOptions;
30
32
  }
31
33
  export declare class Client {
32
34
  static get version(): string;
@@ -17,6 +17,7 @@ import { paginate, streamPages } from '../plugins/pagination.js';
17
17
  import { retryPlugin } from '../plugins/retry.js';
18
18
  import { cachePlugin } from '../plugins/cache.js';
19
19
  import { dedupPlugin } from '../plugins/dedup.js';
20
+ import { queuePlugin } from '../plugins/queue.js';
20
21
  import { createXSRFMiddleware } from '../plugins/xsrf.js';
21
22
  import { createCompressionMiddleware } from '../plugins/compression.js';
22
23
  import { serializeXML } from '../plugins/xml.js';
@@ -389,6 +390,13 @@ export class Client {
389
390
  scope: 'request'
390
391
  });
391
392
  }
393
+ if (options.queue) {
394
+ registerPlugin(queuePlugin(options.queue), {
395
+ name: 'recker:queue',
396
+ priority: 135,
397
+ scope: 'request'
398
+ });
399
+ }
392
400
  if (options.plugins) {
393
401
  options.plugins.forEach((plugin, index) => {
394
402
  const existingManifest = getPluginManifest(plugin);
@@ -20,6 +20,9 @@ export declare class HttpRequest implements ReckerRequest {
20
20
  readonly policyTags: string[];
21
21
  readonly policySource?: string;
22
22
  readonly traceId?: string;
23
+ readonly queue?: boolean | {
24
+ metadata?: Record<string, unknown>;
25
+ };
23
26
  constructor(url: string, options?: RequestOptions);
24
27
  withHeader(name: string, value: string): ReckerRequest;
25
28
  withBody(body: BodyInit): ReckerRequest;
@@ -28,6 +28,7 @@ export class HttpRequest {
28
28
  policyTags;
29
29
  policySource;
30
30
  traceId;
31
+ queue;
31
32
  constructor(url, options = {}) {
32
33
  this.url = url;
33
34
  this.method = options.method || 'GET';
@@ -51,6 +52,7 @@ export class HttpRequest {
51
52
  this.policyTags = options.policyTags ?? [];
52
53
  this.policySource = options.policySource;
53
54
  this.traceId = options.traceId;
55
+ this.queue = options.queue;
54
56
  }
55
57
  withHeader(name, value) {
56
58
  const context = getRequestContext(this);
@@ -75,7 +77,8 @@ export class HttpRequest {
75
77
  tenant: this.tenant,
76
78
  policyTags: this.policyTags,
77
79
  policySource: this.policySource,
78
- traceId: this.traceId
80
+ traceId: this.traceId,
81
+ queue: this.queue,
79
82
  });
80
83
  if (context) {
81
84
  return attachRequestContext(request, context);
@@ -103,7 +106,8 @@ export class HttpRequest {
103
106
  tenant: this.tenant,
104
107
  policyTags: this.policyTags,
105
108
  policySource: this.policySource,
106
- traceId: this.traceId
109
+ traceId: this.traceId,
110
+ queue: this.queue,
107
111
  });
108
112
  if (context) {
109
113
  return attachRequestContext(request, context);
@@ -32,5 +32,7 @@ export * from '../constants/http-status.js';
32
32
  export * from './crypto.js';
33
33
  export * from './cache.js';
34
34
  export * from './compression.js';
35
+ export { createRaffelClient } from 'raffel/client/browser';
36
+ export type { RaffelClientOptions, RaffelClient, ClientStream as RaffelClientStream, CallOptions as RaffelCallOptions, ClientChannel as RaffelClientChannel, ClientChannelMember as RaffelClientChannelMember, } from 'raffel/client/browser';
35
37
  export * from './recker.js';
36
38
  export { Client as Recker } from '../core/client.js';