recker 1.0.86 → 1.0.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/dist/browser/core/client.d.ts +2 -0
  2. package/dist/browser/core/client.js +8 -0
  3. package/dist/browser/core/request.d.ts +3 -0
  4. package/dist/browser/core/request.js +6 -2
  5. package/dist/browser/index.iife.min.js +56 -56
  6. package/dist/browser/index.min.js +56 -56
  7. package/dist/browser/index.mini.iife.js +312 -15
  8. package/dist/browser/index.mini.iife.min.js +38 -38
  9. package/dist/browser/index.mini.min.js +42 -42
  10. package/dist/browser/index.mini.umd.js +312 -15
  11. package/dist/browser/index.mini.umd.min.js +38 -38
  12. package/dist/browser/index.umd.min.js +56 -56
  13. package/dist/browser/plugins/queue.d.ts +41 -0
  14. package/dist/browser/plugins/queue.js +184 -0
  15. package/dist/browser/scrape/crawl-queue.d.ts +31 -0
  16. package/dist/browser/scrape/crawl-queue.js +40 -0
  17. package/dist/browser/scrape/crawl-storage.d.ts +33 -0
  18. package/dist/browser/scrape/crawl-storage.js +26 -0
  19. package/dist/browser/scrape/index.d.ts +6 -0
  20. package/dist/browser/scrape/index.js +3 -0
  21. package/dist/browser/scrape/proxy-adapter.d.ts +12 -0
  22. package/dist/browser/scrape/proxy-adapter.js +17 -0
  23. package/dist/browser/scrape/spider.d.ts +14 -4
  24. package/dist/browser/scrape/spider.js +119 -45
  25. package/dist/browser/transport/curl.js +53 -9
  26. package/dist/browser/transport/undici.js +4 -0
  27. package/dist/browser/types/index.d.ts +53 -2
  28. package/dist/core/client.d.ts +2 -0
  29. package/dist/core/client.js +8 -0
  30. package/dist/core/request.d.ts +3 -0
  31. package/dist/core/request.js +6 -2
  32. package/dist/index.d.ts +2 -0
  33. package/dist/index.js +2 -0
  34. package/dist/plugins/queue.d.ts +41 -0
  35. package/dist/plugins/queue.js +184 -0
  36. package/dist/queue/consumer.d.ts +17 -0
  37. package/dist/queue/consumer.js +48 -0
  38. package/dist/scrape/crawl-queue.d.ts +31 -0
  39. package/dist/scrape/crawl-queue.js +40 -0
  40. package/dist/scrape/crawl-storage.d.ts +33 -0
  41. package/dist/scrape/crawl-storage.js +26 -0
  42. package/dist/scrape/index.d.ts +6 -0
  43. package/dist/scrape/index.js +3 -0
  44. package/dist/scrape/proxy-adapter.d.ts +12 -0
  45. package/dist/scrape/proxy-adapter.js +17 -0
  46. package/dist/scrape/spider.d.ts +14 -4
  47. package/dist/scrape/spider.js +119 -45
  48. package/dist/transport/curl.js +53 -9
  49. package/dist/transport/undici.js +4 -0
  50. package/dist/types/index.d.ts +53 -2
  51. package/dist/version.js +1 -1
  52. package/package.json +1 -1
@@ -1,4 +1,5 @@
1
1
  import { ClientOptions, Middleware, ReckerRequest, ReckerResponse, RequestOptions, CacheStorage, PageResult } from '../types/index.js';
2
+ import type { QueueOptions } from '../types/index.js';
2
3
  import type { ClientAI, ClientOptionsWithAI } from '../types/ai-client.js';
3
4
  import { RequestPromise } from './request-promise.js';
4
5
  import { type RuntimeEventName, type RuntimeEventPayloads, type TypedEventBus } from '../core-runtime/typed-events.js';
@@ -27,6 +28,7 @@ export interface ExtendedClientOptions extends ClientOptions {
27
28
  retry?: RetryOptions;
28
29
  cache?: ClientCacheConfig;
29
30
  dedup?: DedupOptions;
31
+ queue?: QueueOptions;
30
32
  }
31
33
  export declare class Client {
32
34
  static get version(): string;
@@ -17,6 +17,7 @@ import { paginate, streamPages } from '../plugins/pagination.js';
17
17
  import { retryPlugin } from '../plugins/retry.js';
18
18
  import { cachePlugin } from '../plugins/cache.js';
19
19
  import { dedupPlugin } from '../plugins/dedup.js';
20
+ import { queuePlugin } from '../plugins/queue.js';
20
21
  import { createXSRFMiddleware } from '../plugins/xsrf.js';
21
22
  import { createCompressionMiddleware } from '../plugins/compression.js';
22
23
  import { serializeXML } from '../plugins/xml.js';
@@ -389,6 +390,13 @@ export class Client {
389
390
  scope: 'request'
390
391
  });
391
392
  }
393
+ if (options.queue) {
394
+ registerPlugin(queuePlugin(options.queue), {
395
+ name: 'recker:queue',
396
+ priority: 135,
397
+ scope: 'request'
398
+ });
399
+ }
392
400
  if (options.plugins) {
393
401
  options.plugins.forEach((plugin, index) => {
394
402
  const existingManifest = getPluginManifest(plugin);
@@ -20,6 +20,9 @@ export declare class HttpRequest implements ReckerRequest {
20
20
  readonly policyTags: string[];
21
21
  readonly policySource?: string;
22
22
  readonly traceId?: string;
23
+ readonly queue?: boolean | {
24
+ metadata?: Record<string, unknown>;
25
+ };
23
26
  constructor(url: string, options?: RequestOptions);
24
27
  withHeader(name: string, value: string): ReckerRequest;
25
28
  withBody(body: BodyInit): ReckerRequest;
@@ -28,6 +28,7 @@ export class HttpRequest {
28
28
  policyTags;
29
29
  policySource;
30
30
  traceId;
31
+ queue;
31
32
  constructor(url, options = {}) {
32
33
  this.url = url;
33
34
  this.method = options.method || 'GET';
@@ -51,6 +52,7 @@ export class HttpRequest {
51
52
  this.policyTags = options.policyTags ?? [];
52
53
  this.policySource = options.policySource;
53
54
  this.traceId = options.traceId;
55
+ this.queue = options.queue;
54
56
  }
55
57
  withHeader(name, value) {
56
58
  const context = getRequestContext(this);
@@ -75,7 +77,8 @@ export class HttpRequest {
75
77
  tenant: this.tenant,
76
78
  policyTags: this.policyTags,
77
79
  policySource: this.policySource,
78
- traceId: this.traceId
80
+ traceId: this.traceId,
81
+ queue: this.queue,
79
82
  });
80
83
  if (context) {
81
84
  return attachRequestContext(request, context);
@@ -103,7 +106,8 @@ export class HttpRequest {
103
106
  tenant: this.tenant,
104
107
  policyTags: this.policyTags,
105
108
  policySource: this.policySource,
106
- traceId: this.traceId
109
+ traceId: this.traceId,
110
+ queue: this.queue,
107
111
  });
108
112
  if (context) {
109
113
  return attachRequestContext(request, context);