recker 1.0.86 → 1.0.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/core/client.d.ts +2 -0
- package/dist/browser/core/client.js +8 -0
- package/dist/browser/core/request.d.ts +3 -0
- package/dist/browser/core/request.js +6 -2
- package/dist/browser/index.iife.min.js +56 -56
- package/dist/browser/index.min.js +56 -56
- package/dist/browser/index.mini.iife.js +312 -15
- package/dist/browser/index.mini.iife.min.js +38 -38
- package/dist/browser/index.mini.min.js +42 -42
- package/dist/browser/index.mini.umd.js +312 -15
- package/dist/browser/index.mini.umd.min.js +38 -38
- package/dist/browser/index.umd.min.js +56 -56
- package/dist/browser/plugins/queue.d.ts +41 -0
- package/dist/browser/plugins/queue.js +184 -0
- package/dist/browser/scrape/crawl-queue.d.ts +31 -0
- package/dist/browser/scrape/crawl-queue.js +40 -0
- package/dist/browser/scrape/crawl-storage.d.ts +33 -0
- package/dist/browser/scrape/crawl-storage.js +26 -0
- package/dist/browser/scrape/index.d.ts +6 -0
- package/dist/browser/scrape/index.js +3 -0
- package/dist/browser/scrape/proxy-adapter.d.ts +12 -0
- package/dist/browser/scrape/proxy-adapter.js +17 -0
- package/dist/browser/scrape/spider.d.ts +14 -4
- package/dist/browser/scrape/spider.js +119 -45
- package/dist/browser/transport/curl.js +53 -9
- package/dist/browser/transport/undici.js +4 -0
- package/dist/browser/types/index.d.ts +53 -2
- package/dist/core/client.d.ts +2 -0
- package/dist/core/client.js +8 -0
- package/dist/core/request.d.ts +3 -0
- package/dist/core/request.js +6 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/plugins/queue.d.ts +41 -0
- package/dist/plugins/queue.js +184 -0
- package/dist/queue/consumer.d.ts +17 -0
- package/dist/queue/consumer.js +48 -0
- package/dist/scrape/crawl-queue.d.ts +31 -0
- package/dist/scrape/crawl-queue.js +40 -0
- package/dist/scrape/crawl-storage.d.ts +33 -0
- package/dist/scrape/crawl-storage.js +26 -0
- package/dist/scrape/index.d.ts +6 -0
- package/dist/scrape/index.js +3 -0
- package/dist/scrape/proxy-adapter.d.ts +12 -0
- package/dist/scrape/proxy-adapter.js +17 -0
- package/dist/scrape/spider.d.ts +14 -4
- package/dist/scrape/spider.js +119 -45
- package/dist/transport/curl.js +53 -9
- package/dist/transport/undici.js +4 -0
- package/dist/types/index.d.ts +53 -2
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ClientOptions, Middleware, ReckerRequest, ReckerResponse, RequestOptions, CacheStorage, PageResult } from '../types/index.js';
|
|
2
|
+
import type { QueueOptions } from '../types/index.js';
|
|
2
3
|
import type { ClientAI, ClientOptionsWithAI } from '../types/ai-client.js';
|
|
3
4
|
import { RequestPromise } from './request-promise.js';
|
|
4
5
|
import { type RuntimeEventName, type RuntimeEventPayloads, type TypedEventBus } from '../core-runtime/typed-events.js';
|
|
@@ -27,6 +28,7 @@ export interface ExtendedClientOptions extends ClientOptions {
|
|
|
27
28
|
retry?: RetryOptions;
|
|
28
29
|
cache?: ClientCacheConfig;
|
|
29
30
|
dedup?: DedupOptions;
|
|
31
|
+
queue?: QueueOptions;
|
|
30
32
|
}
|
|
31
33
|
export declare class Client {
|
|
32
34
|
static get version(): string;
|
|
@@ -17,6 +17,7 @@ import { paginate, streamPages } from '../plugins/pagination.js';
|
|
|
17
17
|
import { retryPlugin } from '../plugins/retry.js';
|
|
18
18
|
import { cachePlugin } from '../plugins/cache.js';
|
|
19
19
|
import { dedupPlugin } from '../plugins/dedup.js';
|
|
20
|
+
import { queuePlugin } from '../plugins/queue.js';
|
|
20
21
|
import { createXSRFMiddleware } from '../plugins/xsrf.js';
|
|
21
22
|
import { createCompressionMiddleware } from '../plugins/compression.js';
|
|
22
23
|
import { serializeXML } from '../plugins/xml.js';
|
|
@@ -389,6 +390,13 @@ export class Client {
|
|
|
389
390
|
scope: 'request'
|
|
390
391
|
});
|
|
391
392
|
}
|
|
393
|
+
if (options.queue) {
|
|
394
|
+
registerPlugin(queuePlugin(options.queue), {
|
|
395
|
+
name: 'recker:queue',
|
|
396
|
+
priority: 135,
|
|
397
|
+
scope: 'request'
|
|
398
|
+
});
|
|
399
|
+
}
|
|
392
400
|
if (options.plugins) {
|
|
393
401
|
options.plugins.forEach((plugin, index) => {
|
|
394
402
|
const existingManifest = getPluginManifest(plugin);
|
|
@@ -20,6 +20,9 @@ export declare class HttpRequest implements ReckerRequest {
|
|
|
20
20
|
readonly policyTags: string[];
|
|
21
21
|
readonly policySource?: string;
|
|
22
22
|
readonly traceId?: string;
|
|
23
|
+
readonly queue?: boolean | {
|
|
24
|
+
metadata?: Record<string, unknown>;
|
|
25
|
+
};
|
|
23
26
|
constructor(url: string, options?: RequestOptions);
|
|
24
27
|
withHeader(name: string, value: string): ReckerRequest;
|
|
25
28
|
withBody(body: BodyInit): ReckerRequest;
|
|
@@ -28,6 +28,7 @@ export class HttpRequest {
|
|
|
28
28
|
policyTags;
|
|
29
29
|
policySource;
|
|
30
30
|
traceId;
|
|
31
|
+
queue;
|
|
31
32
|
constructor(url, options = {}) {
|
|
32
33
|
this.url = url;
|
|
33
34
|
this.method = options.method || 'GET';
|
|
@@ -51,6 +52,7 @@ export class HttpRequest {
|
|
|
51
52
|
this.policyTags = options.policyTags ?? [];
|
|
52
53
|
this.policySource = options.policySource;
|
|
53
54
|
this.traceId = options.traceId;
|
|
55
|
+
this.queue = options.queue;
|
|
54
56
|
}
|
|
55
57
|
withHeader(name, value) {
|
|
56
58
|
const context = getRequestContext(this);
|
|
@@ -75,7 +77,8 @@ export class HttpRequest {
|
|
|
75
77
|
tenant: this.tenant,
|
|
76
78
|
policyTags: this.policyTags,
|
|
77
79
|
policySource: this.policySource,
|
|
78
|
-
traceId: this.traceId
|
|
80
|
+
traceId: this.traceId,
|
|
81
|
+
queue: this.queue,
|
|
79
82
|
});
|
|
80
83
|
if (context) {
|
|
81
84
|
return attachRequestContext(request, context);
|
|
@@ -103,7 +106,8 @@ export class HttpRequest {
|
|
|
103
106
|
tenant: this.tenant,
|
|
104
107
|
policyTags: this.policyTags,
|
|
105
108
|
policySource: this.policySource,
|
|
106
|
-
traceId: this.traceId
|
|
109
|
+
traceId: this.traceId,
|
|
110
|
+
queue: this.queue,
|
|
107
111
|
});
|
|
108
112
|
if (context) {
|
|
109
113
|
return attachRequestContext(request, context);
|