scrapely 1.2.33 → 1.2.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/NOTICE +12 -26
- package/README.md +1 -2
- package/dist/actor.d.ts +32 -25
- package/dist/actor.js +43 -29
- package/dist/charging.d.ts +49 -4
- package/dist/charging.js +293 -44
- package/dist/configuration.js +2 -2
- package/dist/index.js +1 -1
- package/dist/node-wrapper.js +1 -1
- package/dist/patch-browser-pool.js +7 -3
- package/dist/proxy-configuration.d.ts +11 -11
- package/dist/proxy-configuration.js +1 -1
- package/dist/proxy-input-aliases.d.ts +1 -1
- package/dist/proxy-input-aliases.js +1 -1
- package/dist/request-list.d.ts +2 -2
- package/dist/request-list.js +1 -1
- package/dist/scrapely-proxy-configuration.d.ts +15 -16
- package/dist/scrapely-proxy-configuration.js +27 -31
- package/dist/storage/api-storage-client.d.ts +4 -5
- package/dist/storage/api-storage-client.js +27 -24
- package/dist/storage/billing-tracker.d.ts +1 -1
- package/dist/storage/billing-tracker.js +1 -1
- package/dist/storage/key-value-store-client.d.ts +0 -3
- package/dist/storage/key-value-store-client.js +3 -6
- package/dist/storage/kv-store-collection-client.js +6 -2
- package/dist/storage/kvs-iteration.d.ts +1 -1
- package/dist/storage/kvs-iteration.js +1 -1
- package/dist/storage/request-queue-client.d.ts +1 -1
- package/dist/storage/request-queue-client.js +5 -5
- package/dist/storage/rq-lock-script.d.ts +2 -2
- package/dist/storage/rq-lock-script.js +2 -2
- package/dist/storage/run-metadata.js +1 -1
- package/dist/storage/ttl-manager.d.ts +7 -6
- package/dist/storage/ttl-manager.js +5 -4
- package/dist/storage/utils.d.ts +8 -0
- package/dist/storage/utils.js +17 -0
- package/dist/storage-identifiers.d.ts +1 -2
- package/dist/storage-identifiers.js +1 -2
- package/dist/utils.d.ts +41 -8
- package/dist/utils.js +49 -3
- package/package.json +23 -11
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { KeyValueStoreClientListData, KeyValueStoreClientListOptions, KeyValueStoreItemData } from '@crawlee/types';
|
|
2
2
|
export type ListKeysFn = (options?: KeyValueStoreClientListOptions) => Promise<KeyValueStoreClientListData>;
|
|
3
3
|
/**
|
|
4
|
-
* Filter keys by
|
|
4
|
+
* Filter keys by collection config from a KVS schema.
|
|
5
5
|
*/
|
|
6
6
|
export declare function filterKeysByCollection(allKeys: string[], schemaJson: string | undefined, collection: string): string[];
|
|
7
7
|
export declare function paginateKeyItems(listKeys: ListKeysFn, options?: KeyValueStoreClientListOptions): AsyncGenerator<KeyValueStoreItemData>;
|
|
@@ -7,7 +7,7 @@ exports.makePaginatingKeysIterable = makePaginatingKeysIterable;
|
|
|
7
7
|
exports.makePaginatingValuesIterable = makePaginatingValuesIterable;
|
|
8
8
|
exports.makePaginatingEntriesIterable = makePaginatingEntriesIterable;
|
|
9
9
|
/**
|
|
10
|
-
* Filter keys by
|
|
10
|
+
* Filter keys by collection config from a KVS schema.
|
|
11
11
|
*/
|
|
12
12
|
function filterKeysByCollection(allKeys, schemaJson, collection) {
|
|
13
13
|
if (!schemaJson) {
|
|
@@ -44,7 +44,7 @@ export declare class DragonflyRequestQueueClient implements RequestQueueClient {
|
|
|
44
44
|
*/
|
|
45
45
|
private trackStorageBytes;
|
|
46
46
|
/**
|
|
47
|
-
* Track clientKey access for hadMultipleClients
|
|
47
|
+
* Track clientKey access for hadMultipleClients.
|
|
48
48
|
* Returns the current hadMultipleClients flag after update.
|
|
49
49
|
*/
|
|
50
50
|
private trackClientKey;
|
|
@@ -66,7 +66,7 @@ class DragonflyRequestQueueClient {
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
/**
|
|
69
|
-
* Track clientKey access for hadMultipleClients
|
|
69
|
+
* Track clientKey access for hadMultipleClients.
|
|
70
70
|
* Returns the current hadMultipleClients flag after update.
|
|
71
71
|
*/
|
|
72
72
|
async trackClientKey(clientKey) {
|
|
@@ -333,11 +333,11 @@ class DragonflyRequestQueueClient {
|
|
|
333
333
|
async listRequests(options) {
|
|
334
334
|
const limit = Math.min(options?.limit ?? 100, 10000);
|
|
335
335
|
const now = Date.now();
|
|
336
|
-
// Validate that filter is not used with deprecated exclusiveStartId
|
|
336
|
+
// Validate that filter is not used with deprecated exclusiveStartId
|
|
337
337
|
if (options?.filter && options.filter.length > 0 && options?.exclusiveStartId) {
|
|
338
338
|
throw new Error('filter parameter is not compatible with deprecated exclusiveStartId parameter');
|
|
339
339
|
}
|
|
340
|
-
// Track clientKey for hadMultipleClients
|
|
340
|
+
// Track clientKey for hadMultipleClients
|
|
341
341
|
if (options?.clientKey) {
|
|
342
342
|
const existingClientKey = await this.client.redis.hget(this.infoKey, 'clientKey');
|
|
343
343
|
if (existingClientKey && existingClientKey !== options.clientKey) {
|
|
@@ -348,7 +348,7 @@ class DragonflyRequestQueueClient {
|
|
|
348
348
|
}
|
|
349
349
|
}
|
|
350
350
|
else {
|
|
351
|
-
// No clientKey provided - treat as new client
|
|
351
|
+
// No clientKey provided - treat as new client
|
|
352
352
|
const existingClientKey = await this.client.redis.hget(this.infoKey, 'clientKey');
|
|
353
353
|
if (existingClientKey) {
|
|
354
354
|
await this.client.redis.hset(this.infoKey, 'hadMultipleClients', '1');
|
|
@@ -456,7 +456,7 @@ class DragonflyRequestQueueClient {
|
|
|
456
456
|
lockSecs: options.lockSecs,
|
|
457
457
|
items,
|
|
458
458
|
// From the atomic lock script: active locks held by ANY client
|
|
459
|
-
// (including this call's own)
|
|
459
|
+
// (including this call's own).
|
|
460
460
|
queueHasLockedRequests: activeLockCount > 0,
|
|
461
461
|
};
|
|
462
462
|
}
|
|
@@ -5,7 +5,7 @@ import type Redis from 'ioredis';
|
|
|
5
5
|
* which ioredis delivers as [string[], number]. The ZCARD is taken after the
|
|
6
6
|
* script's expired-lock purge and after this call's own locks are added, so it
|
|
7
7
|
* equals the number of active locks held by ANY client — exactly what the
|
|
8
|
-
*
|
|
8
|
+
* `queueHasLockedRequests` flag means ("locked by any client, either
|
|
9
9
|
* the one calling the endpoint or a different one"). Callers get the flag from
|
|
10
10
|
* this same atomic reply instead of re-reading the whole locked + pending sets.
|
|
11
11
|
*/
|
|
@@ -43,7 +43,7 @@ export declare function matchesLockOwner(owner: string, clientKey: string, runId
|
|
|
43
43
|
* Returns the number of unlocked requests. One round-trip, no race window
|
|
44
44
|
* between ZREM and HDEL, and the whole owners hash never leaves the server.
|
|
45
45
|
*/
|
|
46
|
-
export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n local requestId = flat[i]\n local owner = flat[i + 1]\n --
|
|
46
|
+
export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n local requestId = flat[i]\n local owner = flat[i + 1]\n -- In an Actor run, unlock both (1) requests locked by this\n -- specific run (clientKey:runId) and (2) requests locked by the same\n -- clientKey without run context (clientKey), but NOT requests locked by the\n -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n if owner == expectedOwner or owner == clientKey\n or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then\n redis.call('ZREM', lockedKey, requestId)\n redis.call('HDEL', lockOwnersKey, requestId)\n if forefront == '1' then\n redis.call('ZADD', pendingKey, 0, requestId)\n end\n unlockedCount = unlockedCount + 1\n end\nend\n\nreturn unlockedCount\n";
|
|
47
47
|
export interface UnlockRequestsKeys {
|
|
48
48
|
lockedKey: string;
|
|
49
49
|
lockOwnersKey: string;
|
|
@@ -10,7 +10,7 @@ exports.unlockRequestsAtomic = unlockRequestsAtomic;
|
|
|
10
10
|
* which ioredis delivers as [string[], number]. The ZCARD is taken after the
|
|
11
11
|
* script's expired-lock purge and after this call's own locks are added, so it
|
|
12
12
|
* equals the number of active locks held by ANY client — exactly what the
|
|
13
|
-
*
|
|
13
|
+
* `queueHasLockedRequests` flag means ("locked by any client, either
|
|
14
14
|
* the one calling the endpoint or a different one"). Callers get the flag from
|
|
15
15
|
* this same atomic reply instead of re-reading the whole locked + pending sets.
|
|
16
16
|
*/
|
|
@@ -104,7 +104,7 @@ local unlockedCount = 0
|
|
|
104
104
|
for i = 1, #flat, 2 do
|
|
105
105
|
local requestId = flat[i]
|
|
106
106
|
local owner = flat[i + 1]
|
|
107
|
-
--
|
|
107
|
+
-- In an Actor run, unlock both (1) requests locked by this
|
|
108
108
|
-- specific run (clientKey:runId) and (2) requests locked by the same
|
|
109
109
|
-- clientKey without run context (clientKey), but NOT requests locked by the
|
|
110
110
|
-- same clientKey in DIFFERENT runs (clientKey:otherRunId).
|
|
@@ -4,7 +4,7 @@ exports.RunMetadataManager = void 0;
|
|
|
4
4
|
const billing_tracker_js_1 = require("./billing-tracker.js");
|
|
5
5
|
/**
|
|
6
6
|
* Number of most-recent runs per user that are retained indefinitely (no TTL).
|
|
7
|
-
*
|
|
7
|
+
* The 10 latest runs are always accessible regardless of plan.
|
|
8
8
|
* Older runs beyond this window fall back to the standard plan-based TTL.
|
|
9
9
|
*/
|
|
10
10
|
const KEEP_N_RUNS = 10;
|
|
@@ -3,13 +3,13 @@ import type Redis from 'ioredis';
|
|
|
3
3
|
* TTL configuration in seconds
|
|
4
4
|
*/
|
|
5
5
|
export interface TtlConfig {
|
|
6
|
-
/** TTL for unnamed datasets (free users) - default
|
|
6
|
+
/** TTL for unnamed datasets (free users) - default 48 hours (Free plan) */
|
|
7
7
|
datasetFree: number;
|
|
8
|
-
/** TTL for unnamed datasets (paid users) - default 31 days */
|
|
8
|
+
/** TTL for unnamed datasets (paid users) - default 31 days (Starter, the shortest paid plan) */
|
|
9
9
|
datasetPaid: number;
|
|
10
|
-
/** TTL for unnamed KV stores (free users) - default
|
|
10
|
+
/** TTL for unnamed KV stores (free users) - default 48 hours (Free plan) */
|
|
11
11
|
kvFree: number;
|
|
12
|
-
/** TTL for unnamed KV stores (paid users) - default 31 days */
|
|
12
|
+
/** TTL for unnamed KV stores (paid users) - default 31 days (Starter, the shortest paid plan) */
|
|
13
13
|
kvPaid: number;
|
|
14
14
|
/** Fallback TTL for request queues (safety net for orphaned RQs) - default 24 hours */
|
|
15
15
|
requestQueueFallback: number;
|
|
@@ -21,7 +21,7 @@ export interface TtlConfig {
|
|
|
21
21
|
logMaxSize: number;
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
24
|
-
* Default TTL configuration
|
|
24
|
+
* Default TTL configuration
|
|
25
25
|
*/
|
|
26
26
|
export declare const DEFAULT_TTL_CONFIG: TtlConfig;
|
|
27
27
|
export type StorageType = 'dataset' | 'kv' | 'rq' | 'logs';
|
|
@@ -31,7 +31,8 @@ export type UserPlan = 'free' | 'paid';
|
|
|
31
31
|
*
|
|
32
32
|
* Retention policy:
|
|
33
33
|
* - Named storage: Indefinite (billed), no TTL set
|
|
34
|
-
* - Unnamed datasets/KV stores:
|
|
34
|
+
* - Unnamed datasets/KV stores: the plan's retention (Free 48 h, Starter 31 d,
|
|
35
|
+
* Scale 60 d, Business 90 d); the defaults here are Free and Starter
|
|
35
36
|
* - Request queues: Explicitly deleted after run, 24h fallback TTL for orphaned RQs
|
|
36
37
|
* - Logs: Short TTL in Redis, persisted to S3
|
|
37
38
|
*/
|
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.TtlManager = exports.DEFAULT_TTL_CONFIG = void 0;
|
|
4
4
|
/**
|
|
5
|
-
* Default TTL configuration
|
|
5
|
+
* Default TTL configuration
|
|
6
6
|
*/
|
|
7
7
|
exports.DEFAULT_TTL_CONFIG = {
|
|
8
|
-
datasetFree:
|
|
8
|
+
datasetFree: 48 * 60 * 60, // 48 hours
|
|
9
9
|
datasetPaid: 31 * 24 * 60 * 60, // 31 days
|
|
10
|
-
kvFree:
|
|
10
|
+
kvFree: 48 * 60 * 60, // 48 hours
|
|
11
11
|
kvPaid: 31 * 24 * 60 * 60, // 31 days
|
|
12
12
|
requestQueueFallback: 24 * 60 * 60, // 24 hours
|
|
13
13
|
logsFree: 12 * 60 * 60, // 12 hours
|
|
@@ -19,7 +19,8 @@ exports.DEFAULT_TTL_CONFIG = {
|
|
|
19
19
|
*
|
|
20
20
|
* Retention policy:
|
|
21
21
|
* - Named storage: Indefinite (billed), no TTL set
|
|
22
|
-
* - Unnamed datasets/KV stores:
|
|
22
|
+
* - Unnamed datasets/KV stores: the plan's retention (Free 48 h, Starter 31 d,
|
|
23
|
+
* Scale 60 d, Business 90 d); the defaults here are Free and Starter
|
|
23
24
|
* - Request queues: Explicitly deleted after run, 24h fallback TTL for orphaned RQs
|
|
24
25
|
* - Logs: Short TTL in Redis, persisted to S3
|
|
25
26
|
*/
|
package/dist/storage/utils.d.ts
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
1
|
export declare function uniqueKeyToRequestId(uniqueKey: string): string;
|
|
2
2
|
export declare function generateId(): string;
|
|
3
3
|
export declare function nowISO(): Date;
|
|
4
|
+
/**
|
|
5
|
+
* Percent-encode a value so it stays one URL path segment. Request ids come
|
|
6
|
+
* from queue contents another client may have written: unencoded, an id such
|
|
7
|
+
* as `..` or `../../key-value-stores/x` would send the call to another
|
|
8
|
+
* resource. `''`, `.` and `..` are rejected because a URL parser resolves dot
|
|
9
|
+
* segments even when they are percent-encoded.
|
|
10
|
+
*/
|
|
11
|
+
export declare function pathSegment(value: string): string;
|
package/dist/storage/utils.js
CHANGED
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.uniqueKeyToRequestId = uniqueKeyToRequestId;
|
|
4
4
|
exports.generateId = generateId;
|
|
5
5
|
exports.nowISO = nowISO;
|
|
6
|
+
exports.pathSegment = pathSegment;
|
|
6
7
|
const node_crypto_1 = require("node:crypto");
|
|
7
8
|
function uniqueKeyToRequestId(uniqueKey) {
|
|
8
9
|
return (0, node_crypto_1.createHash)('sha256')
|
|
@@ -17,3 +18,19 @@ function generateId() {
|
|
|
17
18
|
function nowISO() {
|
|
18
19
|
return new Date();
|
|
19
20
|
}
|
|
21
|
+
/**
|
|
22
|
+
* Percent-encode a value so it stays one URL path segment. Request ids come
|
|
23
|
+
* from queue contents another client may have written: unencoded, an id such
|
|
24
|
+
* as `..` or `../../key-value-stores/x` would send the call to another
|
|
25
|
+
* resource. `''`, `.` and `..` are rejected because a URL parser resolves dot
|
|
26
|
+
* segments even when they are percent-encoded.
|
|
27
|
+
*/
|
|
28
|
+
function pathSegment(value) {
|
|
29
|
+
if (typeof value !== 'string') {
|
|
30
|
+
throw new TypeError(`Expected a string for a URL path segment, got ${typeof value}.`);
|
|
31
|
+
}
|
|
32
|
+
if (value === '' || value === '.' || value === '..') {
|
|
33
|
+
throw new Error(`"${value}" cannot be used as a URL path segment.`);
|
|
34
|
+
}
|
|
35
|
+
return encodeURIComponent(value);
|
|
36
|
+
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Storage identifier types
|
|
3
|
-
* them from 'apify' still compile after the apify → scrapely rewrite).
|
|
2
|
+
* Storage identifier types, kept under these names for backward compatibility.
|
|
4
3
|
*/
|
|
5
4
|
/** Identifies a storage by its alias from the Actor's schema storages. */
|
|
6
5
|
export interface StorageAlias {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* Storage identifier types
|
|
4
|
-
* them from 'apify' still compile after the apify → scrapely rewrite).
|
|
3
|
+
* Storage identifier types, kept under these names for backward compatibility.
|
|
5
4
|
*/
|
|
6
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
package/dist/utils.d.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
* These adapt the
|
|
2
|
+
* v2-style utility functions for scrapely.
|
|
3
|
+
* These adapt the v2-style call signatures to Crawlee v3 internally.
|
|
4
4
|
*/
|
|
5
5
|
import type { CheerioAPI } from 'cheerio';
|
|
6
6
|
import type { RequestQueue } from 'crawlee';
|
|
7
7
|
/**
|
|
8
|
-
* Options for enqueueLinks (
|
|
8
|
+
* Options for enqueueLinks (v2-style API).
|
|
9
9
|
*/
|
|
10
10
|
export interface EnqueueLinksOptions {
|
|
11
|
-
/** Cheerio instance ($) - REQUIRED
|
|
11
|
+
/** Cheerio instance ($) - REQUIRED */
|
|
12
12
|
$: CheerioAPI;
|
|
13
|
-
/** RequestQueue instance - REQUIRED
|
|
13
|
+
/** RequestQueue instance - REQUIRED */
|
|
14
14
|
requestQueue: RequestQueue;
|
|
15
15
|
/** CSS selector to find links. Default: 'a[href]' */
|
|
16
16
|
selector?: string;
|
|
@@ -30,13 +30,13 @@ export interface EnqueueLinksOptions {
|
|
|
30
30
|
/**
|
|
31
31
|
* Enqueues links found on the page using a CSS selector.
|
|
32
32
|
*
|
|
33
|
-
* This
|
|
33
|
+
* This v2-style implementation adapts to Crawlee v3 internally.
|
|
34
34
|
* It extracts URLs from the page using cheerio and adds them to the request queue.
|
|
35
35
|
*
|
|
36
36
|
* @example
|
|
37
37
|
* ```typescript
|
|
38
38
|
* // Inside a CheerioCrawler requestHandler:
|
|
39
|
-
* await
|
|
39
|
+
* await Actor.utils.enqueueLinks({
|
|
40
40
|
* $,
|
|
41
41
|
* requestQueue,
|
|
42
42
|
* selector: 'a[href*="/app/"]',
|
|
@@ -45,10 +45,43 @@ export interface EnqueueLinksOptions {
|
|
|
45
45
|
* ```
|
|
46
46
|
*/
|
|
47
47
|
export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<void>;
|
|
48
|
+
/** Options of the v2-style `utils.requestAsBrowser`; other got options pass through. */
|
|
49
|
+
export interface RequestAsBrowserOptions extends Record<string, any> {
|
|
50
|
+
url: string;
|
|
51
|
+
method?: string;
|
|
52
|
+
headers?: Record<string, string>;
|
|
53
|
+
payload?: string | Buffer;
|
|
54
|
+
proxyUrl?: string;
|
|
55
|
+
languageCode?: string;
|
|
56
|
+
countryCode?: string;
|
|
57
|
+
isMobile?: boolean;
|
|
58
|
+
json?: boolean;
|
|
59
|
+
ignoreSslErrors?: boolean;
|
|
60
|
+
useInsecureHttpParser?: boolean;
|
|
61
|
+
abortFunction?: unknown;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* v2-style `utils.requestAsBrowser`: an HTTP request with browser-like headers,
|
|
65
|
+
* here through got-scraping (what v2 itself switched to). Resolves to the got
|
|
66
|
+
* response (`body`, `statusCode`, `headers`, `url`).
|
|
67
|
+
*/
|
|
68
|
+
export declare function requestAsBrowser(options: RequestAsBrowserOptions): Promise<any>;
|
|
69
|
+
/** v2-style `utils.getRandomUserAgent`: a realistic desktop browser User-Agent. */
|
|
70
|
+
export declare function getRandomUserAgent(): string;
|
|
48
71
|
/**
|
|
49
|
-
* Utility namespace providing
|
|
72
|
+
* Utility namespace providing v2-style helper functions.
|
|
50
73
|
* These adapt to Crawlee v3 APIs internally.
|
|
51
74
|
*/
|
|
52
75
|
export declare const utils: {
|
|
53
76
|
enqueueLinks: typeof enqueueLinks;
|
|
77
|
+
requestAsBrowser: typeof requestAsBrowser;
|
|
78
|
+
getRandomUserAgent: typeof getRandomUserAgent;
|
|
79
|
+
sleep: (millis?: number) => any;
|
|
80
|
+
downloadListOfUrls: (options: any) => any;
|
|
81
|
+
extractUrls: (options: any) => any;
|
|
82
|
+
htmlToText: (html: any) => any;
|
|
83
|
+
isDocker: (forceReset?: boolean) => any;
|
|
84
|
+
readonly log: any;
|
|
85
|
+
readonly puppeteer: any;
|
|
86
|
+
readonly playwright: any;
|
|
54
87
|
};
|
package/dist/utils.js
CHANGED
|
@@ -2,16 +2,18 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.utils = void 0;
|
|
4
4
|
exports.enqueueLinks = enqueueLinks;
|
|
5
|
+
exports.requestAsBrowser = requestAsBrowser;
|
|
6
|
+
exports.getRandomUserAgent = getRandomUserAgent;
|
|
5
7
|
/**
|
|
6
8
|
* Enqueues links found on the page using a CSS selector.
|
|
7
9
|
*
|
|
8
|
-
* This
|
|
10
|
+
* This v2-style implementation adapts to Crawlee v3 internally.
|
|
9
11
|
* It extracts URLs from the page using cheerio and adds them to the request queue.
|
|
10
12
|
*
|
|
11
13
|
* @example
|
|
12
14
|
* ```typescript
|
|
13
15
|
* // Inside a CheerioCrawler requestHandler:
|
|
14
|
-
* await
|
|
16
|
+
* await Actor.utils.enqueueLinks({
|
|
15
17
|
* $,
|
|
16
18
|
* requestQueue,
|
|
17
19
|
* selector: 'a[href*="/app/"]',
|
|
@@ -55,9 +57,53 @@ async function enqueueLinks(options) {
|
|
|
55
57
|
}
|
|
56
58
|
}
|
|
57
59
|
/**
|
|
58
|
-
*
|
|
60
|
+
* v2-style `utils.requestAsBrowser`: an HTTP request with browser-like headers,
|
|
61
|
+
* here through got-scraping (what v2 itself switched to). Resolves to the got
|
|
62
|
+
* response (`body`, `statusCode`, `headers`, `url`).
|
|
63
|
+
*/
|
|
64
|
+
async function requestAsBrowser(options) {
|
|
65
|
+
const { payload, languageCode, countryCode, isMobile, json, ignoreSslErrors, abortFunction: _abortFunction, ...gotOptions } = options;
|
|
66
|
+
const { gotScraping } = await import('@crawlee/utils');
|
|
67
|
+
const locale = languageCode ? `${languageCode}${countryCode ? `-${countryCode}` : ''}` : undefined;
|
|
68
|
+
return gotScraping({
|
|
69
|
+
...(payload !== undefined ? { body: payload } : {}),
|
|
70
|
+
...(json ? { responseType: 'json' } : {}),
|
|
71
|
+
...(ignoreSslErrors ? { https: { rejectUnauthorized: false } } : {}),
|
|
72
|
+
headerGeneratorOptions: {
|
|
73
|
+
...(locale ? { locales: [locale] } : {}),
|
|
74
|
+
...(isMobile ? { devices: ['mobile'] } : {}),
|
|
75
|
+
},
|
|
76
|
+
...gotOptions,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
let userAgents;
|
|
80
|
+
/** v2-style `utils.getRandomUserAgent`: a realistic desktop browser User-Agent. */
|
|
81
|
+
function getRandomUserAgent() {
|
|
82
|
+
if (!userAgents) {
|
|
83
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
84
|
+
const { HeaderGenerator } = require('header-generator');
|
|
85
|
+
userAgents = new HeaderGenerator({ devices: ['desktop'] });
|
|
86
|
+
}
|
|
87
|
+
const headers = userAgents.getHeaders();
|
|
88
|
+
return headers['user-agent'] ?? headers['User-Agent'];
|
|
89
|
+
}
|
|
90
|
+
// Loaded on first use: the SDK entry must not load Crawlee (see lazy-exports.ts).
|
|
91
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
92
|
+
const crawleeUtils = () => require('@crawlee/utils');
|
|
93
|
+
/**
|
|
94
|
+
* Utility namespace providing v2-style helper functions.
|
|
59
95
|
* These adapt to Crawlee v3 APIs internally.
|
|
60
96
|
*/
|
|
61
97
|
exports.utils = {
|
|
62
98
|
enqueueLinks,
|
|
99
|
+
requestAsBrowser,
|
|
100
|
+
getRandomUserAgent,
|
|
101
|
+
sleep: (millis) => crawleeUtils().sleep(millis),
|
|
102
|
+
downloadListOfUrls: (options) => crawleeUtils().downloadListOfUrls(options),
|
|
103
|
+
extractUrls: (options) => crawleeUtils().extractUrls(options),
|
|
104
|
+
htmlToText: (html) => crawleeUtils().htmlToText(html),
|
|
105
|
+
isDocker: (forceReset) => crawleeUtils().isDocker(forceReset),
|
|
106
|
+
get log() { return require('@apify/log').default; },
|
|
107
|
+
get puppeteer() { return require('@crawlee/puppeteer').puppeteerUtils; },
|
|
108
|
+
get playwright() { return require('@crawlee/playwright').playwrightUtils; },
|
|
63
109
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scrapely",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.35",
|
|
4
4
|
"description": "Scrapely SDK for building JavaScript and TypeScript Actors",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -21,12 +21,17 @@
|
|
|
21
21
|
"dist",
|
|
22
22
|
"LICENSE",
|
|
23
23
|
"LICENSE-APACHE",
|
|
24
|
-
"NOTICE"
|
|
24
|
+
"NOTICE",
|
|
25
|
+
"CHANGELOG.md"
|
|
25
26
|
],
|
|
26
27
|
"private": false,
|
|
28
|
+
"homepage": "https://scrape.ly/docs/sdk-node",
|
|
27
29
|
"repository": {
|
|
28
30
|
"type": "git",
|
|
29
|
-
"url": "git+https://github.com/
|
|
31
|
+
"url": "git+https://github.com/scrapely-app/scrapely.git"
|
|
32
|
+
},
|
|
33
|
+
"bugs": {
|
|
34
|
+
"url": "https://github.com/scrapely-app/scrapely/issues"
|
|
30
35
|
},
|
|
31
36
|
"publishConfig": {
|
|
32
37
|
"registry": "https://registry.npmjs.org",
|
|
@@ -37,8 +42,12 @@
|
|
|
37
42
|
"test:charging": "npx tsx --test tests/charging-pending.test.ts",
|
|
38
43
|
"test:charge-limit-guard": "npx tsx --test tests/charge-limit-guard.test.ts",
|
|
39
44
|
"test:charge-multi-event": "npx tsx --test tests/charge-multi-event.test.ts",
|
|
45
|
+
"test:charge-multi-event-void": "npx tsx --test tests/charge-multi-event-void.test.ts",
|
|
46
|
+
"test:charge-edge-cases": "npx tsx --test tests/charge-edge-cases.test.ts",
|
|
47
|
+
"test:charge-pending-persist": "npx tsx --test tests/charge-pending-persist.test.ts",
|
|
40
48
|
"test:charge-platform-budget": "npx tsx --test tests/charge-platform-budget.test.ts",
|
|
41
49
|
"test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
|
|
50
|
+
"test:storage-path-segments": "npx tsx --test tests/storage-path-segments.test.ts",
|
|
42
51
|
"test:push-idempotency": "npx tsx --test tests/push-idempotency.test.ts",
|
|
43
52
|
"test:stdio-drain": "npx tsx --test tests/stdio-drain.test.ts",
|
|
44
53
|
"test:charging-client": "npx tsx --test tests/charging-client.test.ts",
|
|
@@ -46,6 +55,7 @@
|
|
|
46
55
|
"test:browser-pool": "npx tsx --test tests/patch-browser-pool.test.ts",
|
|
47
56
|
"test:browser-config": "npx tsx --test tests/browser-configuration.test.ts",
|
|
48
57
|
"build": "tsc",
|
|
58
|
+
"prepack": "npm run build",
|
|
49
59
|
"setup:gvisor": "bash scripts/setup-gvisor.sh",
|
|
50
60
|
"setup:server": "bash scripts/setup-server.sh",
|
|
51
61
|
"prod": "bash scripts/deploy-prod.sh",
|
|
@@ -54,7 +64,7 @@
|
|
|
54
64
|
"test:local-run": "npx tsx --test tests/local-run.test.ts",
|
|
55
65
|
"test:input-from-kv": "npx tsx --test tests/input-from-kv.test.ts",
|
|
56
66
|
"test:init-rollback": "npx tsx --test tests/init-rollback.test.ts",
|
|
57
|
-
"test:
|
|
67
|
+
"test:parity": "npx tsx --test tests/apify-parity.test.ts",
|
|
58
68
|
"test:run-control": "npx tsx --test tests/run-control.test.ts",
|
|
59
69
|
"test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts",
|
|
60
70
|
"test:terminal-status": "npx tsx --test tests/terminal-status-message.test.ts",
|
|
@@ -62,16 +72,17 @@
|
|
|
62
72
|
"test:proxy-input-aliases": "npx tsx --test tests/proxy-input-aliases.test.ts",
|
|
63
73
|
"test:charge-retries": "npx tsx --test tests/charge-retries.test.ts",
|
|
64
74
|
"test:solver": "npx tsx --test tests/solver.test.ts",
|
|
65
|
-
"test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts"
|
|
75
|
+
"test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts",
|
|
76
|
+
"test:v2-utils": "npx tsx --test tests/v2-utils.test.ts"
|
|
66
77
|
},
|
|
67
|
-
"author": "
|
|
78
|
+
"author": "TeraShift GmbH (Scrape.ly)",
|
|
68
79
|
"license": "ISC AND Apache-2.0",
|
|
69
80
|
"dependencies": {
|
|
70
81
|
"@aws-sdk/client-s3": "^3.700.0",
|
|
71
82
|
"@crawlee/browser": "^3.16.0",
|
|
72
83
|
"@crawlee/browser-pool": "^3.16.0",
|
|
73
84
|
"@crawlee/playwright": "^3.16.0",
|
|
74
|
-
"basic-ftp": "^6.
|
|
85
|
+
"basic-ftp": "^6.2.3",
|
|
75
86
|
"crawlee": "^3.16.0",
|
|
76
87
|
"dockerode": "^4.0.9",
|
|
77
88
|
"ioredis": "^6.0.0",
|
|
@@ -93,14 +104,14 @@
|
|
|
93
104
|
"typescript": "^5.7.0"
|
|
94
105
|
},
|
|
95
106
|
"overrides": {
|
|
96
|
-
"adm-zip": "^0.6.
|
|
107
|
+
"adm-zip": "^0.6.1",
|
|
97
108
|
"tar-fs": "2.1.5",
|
|
98
109
|
"yargs": "18.1.0",
|
|
99
|
-
"axios": "^1.
|
|
110
|
+
"axios": "^1.20.0",
|
|
100
111
|
"brace-expansion": "5.0.9",
|
|
101
112
|
"fast-xml-builder": "^1.2.1",
|
|
102
113
|
"file-type": "^21.3.3",
|
|
103
|
-
"follow-redirects": "^1.16.
|
|
114
|
+
"follow-redirects": "^1.16.1",
|
|
104
115
|
"@crawlee/core": {
|
|
105
116
|
"ow": "0.28.2"
|
|
106
117
|
},
|
|
@@ -116,6 +127,7 @@
|
|
|
116
127
|
"uuid": "^11.1.1",
|
|
117
128
|
"@inquirer/external-editor": "3.0.3",
|
|
118
129
|
"nanoid": "5.1.16",
|
|
119
|
-
"browserslist": "4.28.7"
|
|
130
|
+
"browserslist": "4.28.7",
|
|
131
|
+
"basic-ftp": "^6.2.3"
|
|
120
132
|
}
|
|
121
133
|
}
|