scrapely 1.2.32 → 1.2.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/NOTICE +12 -26
- package/README.md +1 -2
- package/dist/actor.d.ts +32 -25
- package/dist/actor.js +39 -23
- package/dist/charging.d.ts +49 -4
- package/dist/charging.js +293 -44
- package/dist/configuration.js +2 -2
- package/dist/index.js +1 -1
- package/dist/node-wrapper.js +1 -1
- package/dist/patch-browser-pool.js +7 -3
- package/dist/proxy-configuration.d.ts +11 -11
- package/dist/proxy-configuration.js +1 -1
- package/dist/proxy-input-aliases.d.ts +1 -1
- package/dist/proxy-input-aliases.js +1 -1
- package/dist/request-list.d.ts +2 -2
- package/dist/request-list.js +1 -1
- package/dist/scrapely-proxy-configuration.d.ts +15 -16
- package/dist/scrapely-proxy-configuration.js +27 -31
- package/dist/solver.js +1 -1
- package/dist/storage/api-storage-client.d.ts +4 -5
- package/dist/storage/api-storage-client.js +17 -15
- package/dist/storage/billing-tracker.d.ts +1 -1
- package/dist/storage/billing-tracker.js +1 -1
- package/dist/storage/key-value-store-client.d.ts +0 -3
- package/dist/storage/key-value-store-client.js +2 -5
- package/dist/storage/kv-store-collection-client.js +6 -2
- package/dist/storage/kvs-iteration.d.ts +1 -1
- package/dist/storage/kvs-iteration.js +1 -1
- package/dist/storage/request-queue-client.d.ts +1 -1
- package/dist/storage/request-queue-client.js +5 -5
- package/dist/storage/rq-lock-script.d.ts +2 -2
- package/dist/storage/rq-lock-script.js +2 -2
- package/dist/storage/run-metadata.js +1 -1
- package/dist/storage/ttl-manager.d.ts +7 -6
- package/dist/storage/ttl-manager.js +5 -4
- package/dist/storage-identifiers.d.ts +1 -2
- package/dist/storage-identifiers.js +1 -2
- package/dist/utils.d.ts +41 -8
- package/dist/utils.js +49 -3
- package/package.json +22 -11
- package/dist/runner/docker-manager.d.ts +0 -88
- package/dist/runner/docker-manager.js +0 -283
- package/dist/runner/index.d.ts +0 -2
- package/dist/runner/index.js +0 -7
- package/dist/runner/timeout-manager.d.ts +0 -43
- package/dist/runner/timeout-manager.js +0 -90
|
@@ -66,7 +66,7 @@ class DragonflyRequestQueueClient {
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
/**
|
|
69
|
-
* Track clientKey access for hadMultipleClients
|
|
69
|
+
* Track clientKey access for hadMultipleClients.
|
|
70
70
|
* Returns the current hadMultipleClients flag after update.
|
|
71
71
|
*/
|
|
72
72
|
async trackClientKey(clientKey) {
|
|
@@ -333,11 +333,11 @@ class DragonflyRequestQueueClient {
|
|
|
333
333
|
async listRequests(options) {
|
|
334
334
|
const limit = Math.min(options?.limit ?? 100, 10000);
|
|
335
335
|
const now = Date.now();
|
|
336
|
-
// Validate that filter is not used with deprecated exclusiveStartId
|
|
336
|
+
// Validate that filter is not used with deprecated exclusiveStartId
|
|
337
337
|
if (options?.filter && options.filter.length > 0 && options?.exclusiveStartId) {
|
|
338
338
|
throw new Error('filter parameter is not compatible with deprecated exclusiveStartId parameter');
|
|
339
339
|
}
|
|
340
|
-
// Track clientKey for hadMultipleClients
|
|
340
|
+
// Track clientKey for hadMultipleClients
|
|
341
341
|
if (options?.clientKey) {
|
|
342
342
|
const existingClientKey = await this.client.redis.hget(this.infoKey, 'clientKey');
|
|
343
343
|
if (existingClientKey && existingClientKey !== options.clientKey) {
|
|
@@ -348,7 +348,7 @@ class DragonflyRequestQueueClient {
|
|
|
348
348
|
}
|
|
349
349
|
}
|
|
350
350
|
else {
|
|
351
|
-
// No clientKey provided - treat as new client
|
|
351
|
+
// No clientKey provided - treat as new client
|
|
352
352
|
const existingClientKey = await this.client.redis.hget(this.infoKey, 'clientKey');
|
|
353
353
|
if (existingClientKey) {
|
|
354
354
|
await this.client.redis.hset(this.infoKey, 'hadMultipleClients', '1');
|
|
@@ -456,7 +456,7 @@ class DragonflyRequestQueueClient {
|
|
|
456
456
|
lockSecs: options.lockSecs,
|
|
457
457
|
items,
|
|
458
458
|
// From the atomic lock script: active locks held by ANY client
|
|
459
|
-
// (including this call's own)
|
|
459
|
+
// (including this call's own).
|
|
460
460
|
queueHasLockedRequests: activeLockCount > 0,
|
|
461
461
|
};
|
|
462
462
|
}
|
|
@@ -5,7 +5,7 @@ import type Redis from 'ioredis';
|
|
|
5
5
|
* which ioredis delivers as [string[], number]. The ZCARD is taken after the
|
|
6
6
|
* script's expired-lock purge and after this call's own locks are added, so it
|
|
7
7
|
* equals the number of active locks held by ANY client — exactly what the
|
|
8
|
-
*
|
|
8
|
+
* `queueHasLockedRequests` flag means ("locked by any client, either
|
|
9
9
|
* the one calling the endpoint or a different one"). Callers get the flag from
|
|
10
10
|
* this same atomic reply instead of re-reading the whole locked + pending sets.
|
|
11
11
|
*/
|
|
@@ -43,7 +43,7 @@ export declare function matchesLockOwner(owner: string, clientKey: string, runId
|
|
|
43
43
|
* Returns the number of unlocked requests. One round-trip, no race window
|
|
44
44
|
* between ZREM and HDEL, and the whole owners hash never leaves the server.
|
|
45
45
|
*/
|
|
46
|
-
export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n local requestId = flat[i]\n local owner = flat[i + 1]\n --
|
|
46
|
+
export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n local requestId = flat[i]\n local owner = flat[i + 1]\n -- In an Actor run, unlock both (1) requests locked by this\n -- specific run (clientKey:runId) and (2) requests locked by the same\n -- clientKey without run context (clientKey), but NOT requests locked by the\n -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n if owner == expectedOwner or owner == clientKey\n or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then\n redis.call('ZREM', lockedKey, requestId)\n redis.call('HDEL', lockOwnersKey, requestId)\n if forefront == '1' then\n redis.call('ZADD', pendingKey, 0, requestId)\n end\n unlockedCount = unlockedCount + 1\n end\nend\n\nreturn unlockedCount\n";
|
|
47
47
|
export interface UnlockRequestsKeys {
|
|
48
48
|
lockedKey: string;
|
|
49
49
|
lockOwnersKey: string;
|
|
@@ -10,7 +10,7 @@ exports.unlockRequestsAtomic = unlockRequestsAtomic;
|
|
|
10
10
|
* which ioredis delivers as [string[], number]. The ZCARD is taken after the
|
|
11
11
|
* script's expired-lock purge and after this call's own locks are added, so it
|
|
12
12
|
* equals the number of active locks held by ANY client — exactly what the
|
|
13
|
-
*
|
|
13
|
+
* `queueHasLockedRequests` flag means ("locked by any client, either
|
|
14
14
|
* the one calling the endpoint or a different one"). Callers get the flag from
|
|
15
15
|
* this same atomic reply instead of re-reading the whole locked + pending sets.
|
|
16
16
|
*/
|
|
@@ -104,7 +104,7 @@ local unlockedCount = 0
|
|
|
104
104
|
for i = 1, #flat, 2 do
|
|
105
105
|
local requestId = flat[i]
|
|
106
106
|
local owner = flat[i + 1]
|
|
107
|
-
--
|
|
107
|
+
-- In an Actor run, unlock both (1) requests locked by this
|
|
108
108
|
-- specific run (clientKey:runId) and (2) requests locked by the same
|
|
109
109
|
-- clientKey without run context (clientKey), but NOT requests locked by the
|
|
110
110
|
-- same clientKey in DIFFERENT runs (clientKey:otherRunId).
|
|
@@ -4,7 +4,7 @@ exports.RunMetadataManager = void 0;
|
|
|
4
4
|
const billing_tracker_js_1 = require("./billing-tracker.js");
|
|
5
5
|
/**
|
|
6
6
|
* Number of most-recent runs per user that are retained indefinitely (no TTL).
|
|
7
|
-
*
|
|
7
|
+
* The 10 latest runs are always accessible regardless of plan.
|
|
8
8
|
* Older runs beyond this window fall back to the standard plan-based TTL.
|
|
9
9
|
*/
|
|
10
10
|
const KEEP_N_RUNS = 10;
|
|
@@ -3,13 +3,13 @@ import type Redis from 'ioredis';
|
|
|
3
3
|
* TTL configuration in seconds
|
|
4
4
|
*/
|
|
5
5
|
export interface TtlConfig {
|
|
6
|
-
/** TTL for unnamed datasets (free users) - default
|
|
6
|
+
/** TTL for unnamed datasets (free users) - default 48 hours (Free plan) */
|
|
7
7
|
datasetFree: number;
|
|
8
|
-
/** TTL for unnamed datasets (paid users) - default 31 days */
|
|
8
|
+
/** TTL for unnamed datasets (paid users) - default 31 days (Starter, the shortest paid plan) */
|
|
9
9
|
datasetPaid: number;
|
|
10
|
-
/** TTL for unnamed KV stores (free users) - default
|
|
10
|
+
/** TTL for unnamed KV stores (free users) - default 48 hours (Free plan) */
|
|
11
11
|
kvFree: number;
|
|
12
|
-
/** TTL for unnamed KV stores (paid users) - default 31 days */
|
|
12
|
+
/** TTL for unnamed KV stores (paid users) - default 31 days (Starter, the shortest paid plan) */
|
|
13
13
|
kvPaid: number;
|
|
14
14
|
/** Fallback TTL for request queues (safety net for orphaned RQs) - default 24 hours */
|
|
15
15
|
requestQueueFallback: number;
|
|
@@ -21,7 +21,7 @@ export interface TtlConfig {
|
|
|
21
21
|
logMaxSize: number;
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
24
|
-
* Default TTL configuration
|
|
24
|
+
* Default TTL configuration
|
|
25
25
|
*/
|
|
26
26
|
export declare const DEFAULT_TTL_CONFIG: TtlConfig;
|
|
27
27
|
export type StorageType = 'dataset' | 'kv' | 'rq' | 'logs';
|
|
@@ -31,7 +31,8 @@ export type UserPlan = 'free' | 'paid';
|
|
|
31
31
|
*
|
|
32
32
|
* Retention policy:
|
|
33
33
|
* - Named storage: Indefinite (billed), no TTL set
|
|
34
|
-
* - Unnamed datasets/KV stores:
|
|
34
|
+
* - Unnamed datasets/KV stores: the plan's retention (Free 48 h, Starter 31 d,
|
|
35
|
+
* Scale 60 d, Business 90 d); the defaults here are Free and Starter
|
|
35
36
|
* - Request queues: Explicitly deleted after run, 24h fallback TTL for orphaned RQs
|
|
36
37
|
* - Logs: Short TTL in Redis, persisted to S3
|
|
37
38
|
*/
|
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.TtlManager = exports.DEFAULT_TTL_CONFIG = void 0;
|
|
4
4
|
/**
|
|
5
|
-
* Default TTL configuration
|
|
5
|
+
* Default TTL configuration
|
|
6
6
|
*/
|
|
7
7
|
exports.DEFAULT_TTL_CONFIG = {
|
|
8
|
-
datasetFree:
|
|
8
|
+
datasetFree: 48 * 60 * 60, // 48 hours
|
|
9
9
|
datasetPaid: 31 * 24 * 60 * 60, // 31 days
|
|
10
|
-
kvFree:
|
|
10
|
+
kvFree: 48 * 60 * 60, // 48 hours
|
|
11
11
|
kvPaid: 31 * 24 * 60 * 60, // 31 days
|
|
12
12
|
requestQueueFallback: 24 * 60 * 60, // 24 hours
|
|
13
13
|
logsFree: 12 * 60 * 60, // 12 hours
|
|
@@ -19,7 +19,8 @@ exports.DEFAULT_TTL_CONFIG = {
|
|
|
19
19
|
*
|
|
20
20
|
* Retention policy:
|
|
21
21
|
* - Named storage: Indefinite (billed), no TTL set
|
|
22
|
-
* - Unnamed datasets/KV stores:
|
|
22
|
+
* - Unnamed datasets/KV stores: the plan's retention (Free 48 h, Starter 31 d,
|
|
23
|
+
* Scale 60 d, Business 90 d); the defaults here are Free and Starter
|
|
23
24
|
* - Request queues: Explicitly deleted after run, 24h fallback TTL for orphaned RQs
|
|
24
25
|
* - Logs: Short TTL in Redis, persisted to S3
|
|
25
26
|
*/
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Storage identifier types
|
|
3
|
-
* them from 'apify' still compile after the apify → scrapely rewrite).
|
|
2
|
+
* Storage identifier types, kept under these names for backward compatibility.
|
|
4
3
|
*/
|
|
5
4
|
/** Identifies a storage by its alias from the Actor's schema storages. */
|
|
6
5
|
export interface StorageAlias {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* Storage identifier types
|
|
4
|
-
* them from 'apify' still compile after the apify → scrapely rewrite).
|
|
3
|
+
* Storage identifier types, kept under these names for backward compatibility.
|
|
5
4
|
*/
|
|
6
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
package/dist/utils.d.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
* These adapt the
|
|
2
|
+
* v2-style utility functions for scrapely.
|
|
3
|
+
* These adapt the v2-style call signatures to Crawlee v3 internally.
|
|
4
4
|
*/
|
|
5
5
|
import type { CheerioAPI } from 'cheerio';
|
|
6
6
|
import type { RequestQueue } from 'crawlee';
|
|
7
7
|
/**
|
|
8
|
-
* Options for enqueueLinks (
|
|
8
|
+
* Options for enqueueLinks (v2-style API).
|
|
9
9
|
*/
|
|
10
10
|
export interface EnqueueLinksOptions {
|
|
11
|
-
/** Cheerio instance ($) - REQUIRED
|
|
11
|
+
/** Cheerio instance ($) - REQUIRED */
|
|
12
12
|
$: CheerioAPI;
|
|
13
|
-
/** RequestQueue instance - REQUIRED
|
|
13
|
+
/** RequestQueue instance - REQUIRED */
|
|
14
14
|
requestQueue: RequestQueue;
|
|
15
15
|
/** CSS selector to find links. Default: 'a[href]' */
|
|
16
16
|
selector?: string;
|
|
@@ -30,13 +30,13 @@ export interface EnqueueLinksOptions {
|
|
|
30
30
|
/**
|
|
31
31
|
* Enqueues links found on the page using a CSS selector.
|
|
32
32
|
*
|
|
33
|
-
* This
|
|
33
|
+
* This v2-style implementation adapts to Crawlee v3 internally.
|
|
34
34
|
* It extracts URLs from the page using cheerio and adds them to the request queue.
|
|
35
35
|
*
|
|
36
36
|
* @example
|
|
37
37
|
* ```typescript
|
|
38
38
|
* // Inside a CheerioCrawler requestHandler:
|
|
39
|
-
* await
|
|
39
|
+
* await Actor.utils.enqueueLinks({
|
|
40
40
|
* $,
|
|
41
41
|
* requestQueue,
|
|
42
42
|
* selector: 'a[href*="/app/"]',
|
|
@@ -45,10 +45,43 @@ export interface EnqueueLinksOptions {
|
|
|
45
45
|
* ```
|
|
46
46
|
*/
|
|
47
47
|
export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<void>;
|
|
48
|
+
/** Options of the v2-style `utils.requestAsBrowser`; other got options pass through. */
|
|
49
|
+
export interface RequestAsBrowserOptions extends Record<string, any> {
|
|
50
|
+
url: string;
|
|
51
|
+
method?: string;
|
|
52
|
+
headers?: Record<string, string>;
|
|
53
|
+
payload?: string | Buffer;
|
|
54
|
+
proxyUrl?: string;
|
|
55
|
+
languageCode?: string;
|
|
56
|
+
countryCode?: string;
|
|
57
|
+
isMobile?: boolean;
|
|
58
|
+
json?: boolean;
|
|
59
|
+
ignoreSslErrors?: boolean;
|
|
60
|
+
useInsecureHttpParser?: boolean;
|
|
61
|
+
abortFunction?: unknown;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* v2-style `utils.requestAsBrowser`: an HTTP request with browser-like headers,
|
|
65
|
+
* here through got-scraping (what v2 itself switched to). Resolves to the got
|
|
66
|
+
* response (`body`, `statusCode`, `headers`, `url`).
|
|
67
|
+
*/
|
|
68
|
+
export declare function requestAsBrowser(options: RequestAsBrowserOptions): Promise<any>;
|
|
69
|
+
/** v2-style `utils.getRandomUserAgent`: a realistic desktop browser User-Agent. */
|
|
70
|
+
export declare function getRandomUserAgent(): string;
|
|
48
71
|
/**
|
|
49
|
-
* Utility namespace providing
|
|
72
|
+
* Utility namespace providing v2-style helper functions.
|
|
50
73
|
* These adapt to Crawlee v3 APIs internally.
|
|
51
74
|
*/
|
|
52
75
|
export declare const utils: {
|
|
53
76
|
enqueueLinks: typeof enqueueLinks;
|
|
77
|
+
requestAsBrowser: typeof requestAsBrowser;
|
|
78
|
+
getRandomUserAgent: typeof getRandomUserAgent;
|
|
79
|
+
sleep: (millis?: number) => any;
|
|
80
|
+
downloadListOfUrls: (options: any) => any;
|
|
81
|
+
extractUrls: (options: any) => any;
|
|
82
|
+
htmlToText: (html: any) => any;
|
|
83
|
+
isDocker: (forceReset?: boolean) => any;
|
|
84
|
+
readonly log: any;
|
|
85
|
+
readonly puppeteer: any;
|
|
86
|
+
readonly playwright: any;
|
|
54
87
|
};
|
package/dist/utils.js
CHANGED
|
@@ -2,16 +2,18 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.utils = void 0;
|
|
4
4
|
exports.enqueueLinks = enqueueLinks;
|
|
5
|
+
exports.requestAsBrowser = requestAsBrowser;
|
|
6
|
+
exports.getRandomUserAgent = getRandomUserAgent;
|
|
5
7
|
/**
|
|
6
8
|
* Enqueues links found on the page using a CSS selector.
|
|
7
9
|
*
|
|
8
|
-
* This
|
|
10
|
+
* This v2-style implementation adapts to Crawlee v3 internally.
|
|
9
11
|
* It extracts URLs from the page using cheerio and adds them to the request queue.
|
|
10
12
|
*
|
|
11
13
|
* @example
|
|
12
14
|
* ```typescript
|
|
13
15
|
* // Inside a CheerioCrawler requestHandler:
|
|
14
|
-
* await
|
|
16
|
+
* await Actor.utils.enqueueLinks({
|
|
15
17
|
* $,
|
|
16
18
|
* requestQueue,
|
|
17
19
|
* selector: 'a[href*="/app/"]',
|
|
@@ -55,9 +57,53 @@ async function enqueueLinks(options) {
|
|
|
55
57
|
}
|
|
56
58
|
}
|
|
57
59
|
/**
|
|
58
|
-
*
|
|
60
|
+
* v2-style `utils.requestAsBrowser`: an HTTP request with browser-like headers,
|
|
61
|
+
* here through got-scraping (what v2 itself switched to). Resolves to the got
|
|
62
|
+
* response (`body`, `statusCode`, `headers`, `url`).
|
|
63
|
+
*/
|
|
64
|
+
async function requestAsBrowser(options) {
|
|
65
|
+
const { payload, languageCode, countryCode, isMobile, json, ignoreSslErrors, abortFunction: _abortFunction, ...gotOptions } = options;
|
|
66
|
+
const { gotScraping } = await import('@crawlee/utils');
|
|
67
|
+
const locale = languageCode ? `${languageCode}${countryCode ? `-${countryCode}` : ''}` : undefined;
|
|
68
|
+
return gotScraping({
|
|
69
|
+
...(payload !== undefined ? { body: payload } : {}),
|
|
70
|
+
...(json ? { responseType: 'json' } : {}),
|
|
71
|
+
...(ignoreSslErrors ? { https: { rejectUnauthorized: false } } : {}),
|
|
72
|
+
headerGeneratorOptions: {
|
|
73
|
+
...(locale ? { locales: [locale] } : {}),
|
|
74
|
+
...(isMobile ? { devices: ['mobile'] } : {}),
|
|
75
|
+
},
|
|
76
|
+
...gotOptions,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
let userAgents;
|
|
80
|
+
/** v2-style `utils.getRandomUserAgent`: a realistic desktop browser User-Agent. */
|
|
81
|
+
function getRandomUserAgent() {
|
|
82
|
+
if (!userAgents) {
|
|
83
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
84
|
+
const { HeaderGenerator } = require('header-generator');
|
|
85
|
+
userAgents = new HeaderGenerator({ devices: ['desktop'] });
|
|
86
|
+
}
|
|
87
|
+
const headers = userAgents.getHeaders();
|
|
88
|
+
return headers['user-agent'] ?? headers['User-Agent'];
|
|
89
|
+
}
|
|
90
|
+
// Loaded on first use: the SDK entry must not load Crawlee (see lazy-exports.ts).
|
|
91
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
92
|
+
const crawleeUtils = () => require('@crawlee/utils');
|
|
93
|
+
/**
|
|
94
|
+
* Utility namespace providing v2-style helper functions.
|
|
59
95
|
* These adapt to Crawlee v3 APIs internally.
|
|
60
96
|
*/
|
|
61
97
|
exports.utils = {
|
|
62
98
|
enqueueLinks,
|
|
99
|
+
requestAsBrowser,
|
|
100
|
+
getRandomUserAgent,
|
|
101
|
+
sleep: (millis) => crawleeUtils().sleep(millis),
|
|
102
|
+
downloadListOfUrls: (options) => crawleeUtils().downloadListOfUrls(options),
|
|
103
|
+
extractUrls: (options) => crawleeUtils().extractUrls(options),
|
|
104
|
+
htmlToText: (html) => crawleeUtils().htmlToText(html),
|
|
105
|
+
isDocker: (forceReset) => crawleeUtils().isDocker(forceReset),
|
|
106
|
+
get log() { return require('@apify/log').default; },
|
|
107
|
+
get puppeteer() { return require('@crawlee/puppeteer').puppeteerUtils; },
|
|
108
|
+
get playwright() { return require('@crawlee/playwright').playwrightUtils; },
|
|
63
109
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scrapely",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.34",
|
|
4
4
|
"description": "Scrapely SDK for building JavaScript and TypeScript Actors",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -21,12 +21,17 @@
|
|
|
21
21
|
"dist",
|
|
22
22
|
"LICENSE",
|
|
23
23
|
"LICENSE-APACHE",
|
|
24
|
-
"NOTICE"
|
|
24
|
+
"NOTICE",
|
|
25
|
+
"CHANGELOG.md"
|
|
25
26
|
],
|
|
26
27
|
"private": false,
|
|
28
|
+
"homepage": "https://scrape.ly/docs/sdk-node",
|
|
27
29
|
"repository": {
|
|
28
30
|
"type": "git",
|
|
29
|
-
"url": "git+https://github.com/
|
|
31
|
+
"url": "git+https://github.com/scrapely-app/scrapely.git"
|
|
32
|
+
},
|
|
33
|
+
"bugs": {
|
|
34
|
+
"url": "https://github.com/scrapely-app/scrapely/issues"
|
|
30
35
|
},
|
|
31
36
|
"publishConfig": {
|
|
32
37
|
"registry": "https://registry.npmjs.org",
|
|
@@ -37,6 +42,9 @@
|
|
|
37
42
|
"test:charging": "npx tsx --test tests/charging-pending.test.ts",
|
|
38
43
|
"test:charge-limit-guard": "npx tsx --test tests/charge-limit-guard.test.ts",
|
|
39
44
|
"test:charge-multi-event": "npx tsx --test tests/charge-multi-event.test.ts",
|
|
45
|
+
"test:charge-multi-event-void": "npx tsx --test tests/charge-multi-event-void.test.ts",
|
|
46
|
+
"test:charge-edge-cases": "npx tsx --test tests/charge-edge-cases.test.ts",
|
|
47
|
+
"test:charge-pending-persist": "npx tsx --test tests/charge-pending-persist.test.ts",
|
|
40
48
|
"test:charge-platform-budget": "npx tsx --test tests/charge-platform-budget.test.ts",
|
|
41
49
|
"test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
|
|
42
50
|
"test:push-idempotency": "npx tsx --test tests/push-idempotency.test.ts",
|
|
@@ -46,6 +54,7 @@
|
|
|
46
54
|
"test:browser-pool": "npx tsx --test tests/patch-browser-pool.test.ts",
|
|
47
55
|
"test:browser-config": "npx tsx --test tests/browser-configuration.test.ts",
|
|
48
56
|
"build": "tsc",
|
|
57
|
+
"prepack": "npm run build",
|
|
49
58
|
"setup:gvisor": "bash scripts/setup-gvisor.sh",
|
|
50
59
|
"setup:server": "bash scripts/setup-server.sh",
|
|
51
60
|
"prod": "bash scripts/deploy-prod.sh",
|
|
@@ -54,7 +63,7 @@
|
|
|
54
63
|
"test:local-run": "npx tsx --test tests/local-run.test.ts",
|
|
55
64
|
"test:input-from-kv": "npx tsx --test tests/input-from-kv.test.ts",
|
|
56
65
|
"test:init-rollback": "npx tsx --test tests/init-rollback.test.ts",
|
|
57
|
-
"test:
|
|
66
|
+
"test:parity": "npx tsx --test tests/apify-parity.test.ts",
|
|
58
67
|
"test:run-control": "npx tsx --test tests/run-control.test.ts",
|
|
59
68
|
"test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts",
|
|
60
69
|
"test:terminal-status": "npx tsx --test tests/terminal-status-message.test.ts",
|
|
@@ -62,16 +71,17 @@
|
|
|
62
71
|
"test:proxy-input-aliases": "npx tsx --test tests/proxy-input-aliases.test.ts",
|
|
63
72
|
"test:charge-retries": "npx tsx --test tests/charge-retries.test.ts",
|
|
64
73
|
"test:solver": "npx tsx --test tests/solver.test.ts",
|
|
65
|
-
"test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts"
|
|
74
|
+
"test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts",
|
|
75
|
+
"test:v2-utils": "npx tsx --test tests/v2-utils.test.ts"
|
|
66
76
|
},
|
|
67
|
-
"author": "
|
|
77
|
+
"author": "TeraShift GmbH (Scrape.ly)",
|
|
68
78
|
"license": "ISC AND Apache-2.0",
|
|
69
79
|
"dependencies": {
|
|
70
80
|
"@aws-sdk/client-s3": "^3.700.0",
|
|
71
81
|
"@crawlee/browser": "^3.16.0",
|
|
72
82
|
"@crawlee/browser-pool": "^3.16.0",
|
|
73
83
|
"@crawlee/playwright": "^3.16.0",
|
|
74
|
-
"basic-ftp": "^6.
|
|
84
|
+
"basic-ftp": "^6.2.3",
|
|
75
85
|
"crawlee": "^3.16.0",
|
|
76
86
|
"dockerode": "^4.0.9",
|
|
77
87
|
"ioredis": "^6.0.0",
|
|
@@ -93,14 +103,14 @@
|
|
|
93
103
|
"typescript": "^5.7.0"
|
|
94
104
|
},
|
|
95
105
|
"overrides": {
|
|
96
|
-
"adm-zip": "^0.6.
|
|
106
|
+
"adm-zip": "^0.6.1",
|
|
97
107
|
"tar-fs": "2.1.5",
|
|
98
108
|
"yargs": "18.1.0",
|
|
99
|
-
"axios": "^1.
|
|
109
|
+
"axios": "^1.20.0",
|
|
100
110
|
"brace-expansion": "5.0.9",
|
|
101
111
|
"fast-xml-builder": "^1.2.1",
|
|
102
112
|
"file-type": "^21.3.3",
|
|
103
|
-
"follow-redirects": "^1.16.
|
|
113
|
+
"follow-redirects": "^1.16.1",
|
|
104
114
|
"@crawlee/core": {
|
|
105
115
|
"ow": "0.28.2"
|
|
106
116
|
},
|
|
@@ -116,6 +126,7 @@
|
|
|
116
126
|
"uuid": "^11.1.1",
|
|
117
127
|
"@inquirer/external-editor": "3.0.3",
|
|
118
128
|
"nanoid": "5.1.16",
|
|
119
|
-
"browserslist": "4.28.7"
|
|
129
|
+
"browserslist": "4.28.7",
|
|
130
|
+
"basic-ftp": "^6.2.3"
|
|
120
131
|
}
|
|
121
132
|
}
|
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
import Docker from 'dockerode';
|
|
2
|
-
import { TimeoutManager } from './timeout-manager.js';
|
|
3
|
-
import { DragonflyClient } from '../storage/dragonfly-client.js';
|
|
4
|
-
import type { UserPlan } from '../storage/ttl-manager.js';
|
|
5
|
-
export interface RunOptions {
|
|
6
|
-
/** Path to the actor package directory */
|
|
7
|
-
packagePath: string;
|
|
8
|
-
/** Path to the .env file */
|
|
9
|
-
envPath?: string;
|
|
10
|
-
/** Environment variables to pass to container */
|
|
11
|
-
env?: Record<string, string>;
|
|
12
|
-
/** Maximum memory in MB (default: from manifest or 512) */
|
|
13
|
-
memoryMb?: number;
|
|
14
|
-
/** Maximum timeout in seconds (default: from manifest or 1800) */
|
|
15
|
-
timeout?: number;
|
|
16
|
-
/** Run ID for tracking */
|
|
17
|
-
runId?: string;
|
|
18
|
-
/** User ID */
|
|
19
|
-
userId?: string;
|
|
20
|
-
/** User plan for TTL */
|
|
21
|
-
userPlan?: UserPlan;
|
|
22
|
-
/** Dragonfly client for storage cleanup */
|
|
23
|
-
dragonflyClient?: DragonflyClient;
|
|
24
|
-
}
|
|
25
|
-
export interface RunResult {
|
|
26
|
-
containerId: string;
|
|
27
|
-
exitCode: number;
|
|
28
|
-
logs: string;
|
|
29
|
-
status: 'success' | 'failed' | 'timed_out' | 'killed';
|
|
30
|
-
duration: number;
|
|
31
|
-
/** Storage IDs that were cleaned up */
|
|
32
|
-
cleanedUp?: {
|
|
33
|
-
requestQueueDeleted: boolean;
|
|
34
|
-
logsUploadedToS3: boolean;
|
|
35
|
-
};
|
|
36
|
-
}
|
|
37
|
-
export declare class DockerManager {
|
|
38
|
-
private docker;
|
|
39
|
-
private timeoutManager;
|
|
40
|
-
private imageName;
|
|
41
|
-
constructor(options?: {
|
|
42
|
-
socketPath?: string;
|
|
43
|
-
imageName?: string;
|
|
44
|
-
});
|
|
45
|
-
/**
|
|
46
|
-
* Run an actor package in a container
|
|
47
|
-
*/
|
|
48
|
-
run(options: RunOptions): Promise<RunResult>;
|
|
49
|
-
/**
|
|
50
|
-
* Cleanup storage after run completes.
|
|
51
|
-
* - Deletes request queue immediately (no longer needed)
|
|
52
|
-
* - Uploads logs to S3 for long-term storage
|
|
53
|
-
* - Updates run metadata with final status
|
|
54
|
-
*/
|
|
55
|
-
private cleanupAfterRun;
|
|
56
|
-
/**
|
|
57
|
-
* Stop a container (SIGTERM)
|
|
58
|
-
*/
|
|
59
|
-
stop(containerId: string, timeoutSeconds?: number): Promise<void>;
|
|
60
|
-
/**
|
|
61
|
-
* Kill a container (SIGKILL)
|
|
62
|
-
*/
|
|
63
|
-
kill(containerId: string): Promise<void>;
|
|
64
|
-
/**
|
|
65
|
-
* Get logs from a container
|
|
66
|
-
*/
|
|
67
|
-
getLogs(containerId: string): Promise<string>;
|
|
68
|
-
/**
|
|
69
|
-
* Get container status
|
|
70
|
-
*/
|
|
71
|
-
getStatus(containerId: string): Promise<string | null>;
|
|
72
|
-
/**
|
|
73
|
-
* List all scrapely containers
|
|
74
|
-
*/
|
|
75
|
-
listContainers(): Promise<Docker.ContainerInfo[]>;
|
|
76
|
-
/**
|
|
77
|
-
* Get the timeout manager
|
|
78
|
-
*/
|
|
79
|
-
getTimeoutManager(): TimeoutManager;
|
|
80
|
-
/**
|
|
81
|
-
* Track internal data transfer from container network stats.
|
|
82
|
-
* Following Apify's model:
|
|
83
|
-
* - Ingress (rx_bytes) is FREE - scraped content coming in
|
|
84
|
-
* - Internal transfer is tracked separately in storage clients
|
|
85
|
-
* - External egress is tracked in API layer when users fetch results
|
|
86
|
-
*/
|
|
87
|
-
private trackDataTransfer;
|
|
88
|
-
}
|