scrapely 1.2.32 → 1.2.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +7 -0
  2. package/NOTICE +12 -26
  3. package/README.md +1 -2
  4. package/dist/actor.d.ts +32 -25
  5. package/dist/actor.js +39 -23
  6. package/dist/charging.d.ts +49 -4
  7. package/dist/charging.js +293 -44
  8. package/dist/configuration.js +2 -2
  9. package/dist/index.js +1 -1
  10. package/dist/node-wrapper.js +1 -1
  11. package/dist/patch-browser-pool.js +7 -3
  12. package/dist/proxy-configuration.d.ts +11 -11
  13. package/dist/proxy-configuration.js +1 -1
  14. package/dist/proxy-input-aliases.d.ts +1 -1
  15. package/dist/proxy-input-aliases.js +1 -1
  16. package/dist/request-list.d.ts +2 -2
  17. package/dist/request-list.js +1 -1
  18. package/dist/scrapely-proxy-configuration.d.ts +15 -16
  19. package/dist/scrapely-proxy-configuration.js +27 -31
  20. package/dist/solver.js +1 -1
  21. package/dist/storage/api-storage-client.d.ts +4 -5
  22. package/dist/storage/api-storage-client.js +17 -15
  23. package/dist/storage/billing-tracker.d.ts +1 -1
  24. package/dist/storage/billing-tracker.js +1 -1
  25. package/dist/storage/key-value-store-client.d.ts +0 -3
  26. package/dist/storage/key-value-store-client.js +2 -5
  27. package/dist/storage/kv-store-collection-client.js +6 -2
  28. package/dist/storage/kvs-iteration.d.ts +1 -1
  29. package/dist/storage/kvs-iteration.js +1 -1
  30. package/dist/storage/request-queue-client.d.ts +1 -1
  31. package/dist/storage/request-queue-client.js +5 -5
  32. package/dist/storage/rq-lock-script.d.ts +2 -2
  33. package/dist/storage/rq-lock-script.js +2 -2
  34. package/dist/storage/run-metadata.js +1 -1
  35. package/dist/storage/ttl-manager.d.ts +7 -6
  36. package/dist/storage/ttl-manager.js +5 -4
  37. package/dist/storage-identifiers.d.ts +1 -2
  38. package/dist/storage-identifiers.js +1 -2
  39. package/dist/utils.d.ts +41 -8
  40. package/dist/utils.js +49 -3
  41. package/package.json +22 -11
  42. package/dist/runner/docker-manager.d.ts +0 -88
  43. package/dist/runner/docker-manager.js +0 -283
  44. package/dist/runner/index.d.ts +0 -2
  45. package/dist/runner/index.js +0 -7
  46. package/dist/runner/timeout-manager.d.ts +0 -43
  47. package/dist/runner/timeout-manager.js +0 -90
@@ -66,7 +66,7 @@ class DragonflyRequestQueueClient {
66
66
  }
67
67
  }
68
68
  /**
69
- * Track clientKey access for hadMultipleClients (per Apify spec).
69
+ * Track clientKey access for hadMultipleClients.
70
70
  * Returns the current hadMultipleClients flag after update.
71
71
  */
72
72
  async trackClientKey(clientKey) {
@@ -333,11 +333,11 @@ class DragonflyRequestQueueClient {
333
333
  async listRequests(options) {
334
334
  const limit = Math.min(options?.limit ?? 100, 10000);
335
335
  const now = Date.now();
336
- // Validate that filter is not used with deprecated exclusiveStartId (per Apify spec)
336
+ // Validate that filter is not used with deprecated exclusiveStartId
337
337
  if (options?.filter && options.filter.length > 0 && options?.exclusiveStartId) {
338
338
  throw new Error('filter parameter is not compatible with deprecated exclusiveStartId parameter');
339
339
  }
340
- // Track clientKey for hadMultipleClients (per Apify spec)
340
+ // Track clientKey for hadMultipleClients
341
341
  if (options?.clientKey) {
342
342
  const existingClientKey = await this.client.redis.hget(this.infoKey, 'clientKey');
343
343
  if (existingClientKey && existingClientKey !== options.clientKey) {
@@ -348,7 +348,7 @@ class DragonflyRequestQueueClient {
348
348
  }
349
349
  }
350
350
  else {
351
- // No clientKey provided - treat as new client per Apify spec
351
+ // No clientKey provided - treat as new client
352
352
  const existingClientKey = await this.client.redis.hget(this.infoKey, 'clientKey');
353
353
  if (existingClientKey) {
354
354
  await this.client.redis.hset(this.infoKey, 'hadMultipleClients', '1');
@@ -456,7 +456,7 @@ class DragonflyRequestQueueClient {
456
456
  lockSecs: options.lockSecs,
457
457
  items,
458
458
  // From the atomic lock script: active locks held by ANY client
459
- // (including this call's own), per the Apify spec.
459
+ // (including this call's own).
460
460
  queueHasLockedRequests: activeLockCount > 0,
461
461
  };
462
462
  }
@@ -5,7 +5,7 @@ import type Redis from 'ioredis';
5
5
  * which ioredis delivers as [string[], number]. The ZCARD is taken after the
6
6
  * script's expired-lock purge and after this call's own locks are added, so it
7
7
  * equals the number of active locks held by ANY client — exactly what the
8
- * Apify `queueHasLockedRequests` spec describes ("locked by any client, either
8
+ * `queueHasLockedRequests` flag means ("locked by any client, either
9
9
  * the one calling the endpoint or a different one"). Callers get the flag from
10
10
  * this same atomic reply instead of re-reading the whole locked + pending sets.
11
11
  */
@@ -43,7 +43,7 @@ export declare function matchesLockOwner(owner: string, clientKey: string, runId
43
43
  * Returns the number of unlocked requests. One round-trip, no race window
44
44
  * between ZREM and HDEL, and the whole owners hash never leaves the server.
45
45
  */
46
- export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n local requestId = flat[i]\n local owner = flat[i + 1]\n -- Per Apify spec: in an Actor run, unlock both (1) requests locked by this\n -- specific run (clientKey:runId) and (2) requests locked by the same\n -- clientKey without run context (clientKey), but NOT requests locked by the\n -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n if owner == expectedOwner or owner == clientKey\n or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then\n redis.call('ZREM', lockedKey, requestId)\n redis.call('HDEL', lockOwnersKey, requestId)\n if forefront == '1' then\n redis.call('ZADD', pendingKey, 0, requestId)\n end\n unlockedCount = unlockedCount + 1\n end\nend\n\nreturn unlockedCount\n";
46
+ export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n local requestId = flat[i]\n local owner = flat[i + 1]\n -- In an Actor run, unlock both (1) requests locked by this\n -- specific run (clientKey:runId) and (2) requests locked by the same\n -- clientKey without run context (clientKey), but NOT requests locked by the\n -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n if owner == expectedOwner or owner == clientKey\n or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then\n redis.call('ZREM', lockedKey, requestId)\n redis.call('HDEL', lockOwnersKey, requestId)\n if forefront == '1' then\n redis.call('ZADD', pendingKey, 0, requestId)\n end\n unlockedCount = unlockedCount + 1\n end\nend\n\nreturn unlockedCount\n";
47
47
  export interface UnlockRequestsKeys {
48
48
  lockedKey: string;
49
49
  lockOwnersKey: string;
@@ -10,7 +10,7 @@ exports.unlockRequestsAtomic = unlockRequestsAtomic;
10
10
  * which ioredis delivers as [string[], number]. The ZCARD is taken after the
11
11
  * script's expired-lock purge and after this call's own locks are added, so it
12
12
  * equals the number of active locks held by ANY client — exactly what the
13
- * Apify `queueHasLockedRequests` spec describes ("locked by any client, either
13
+ * `queueHasLockedRequests` flag means ("locked by any client, either
14
14
  * the one calling the endpoint or a different one"). Callers get the flag from
15
15
  * this same atomic reply instead of re-reading the whole locked + pending sets.
16
16
  */
@@ -104,7 +104,7 @@ local unlockedCount = 0
104
104
  for i = 1, #flat, 2 do
105
105
  local requestId = flat[i]
106
106
  local owner = flat[i + 1]
107
- -- Per Apify spec: in an Actor run, unlock both (1) requests locked by this
107
+ -- In an Actor run, unlock both (1) requests locked by this
108
108
  -- specific run (clientKey:runId) and (2) requests locked by the same
109
109
  -- clientKey without run context (clientKey), but NOT requests locked by the
110
110
  -- same clientKey in DIFFERENT runs (clientKey:otherRunId).
@@ -4,7 +4,7 @@ exports.RunMetadataManager = void 0;
4
4
  const billing_tracker_js_1 = require("./billing-tracker.js");
5
5
  /**
6
6
  * Number of most-recent runs per user that are retained indefinitely (no TTL).
7
- * Mirrors Apify's model: the 10 latest runs are always accessible regardless of plan.
7
+ * The 10 latest runs are always accessible regardless of plan.
8
8
  * Older runs beyond this window fall back to the standard plan-based TTL.
9
9
  */
10
10
  const KEEP_N_RUNS = 10;
@@ -3,13 +3,13 @@ import type Redis from 'ioredis';
3
3
  * TTL configuration in seconds
4
4
  */
5
5
  export interface TtlConfig {
6
- /** TTL for unnamed datasets (free users) - default 7 days */
6
+ /** TTL for unnamed datasets (free users) - default 48 hours (Free plan) */
7
7
  datasetFree: number;
8
- /** TTL for unnamed datasets (paid users) - default 31 days */
8
+ /** TTL for unnamed datasets (paid users) - default 31 days (Starter, the shortest paid plan) */
9
9
  datasetPaid: number;
10
- /** TTL for unnamed KV stores (free users) - default 7 days */
10
+ /** TTL for unnamed KV stores (free users) - default 48 hours (Free plan) */
11
11
  kvFree: number;
12
- /** TTL for unnamed KV stores (paid users) - default 31 days */
12
+ /** TTL for unnamed KV stores (paid users) - default 31 days (Starter, the shortest paid plan) */
13
13
  kvPaid: number;
14
14
  /** Fallback TTL for request queues (safety net for orphaned RQs) - default 24 hours */
15
15
  requestQueueFallback: number;
@@ -21,7 +21,7 @@ export interface TtlConfig {
21
21
  logMaxSize: number;
22
22
  }
23
23
  /**
24
- * Default TTL configuration based on Apify's model
24
+ * Default TTL configuration
25
25
  */
26
26
  export declare const DEFAULT_TTL_CONFIG: TtlConfig;
27
27
  export type StorageType = 'dataset' | 'kv' | 'rq' | 'logs';
@@ -31,7 +31,8 @@ export type UserPlan = 'free' | 'paid';
31
31
  *
32
32
  * Retention policy:
33
33
  * - Named storage: Indefinite (billed), no TTL set
34
- * - Unnamed datasets/KV stores: 7 days (free) or 31 days (paid)
34
+ * - Unnamed datasets/KV stores: the plan's retention (Free 48 h, Starter 31 d,
35
+ * Scale 60 d, Business 90 d); the defaults here are Free and Starter
35
36
  * - Request queues: Explicitly deleted after run, 24h fallback TTL for orphaned RQs
36
37
  * - Logs: Short TTL in Redis, persisted to S3
37
38
  */
@@ -2,12 +2,12 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.TtlManager = exports.DEFAULT_TTL_CONFIG = void 0;
4
4
  /**
5
- * Default TTL configuration based on Apify's model
5
+ * Default TTL configuration
6
6
  */
7
7
  exports.DEFAULT_TTL_CONFIG = {
8
- datasetFree: 7 * 24 * 60 * 60, // 7 days
8
+ datasetFree: 48 * 60 * 60, // 48 hours
9
9
  datasetPaid: 31 * 24 * 60 * 60, // 31 days
10
- kvFree: 7 * 24 * 60 * 60, // 7 days
10
+ kvFree: 48 * 60 * 60, // 48 hours
11
11
  kvPaid: 31 * 24 * 60 * 60, // 31 days
12
12
  requestQueueFallback: 24 * 60 * 60, // 24 hours
13
13
  logsFree: 12 * 60 * 60, // 12 hours
@@ -19,7 +19,8 @@ exports.DEFAULT_TTL_CONFIG = {
19
19
  *
20
20
  * Retention policy:
21
21
  * - Named storage: Indefinite (billed), no TTL set
22
- * - Unnamed datasets/KV stores: 7 days (free) or 31 days (paid)
22
+ * - Unnamed datasets/KV stores: the plan's retention (Free 48 h, Starter 31 d,
23
+ * Scale 60 d, Business 90 d); the defaults here are Free and Starter
23
24
  * - Request queues: Explicitly deleted after run, 24h fallback TTL for orphaned RQs
24
25
  * - Logs: Short TTL in Redis, persisted to S3
25
26
  */
@@ -1,6 +1,5 @@
1
1
  /**
2
- * Storage identifier types (Apify-compatible names, so TypeScript actors importing
3
- * them from 'apify' still compile after the apify → scrapely rewrite).
2
+ * Storage identifier types, kept under these names for backward compatibility.
4
3
  */
5
4
  /** Identifies a storage by its alias from the Actor's schema storages. */
6
5
  export interface StorageAlias {
@@ -1,6 +1,5 @@
1
1
  "use strict";
2
2
  /**
3
- * Storage identifier types (Apify-compatible names, so TypeScript actors importing
4
- * them from 'apify' still compile after the apify → scrapely rewrite).
3
+ * Storage identifier types, kept under these names for backward compatibility.
5
4
  */
6
5
  Object.defineProperty(exports, "__esModule", { value: true });
package/dist/utils.d.ts CHANGED
@@ -1,16 +1,16 @@
1
1
  /**
2
- * Apify v2-compatible utility functions for scrapely.
3
- * These adapt the Apify v2 API to Crawlee v3 internally.
2
+ * v2-style utility functions for scrapely.
3
+ * These adapt the v2-style call signatures to Crawlee v3 internally.
4
4
  */
5
5
  import type { CheerioAPI } from 'cheerio';
6
6
  import type { RequestQueue } from 'crawlee';
7
7
  /**
8
- * Options for enqueueLinks (Apify v2 API compatible).
8
+ * Options for enqueueLinks (v2-style API).
9
9
  */
10
10
  export interface EnqueueLinksOptions {
11
- /** Cheerio instance ($) - REQUIRED in Apify v2 */
11
+ /** Cheerio instance ($) - REQUIRED */
12
12
  $: CheerioAPI;
13
- /** RequestQueue instance - REQUIRED in Apify v2 */
13
+ /** RequestQueue instance - REQUIRED */
14
14
  requestQueue: RequestQueue;
15
15
  /** CSS selector to find links. Default: 'a[href]' */
16
16
  selector?: string;
@@ -30,13 +30,13 @@ export interface EnqueueLinksOptions {
30
30
  /**
31
31
  * Enqueues links found on the page using a CSS selector.
32
32
  *
33
- * This is an Apify v2-compatible implementation that adapts to Crawlee v3 internally.
33
+ * This v2-style implementation adapts to Crawlee v3 internally.
34
34
  * It extracts URLs from the page using cheerio and adds them to the request queue.
35
35
  *
36
36
  * @example
37
37
  * ```typescript
38
38
  * // Inside a CheerioCrawler requestHandler:
39
- * await Apify.utils.enqueueLinks({
39
+ * await Actor.utils.enqueueLinks({
40
40
  * $,
41
41
  * requestQueue,
42
42
  * selector: 'a[href*="/app/"]',
@@ -45,10 +45,43 @@ export interface EnqueueLinksOptions {
45
45
  * ```
46
46
  */
47
47
  export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<void>;
48
+ /** Options of the v2-style `utils.requestAsBrowser`; other got options pass through. */
49
+ export interface RequestAsBrowserOptions extends Record<string, any> {
50
+ url: string;
51
+ method?: string;
52
+ headers?: Record<string, string>;
53
+ payload?: string | Buffer;
54
+ proxyUrl?: string;
55
+ languageCode?: string;
56
+ countryCode?: string;
57
+ isMobile?: boolean;
58
+ json?: boolean;
59
+ ignoreSslErrors?: boolean;
60
+ useInsecureHttpParser?: boolean;
61
+ abortFunction?: unknown;
62
+ }
63
+ /**
64
+ * v2-style `utils.requestAsBrowser`: an HTTP request with browser-like headers,
65
+ * here through got-scraping (what v2 itself switched to). Resolves to the got
66
+ * response (`body`, `statusCode`, `headers`, `url`).
67
+ */
68
+ export declare function requestAsBrowser(options: RequestAsBrowserOptions): Promise<any>;
69
+ /** v2-style `utils.getRandomUserAgent`: a realistic desktop browser User-Agent. */
70
+ export declare function getRandomUserAgent(): string;
48
71
  /**
49
- * Utility namespace providing Apify v2-compatible helper functions.
72
+ * Utility namespace providing v2-style helper functions.
50
73
  * These adapt to Crawlee v3 APIs internally.
51
74
  */
52
75
  export declare const utils: {
53
76
  enqueueLinks: typeof enqueueLinks;
77
+ requestAsBrowser: typeof requestAsBrowser;
78
+ getRandomUserAgent: typeof getRandomUserAgent;
79
+ sleep: (millis?: number) => any;
80
+ downloadListOfUrls: (options: any) => any;
81
+ extractUrls: (options: any) => any;
82
+ htmlToText: (html: any) => any;
83
+ isDocker: (forceReset?: boolean) => any;
84
+ readonly log: any;
85
+ readonly puppeteer: any;
86
+ readonly playwright: any;
54
87
  };
package/dist/utils.js CHANGED
@@ -2,16 +2,18 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.utils = void 0;
4
4
  exports.enqueueLinks = enqueueLinks;
5
+ exports.requestAsBrowser = requestAsBrowser;
6
+ exports.getRandomUserAgent = getRandomUserAgent;
5
7
  /**
6
8
  * Enqueues links found on the page using a CSS selector.
7
9
  *
8
- * This is an Apify v2-compatible implementation that adapts to Crawlee v3 internally.
10
+ * This v2-style implementation adapts to Crawlee v3 internally.
9
11
  * It extracts URLs from the page using cheerio and adds them to the request queue.
10
12
  *
11
13
  * @example
12
14
  * ```typescript
13
15
  * // Inside a CheerioCrawler requestHandler:
14
- * await Apify.utils.enqueueLinks({
16
+ * await Actor.utils.enqueueLinks({
15
17
  * $,
16
18
  * requestQueue,
17
19
  * selector: 'a[href*="/app/"]',
@@ -55,9 +57,53 @@ async function enqueueLinks(options) {
55
57
  }
56
58
  }
57
59
  /**
58
- * Utility namespace providing Apify v2-compatible helper functions.
60
+ * v2-style `utils.requestAsBrowser`: an HTTP request with browser-like headers,
61
+ * here through got-scraping (what v2 itself switched to). Resolves to the got
62
+ * response (`body`, `statusCode`, `headers`, `url`).
63
+ */
64
+ async function requestAsBrowser(options) {
65
+ const { payload, languageCode, countryCode, isMobile, json, ignoreSslErrors, abortFunction: _abortFunction, ...gotOptions } = options;
66
+ const { gotScraping } = await import('@crawlee/utils');
67
+ const locale = languageCode ? `${languageCode}${countryCode ? `-${countryCode}` : ''}` : undefined;
68
+ return gotScraping({
69
+ ...(payload !== undefined ? { body: payload } : {}),
70
+ ...(json ? { responseType: 'json' } : {}),
71
+ ...(ignoreSslErrors ? { https: { rejectUnauthorized: false } } : {}),
72
+ headerGeneratorOptions: {
73
+ ...(locale ? { locales: [locale] } : {}),
74
+ ...(isMobile ? { devices: ['mobile'] } : {}),
75
+ },
76
+ ...gotOptions,
77
+ });
78
+ }
79
+ let userAgents;
80
+ /** v2-style `utils.getRandomUserAgent`: a realistic desktop browser User-Agent. */
81
+ function getRandomUserAgent() {
82
+ if (!userAgents) {
83
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
84
+ const { HeaderGenerator } = require('header-generator');
85
+ userAgents = new HeaderGenerator({ devices: ['desktop'] });
86
+ }
87
+ const headers = userAgents.getHeaders();
88
+ return headers['user-agent'] ?? headers['User-Agent'];
89
+ }
90
+ // Loaded on first use: the SDK entry must not load Crawlee (see lazy-exports.ts).
91
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
92
+ const crawleeUtils = () => require('@crawlee/utils');
93
+ /**
94
+ * Utility namespace providing v2-style helper functions.
59
95
  * These adapt to Crawlee v3 APIs internally.
60
96
  */
61
97
  exports.utils = {
62
98
  enqueueLinks,
99
+ requestAsBrowser,
100
+ getRandomUserAgent,
101
+ sleep: (millis) => crawleeUtils().sleep(millis),
102
+ downloadListOfUrls: (options) => crawleeUtils().downloadListOfUrls(options),
103
+ extractUrls: (options) => crawleeUtils().extractUrls(options),
104
+ htmlToText: (html) => crawleeUtils().htmlToText(html),
105
+ isDocker: (forceReset) => crawleeUtils().isDocker(forceReset),
106
+ get log() { return require('@apify/log').default; },
107
+ get puppeteer() { return require('@crawlee/puppeteer').puppeteerUtils; },
108
+ get playwright() { return require('@crawlee/playwright').playwrightUtils; },
63
109
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "scrapely",
3
- "version": "1.2.32",
3
+ "version": "1.2.34",
4
4
  "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -21,12 +21,17 @@
21
21
  "dist",
22
22
  "LICENSE",
23
23
  "LICENSE-APACHE",
24
- "NOTICE"
24
+ "NOTICE",
25
+ "CHANGELOG.md"
25
26
  ],
26
27
  "private": false,
28
+ "homepage": "https://scrape.ly/docs/sdk-node",
27
29
  "repository": {
28
30
  "type": "git",
29
- "url": "git+https://github.com/SandeepDev1/scrapely.git"
31
+ "url": "git+https://github.com/scrapely-app/scrapely.git"
32
+ },
33
+ "bugs": {
34
+ "url": "https://github.com/scrapely-app/scrapely/issues"
30
35
  },
31
36
  "publishConfig": {
32
37
  "registry": "https://registry.npmjs.org",
@@ -37,6 +42,9 @@
37
42
  "test:charging": "npx tsx --test tests/charging-pending.test.ts",
38
43
  "test:charge-limit-guard": "npx tsx --test tests/charge-limit-guard.test.ts",
39
44
  "test:charge-multi-event": "npx tsx --test tests/charge-multi-event.test.ts",
45
+ "test:charge-multi-event-void": "npx tsx --test tests/charge-multi-event-void.test.ts",
46
+ "test:charge-edge-cases": "npx tsx --test tests/charge-edge-cases.test.ts",
47
+ "test:charge-pending-persist": "npx tsx --test tests/charge-pending-persist.test.ts",
40
48
  "test:charge-platform-budget": "npx tsx --test tests/charge-platform-budget.test.ts",
41
49
  "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
42
50
  "test:push-idempotency": "npx tsx --test tests/push-idempotency.test.ts",
@@ -46,6 +54,7 @@
46
54
  "test:browser-pool": "npx tsx --test tests/patch-browser-pool.test.ts",
47
55
  "test:browser-config": "npx tsx --test tests/browser-configuration.test.ts",
48
56
  "build": "tsc",
57
+ "prepack": "npm run build",
49
58
  "setup:gvisor": "bash scripts/setup-gvisor.sh",
50
59
  "setup:server": "bash scripts/setup-server.sh",
51
60
  "prod": "bash scripts/deploy-prod.sh",
@@ -54,7 +63,7 @@
54
63
  "test:local-run": "npx tsx --test tests/local-run.test.ts",
55
64
  "test:input-from-kv": "npx tsx --test tests/input-from-kv.test.ts",
56
65
  "test:init-rollback": "npx tsx --test tests/init-rollback.test.ts",
57
- "test:apify-parity": "npx tsx --test tests/apify-parity.test.ts",
66
+ "test:parity": "npx tsx --test tests/apify-parity.test.ts",
58
67
  "test:run-control": "npx tsx --test tests/run-control.test.ts",
59
68
  "test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts",
60
69
  "test:terminal-status": "npx tsx --test tests/terminal-status-message.test.ts",
@@ -62,16 +71,17 @@
62
71
  "test:proxy-input-aliases": "npx tsx --test tests/proxy-input-aliases.test.ts",
63
72
  "test:charge-retries": "npx tsx --test tests/charge-retries.test.ts",
64
73
  "test:solver": "npx tsx --test tests/solver.test.ts",
65
- "test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts"
74
+ "test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts",
75
+ "test:v2-utils": "npx tsx --test tests/v2-utils.test.ts"
66
76
  },
67
- "author": "SandeepDev1",
77
+ "author": "TeraShift GmbH (Scrape.ly)",
68
78
  "license": "ISC AND Apache-2.0",
69
79
  "dependencies": {
70
80
  "@aws-sdk/client-s3": "^3.700.0",
71
81
  "@crawlee/browser": "^3.16.0",
72
82
  "@crawlee/browser-pool": "^3.16.0",
73
83
  "@crawlee/playwright": "^3.16.0",
74
- "basic-ftp": "^6.0.1",
84
+ "basic-ftp": "^6.2.3",
75
85
  "crawlee": "^3.16.0",
76
86
  "dockerode": "^4.0.9",
77
87
  "ioredis": "^6.0.0",
@@ -93,14 +103,14 @@
93
103
  "typescript": "^5.7.0"
94
104
  },
95
105
  "overrides": {
96
- "adm-zip": "^0.6.0",
106
+ "adm-zip": "^0.6.1",
97
107
  "tar-fs": "2.1.5",
98
108
  "yargs": "18.1.0",
99
- "axios": "^1.18.0",
109
+ "axios": "^1.20.0",
100
110
  "brace-expansion": "5.0.9",
101
111
  "fast-xml-builder": "^1.2.1",
102
112
  "file-type": "^21.3.3",
103
- "follow-redirects": "^1.16.0",
113
+ "follow-redirects": "^1.16.1",
104
114
  "@crawlee/core": {
105
115
  "ow": "0.28.2"
106
116
  },
@@ -116,6 +126,7 @@
116
126
  "uuid": "^11.1.1",
117
127
  "@inquirer/external-editor": "3.0.3",
118
128
  "nanoid": "5.1.16",
119
- "browserslist": "4.28.7"
129
+ "browserslist": "4.28.7",
130
+ "basic-ftp": "^6.2.3"
120
131
  }
121
132
  }
@@ -1,88 +0,0 @@
1
- import Docker from 'dockerode';
2
- import { TimeoutManager } from './timeout-manager.js';
3
- import { DragonflyClient } from '../storage/dragonfly-client.js';
4
- import type { UserPlan } from '../storage/ttl-manager.js';
5
- export interface RunOptions {
6
- /** Path to the actor package directory */
7
- packagePath: string;
8
- /** Path to the .env file */
9
- envPath?: string;
10
- /** Environment variables to pass to container */
11
- env?: Record<string, string>;
12
- /** Maximum memory in MB (default: from manifest or 512) */
13
- memoryMb?: number;
14
- /** Maximum timeout in seconds (default: from manifest or 1800) */
15
- timeout?: number;
16
- /** Run ID for tracking */
17
- runId?: string;
18
- /** User ID */
19
- userId?: string;
20
- /** User plan for TTL */
21
- userPlan?: UserPlan;
22
- /** Dragonfly client for storage cleanup */
23
- dragonflyClient?: DragonflyClient;
24
- }
25
- export interface RunResult {
26
- containerId: string;
27
- exitCode: number;
28
- logs: string;
29
- status: 'success' | 'failed' | 'timed_out' | 'killed';
30
- duration: number;
31
- /** Storage IDs that were cleaned up */
32
- cleanedUp?: {
33
- requestQueueDeleted: boolean;
34
- logsUploadedToS3: boolean;
35
- };
36
- }
37
- export declare class DockerManager {
38
- private docker;
39
- private timeoutManager;
40
- private imageName;
41
- constructor(options?: {
42
- socketPath?: string;
43
- imageName?: string;
44
- });
45
- /**
46
- * Run an actor package in a container
47
- */
48
- run(options: RunOptions): Promise<RunResult>;
49
- /**
50
- * Cleanup storage after run completes.
51
- * - Deletes request queue immediately (no longer needed)
52
- * - Uploads logs to S3 for long-term storage
53
- * - Updates run metadata with final status
54
- */
55
- private cleanupAfterRun;
56
- /**
57
- * Stop a container (SIGTERM)
58
- */
59
- stop(containerId: string, timeoutSeconds?: number): Promise<void>;
60
- /**
61
- * Kill a container (SIGKILL)
62
- */
63
- kill(containerId: string): Promise<void>;
64
- /**
65
- * Get logs from a container
66
- */
67
- getLogs(containerId: string): Promise<string>;
68
- /**
69
- * Get container status
70
- */
71
- getStatus(containerId: string): Promise<string | null>;
72
- /**
73
- * List all scrapely containers
74
- */
75
- listContainers(): Promise<Docker.ContainerInfo[]>;
76
- /**
77
- * Get the timeout manager
78
- */
79
- getTimeoutManager(): TimeoutManager;
80
- /**
81
- * Track internal data transfer from container network stats.
82
- * Following Apify's model:
83
- * - Ingress (rx_bytes) is FREE - scraped content coming in
84
- * - Internal transfer is tracked separately in storage clients
85
- * - External egress is tracked in API layer when users fetch results
86
- */
87
- private trackDataTransfer;
88
- }