scrapely 1.2.3 → 1.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +119 -1
  2. package/dist/actor.d.ts +703 -0
  3. package/dist/actor.js +1441 -0
  4. package/dist/browser-configuration.d.ts +58 -0
  5. package/dist/browser-configuration.js +128 -0
  6. package/dist/browser-locations.d.ts +5 -0
  7. package/dist/browser-locations.js +5469 -0
  8. package/dist/charging.d.ts +153 -0
  9. package/dist/charging.js +782 -0
  10. package/dist/charging_client.d.ts +18 -0
  11. package/dist/charging_client.js +49 -0
  12. package/dist/configuration.d.ts +121 -0
  13. package/dist/configuration.js +128 -0
  14. package/dist/crypto/secrets.d.ts +44 -0
  15. package/dist/crypto/secrets.js +164 -0
  16. package/dist/events/events-client.d.ts +59 -0
  17. package/dist/events/events-client.js +120 -0
  18. package/dist/events/scrapely-event-manager.d.ts +13 -0
  19. package/dist/events/scrapely-event-manager.js +61 -0
  20. package/dist/index.d.ts +29 -0
  21. package/dist/index.js +95 -0
  22. package/dist/log-env.d.ts +1 -0
  23. package/dist/log-env.js +12 -0
  24. package/dist/manifest.d.ts +30 -0
  25. package/dist/manifest.js +47 -0
  26. package/dist/patch-browser-pool.d.ts +66 -0
  27. package/dist/patch-browser-pool.js +827 -0
  28. package/dist/proxy-configuration.d.ts +118 -0
  29. package/dist/proxy-configuration.js +88 -0
  30. package/dist/request-list.d.ts +57 -0
  31. package/dist/request-list.js +79 -0
  32. package/dist/runner/docker-manager.d.ts +88 -0
  33. package/dist/runner/docker-manager.js +283 -0
  34. package/dist/runner/index.d.ts +2 -0
  35. package/dist/runner/index.js +7 -0
  36. package/dist/runner/timeout-manager.d.ts +43 -0
  37. package/dist/runner/timeout-manager.js +90 -0
  38. package/dist/scrapely-proxy-configuration.d.ts +99 -0
  39. package/dist/scrapely-proxy-configuration.js +559 -0
  40. package/dist/storage/api-storage-client.d.ts +199 -0
  41. package/dist/storage/api-storage-client.js +910 -0
  42. package/dist/storage/billing-tracker.d.ts +163 -0
  43. package/dist/storage/billing-tracker.js +260 -0
  44. package/dist/storage/dataset-client.d.ts +43 -0
  45. package/dist/storage/dataset-client.js +232 -0
  46. package/dist/storage/dataset-collection-client.d.ts +8 -0
  47. package/dist/storage/dataset-collection-client.js +75 -0
  48. package/dist/storage/dataset-push-script.d.ts +18 -0
  49. package/dist/storage/dataset-push-script.js +38 -0
  50. package/dist/storage/dragonfly-client.d.ts +94 -0
  51. package/dist/storage/dragonfly-client.js +167 -0
  52. package/dist/storage/index.d.ts +11 -0
  53. package/dist/storage/index.js +25 -0
  54. package/dist/storage/key-value-store-client.d.ts +61 -0
  55. package/dist/storage/key-value-store-client.js +306 -0
  56. package/dist/storage/kv-store-collection-client.d.ts +8 -0
  57. package/dist/storage/kv-store-collection-client.js +76 -0
  58. package/dist/storage/kvs-iteration.d.ts +17 -0
  59. package/dist/storage/kvs-iteration.js +126 -0
  60. package/dist/storage/log-manager.d.ts +114 -0
  61. package/dist/storage/log-manager.js +244 -0
  62. package/dist/storage/patch-storage.d.ts +11 -0
  63. package/dist/storage/patch-storage.js +177 -0
  64. package/dist/storage/request-queue-client.d.ts +93 -0
  65. package/dist/storage/request-queue-client.js +522 -0
  66. package/dist/storage/rq-collection-client.d.ts +8 -0
  67. package/dist/storage/rq-collection-client.js +89 -0
  68. package/dist/storage/rq-list-requests.d.ts +59 -0
  69. package/dist/storage/rq-list-requests.js +114 -0
  70. package/dist/storage/rq-lock-script.d.ts +56 -0
  71. package/dist/storage/rq-lock-script.js +127 -0
  72. package/dist/storage/run-metadata.d.ts +98 -0
  73. package/dist/storage/run-metadata.js +256 -0
  74. package/dist/storage/s3-client.d.ts +103 -0
  75. package/dist/storage/s3-client.js +302 -0
  76. package/dist/storage/ttl-manager.d.ts +70 -0
  77. package/dist/storage/ttl-manager.js +115 -0
  78. package/dist/storage/utils.d.ts +3 -0
  79. package/dist/storage/utils.js +19 -0
  80. package/dist/storage-identifiers.d.ts +20 -0
  81. package/dist/storage-identifiers.js +6 -0
  82. package/dist/utils.d.ts +54 -0
  83. package/dist/utils.js +63 -0
  84. package/dist/validator.d.ts +24 -0
  85. package/dist/validator.js +158 -0
  86. package/package.json +90 -9
package/dist/actor.js ADDED
@@ -0,0 +1,1441 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Actor = exports.EXIT_CODES = exports.ACTOR_PERMISSION_LEVEL = void 0;
4
+ require("./log-env.js");
5
+ const node_events_1 = require("node:events");
6
+ /** Actor permission level - matches Apify's ACTOR_PERMISSION_LEVEL enum */
7
+ var ACTOR_PERMISSION_LEVEL;
8
+ (function (ACTOR_PERMISSION_LEVEL) {
9
+ /** User has no special permissions */
10
+ ACTOR_PERMISSION_LEVEL["NONE"] = "NONE";
11
+ /** User can run the actor */
12
+ ACTOR_PERMISSION_LEVEL["RUN"] = "RUN";
13
+ /** User can modify the actor */
14
+ ACTOR_PERMISSION_LEVEL["WRITE"] = "WRITE";
15
+ /** User has full administrative access */
16
+ ACTOR_PERMISSION_LEVEL["ADMIN"] = "ADMIN";
17
+ })(ACTOR_PERMISSION_LEVEL || (exports.ACTOR_PERMISSION_LEVEL = ACTOR_PERMISSION_LEVEL = {}));
18
+ const core_1 = require("@crawlee/core");
19
+ const configuration_js_1 = require("./configuration.js");
20
+ const patch_storage_js_1 = require("./storage/patch-storage.js");
21
+ const api_storage_client_js_1 = require("./storage/api-storage-client.js");
22
+ const events_client_js_1 = require("./events/events-client.js");
23
+ const scrapely_event_manager_js_1 = require("./events/scrapely-event-manager.js");
24
+ const secrets_js_1 = require("./crypto/secrets.js");
25
+ const charging_js_1 = require("./charging.js");
26
+ const charging_client_js_1 = require("./charging_client.js");
27
+ const proxy_configuration_js_1 = require("./proxy-configuration.js");
28
+ const browser_configuration_js_1 = require("./browser-configuration.js");
29
+ const utils_js_1 = require("./utils.js");
30
+ /** Pre-init fallback; after init, {@link Actor.events} returns the EventManager emitter. */
31
+ const fallbackEvents = new node_events_1.EventEmitter();
32
+ let apifyClientInstance;
33
+ let currentUserPlan = 'free';
34
+ let currentRunId;
35
+ let currentUserId;
36
+ let cachedInput = undefined;
37
+ const gracefulShutdownHandlers = {};
38
+ /**
39
+ * Registered SIGINT/SIGTERM flush handlers. Tracked (module-level, like
40
+ * {@link gracefulShutdownHandlers}) so `_shutdownActor` and the failed-init
41
+ * rollback can remove them — otherwise an init retry would stack duplicates.
42
+ */
43
+ const signalExitHandlers = {};
44
+ /**
45
+ * Exit codes for the Actor process (Apify-compatible).
46
+ * Must be in the range 1-128 to avoid collision with signal exits.
47
+ */
48
+ exports.EXIT_CODES = {
49
+ SUCCESS: 0,
50
+ ERROR_USER_FUNCTION_THREW: 91,
51
+ ERROR_UNKNOWN: 92,
52
+ };
53
+ function resolveCallTimeout(timeout) {
54
+ if (timeout === 'inherit') {
55
+ const remainingMs = Actor.getRemainingTime();
56
+ if (remainingMs == null) {
57
+ console.warn('[Scrapely] Using `inherit` for timeout is only possible when running on the platform and when ' +
58
+ 'the timeout for the Actor run is set (ACTOR_TIMEOUT_AT).');
59
+ return undefined;
60
+ }
61
+ return Math.max(0, Math.ceil(remainingMs / 1000));
62
+ }
63
+ return timeout;
64
+ }
65
+ function isApiStorageConfig(storage) {
66
+ return (typeof storage === 'object' &&
67
+ storage !== null &&
68
+ 'apiUrl' in storage &&
69
+ 'token' in storage &&
70
+ 'runId' in storage);
71
+ }
72
+ function isStorageClient(storage) {
73
+ return typeof storage === 'object' && storage !== null && 'datasets' in storage;
74
+ }
75
+ function applyConfigurationOptions(configuration, options) {
76
+ if (!options)
77
+ return;
78
+ for (const key of Object.keys(options)) {
79
+ const value = options[key];
80
+ if (value !== undefined) {
81
+ configuration.set(key, value);
82
+ }
83
+ }
84
+ }
85
+ class Actor {
86
+ /** Configuration for this Actor instance (pass to crawlers as second argument). */
87
+ config;
88
+ /** Whether this Actor instance was initialized. Set by {@link Actor.init}. */
89
+ initialized = false;
90
+ /**
91
+ * Tracks that init() has started installing process-global lifecycle state
92
+ * (storage factory with run credentials, events WebSocket, event manager,
93
+ * graceful-shutdown/signal handlers, charging). Deliberately separate from
94
+ * the public {@link initialized} flag: `_shutdownActor` must tear down
95
+ * partial state from a FAILED init (so it stays retryable), while
96
+ * `initialized` — used by `_ensureActorInit` — flips only after init fully
97
+ * succeeds.
98
+ */
99
+ _lifecycleStarted = false;
100
+ _apifyClient;
101
+ chargingManager;
102
+ /**
103
+ * Creates an Actor instance (Apify SDK-compatible alternative to static helpers).
104
+ */
105
+ constructor(options = {}) {
106
+ this.config =
107
+ Object.keys(options).length === 0 ? configuration_js_1.Configuration.getGlobalConfig() : new configuration_js_1.Configuration(options);
108
+ this._apifyClient = this.newClient();
109
+ this.chargingManager = new charging_js_1.ChargingManager(this.config, this._apifyClient);
110
+ Actor._instance ??= this;
111
+ }
112
+ /** Default ApifyClient instance for this Actor (Apify-compatible instance accessor). */
113
+ get apifyClient() {
114
+ return this._apifyClient;
115
+ }
116
+ _ensureInit(methodName) {
117
+ if (!this.initialized) {
118
+ throw new Error(`Actor.${methodName}() requires Actor.init() to be called first. ` +
119
+ `Hint: Use Actor.main() which calls init() for you.`);
120
+ }
121
+ }
122
+ static _actorConfig() {
123
+ return Actor.getDefaultInstance().config;
124
+ }
125
+ /**
126
+ * Event emitter for Actor events (same instance as the platform EventManager).
127
+ * Supports events: 'persistState', 'migrating', 'aborting', 'cpuInfo'.
128
+ *
129
+ * Usage:
130
+ * ```typescript
131
+ * Actor.events.on('persistState', () => {
132
+ * // Save state before migration
133
+ * });
134
+ * ```
135
+ */
136
+ static get events() {
137
+ const actor = Actor._instance;
138
+ if (actor?.initialized) {
139
+ return actor.config.getEventManager().events;
140
+ }
141
+ return fallbackEvents;
142
+ }
143
+ /**
144
+ * Throws an error if the Actor was not initialized.
145
+ * Used internally to ensure methods are called after init().
146
+ */
147
+ static _ensureActorInit(methodName) {
148
+ if (!Actor.getDefaultInstance().initialized) {
149
+ throw new Error(`Actor.${methodName}() requires Actor.init() to be called first. ` +
150
+ `Hint: Use Actor.main() which calls init() for you.`);
151
+ }
152
+ }
153
+ /**
154
+ * Returns the remaining time in milliseconds until the Actor run times out.
155
+ * Uses the ACTOR_TIMEOUT_AT environment variable set by the platform.
156
+ *
157
+ * @returns Milliseconds remaining, or undefined if no timeout is set (not running on platform)
158
+ */
159
+ static getRemainingTime() {
160
+ const timeoutAt = process.env.ACTOR_TIMEOUT_AT;
161
+ if (!timeoutAt)
162
+ return undefined;
163
+ const timeoutDate = new Date(timeoutAt);
164
+ const remaining = timeoutDate.getTime() - Date.now();
165
+ return Math.max(0, remaining);
166
+ }
167
+ /**
168
+ * Initializes this Actor instance.
169
+ *
170
+ * @param options - Initialization options
171
+ */
172
+ async init(options = {}) {
173
+ if (this.initialized)
174
+ return;
175
+ // NOTE: `initialized` is set at the END of init — a failed platform init
176
+ // must stay retryable instead of leaving a half-initialized Actor.
177
+ Actor.installFatalChargeHandlers();
178
+ const config = this.config;
179
+ core_1.Configuration.globalConfig = configuration_js_1.Configuration.getGlobalConfig();
180
+ applyConfigurationOptions(config, options.configuration);
181
+ if (config.get('isAtHome')) {
182
+ config.set('purgeOnStart', false);
183
+ }
184
+ // NOTE: CDP URL is set lazily in patch-browser-pool.ts (remote browser via CDP_URL).
185
+ // From here on, init() mutates process-global lifecycle state (storage
186
+ // factory with run credentials, events WebSocket, event manager,
187
+ // graceful-shutdown handlers, charging). `_lifecycleStarted` marks that
188
+ // this state exists even before init finishes, so `_shutdownActor` can
189
+ // tear it down if init fails — the public `initialized` flag must NOT
190
+ // be used for that (it stays false until the whole init succeeds).
191
+ this._lifecycleStarted = true;
192
+ let eventManager;
193
+ const prevApifyClientInstance = apifyClientInstance;
194
+ let apifyClientAssigned = false;
195
+ try {
196
+ if (options.storage) {
197
+ if (isApiStorageConfig(options.storage)) {
198
+ console.log(`[Scrapely] Initializing API storage for run ${options.storage.runId}`);
199
+ (0, patch_storage_js_1.setApiStorage)(options.storage, config);
200
+ }
201
+ else if (isStorageClient(options.storage)) {
202
+ config.useStorageClient(options.storage);
203
+ }
204
+ else {
205
+ throw new Error('Actor.init() storage option must be ApiStorageConfig or StorageClient');
206
+ }
207
+ }
208
+ else {
209
+ const apiUrl = config.get('apiBaseUrl');
210
+ const token = config.get('token');
211
+ const runId = config.get('actorRunId');
212
+ if (!apiUrl || !token || !runId) {
213
+ if (config.get('isAtHome')) {
214
+ throw new Error('Missing required configuration for secure storage. ' +
215
+ 'Required: apiBaseUrl (SCRAPELY_API_URL), token (SCRAPELY_TOKEN/APIFY_TOKEN), actorRunId (ACTOR_RUN_ID). ' +
216
+ 'Containers must be started by the scrapely-server, or pass storage in Actor.init().');
217
+ }
218
+ // Local development without platform credentials: keep Crawlee's
219
+ // default storage client (memory, or `./storage` filesystem), the
220
+ // same fallback the Python SDK uses (`Smart → FS`). Storage, input
221
+ // (KeyValueStore INPUT key), charging, and platform events all
222
+ // degrade to local behavior — no remote run exists to talk to.
223
+ console.log('[Scrapely] Local run detected — using Crawlee local storage');
224
+ }
225
+ else {
226
+ console.log(`[Scrapely] Initializing API storage for run ${runId}`);
227
+ (0, patch_storage_js_1.setApiStorage)({ apiUrl, token, runId }, config);
228
+ }
229
+ }
230
+ const runToken = (options.storage && isApiStorageConfig(options.storage)
231
+ ? options.storage.token
232
+ : undefined) ?? config.get('token');
233
+ const runId = (options.storage && isApiStorageConfig(options.storage)
234
+ ? options.storage.runId
235
+ : undefined) ?? config.get('actorRunId');
236
+ const eventsWsUrl = config.get('actorEventsWsUrl');
237
+ if (eventsWsUrl && runToken && runId) {
238
+ (0, events_client_js_1.connectEvents)(eventsWsUrl, runToken, runId);
239
+ }
240
+ configuration_js_1.Configuration.storage.enterWith(config);
241
+ eventManager = new scrapely_event_manager_js_1.ScrapelyEventManager(config);
242
+ config.useEventManager(eventManager);
243
+ await eventManager.init();
244
+ // Register handlers for aborting and migrating events for automatic graceful shutdown.
245
+ // - aborting: calls Actor.exit() to terminate the run gracefully
246
+ // - migrating: calls Actor.reboot() to speed up migration (the run continues on a new worker)
247
+ if (options.gracefulShutdown !== false) {
248
+ const delay = options.gracefulShutdownDelayMillis ?? 0;
249
+ gracefulShutdownHandlers.aborting = () => {
250
+ setTimeout(() => {
251
+ Actor.exit().catch((err) => {
252
+ console.error(`[Scrapely] Failed to exit gracefully: ${err.message}`);
253
+ });
254
+ }, delay);
255
+ };
256
+ this.on('aborting', gracefulShutdownHandlers.aborting);
257
+ gracefulShutdownHandlers.migrating = () => {
258
+ setTimeout(() => {
259
+ Actor.reboot().catch((err) => {
260
+ console.error(`[Scrapely] Failed to reboot on migration: ${err.message}`);
261
+ });
262
+ }, delay);
263
+ };
264
+ this.on('migrating', gracefulShutdownHandlers.migrating);
265
+ }
266
+ apifyClientInstance = this._apifyClient;
267
+ apifyClientAssigned = true;
268
+ await this.chargingManager.init();
269
+ // Flush pending PPE intents before hard exit so uncertain charges can still settle.
270
+ const gracefulSignalExit = (signal) => {
271
+ const doExit = () => {
272
+ process.exit(0);
273
+ };
274
+ // Bound flush so a hung network cannot block container stop forever.
275
+ const flushTimeout = setTimeout(doExit, 5_000);
276
+ Actor._shutdownActor(5)
277
+ .catch((err) => {
278
+ console.warn(`[Scrapely] Shutdown flush on ${signal} failed: ${err.message}`);
279
+ })
280
+ .finally(() => {
281
+ clearTimeout(flushTimeout);
282
+ doExit();
283
+ });
284
+ };
285
+ signalExitHandlers.sigint = () => gracefulSignalExit('SIGINT');
286
+ signalExitHandlers.sigterm = () => gracefulSignalExit('SIGTERM');
287
+ process.on('SIGINT', signalExitHandlers.sigint);
288
+ process.on('SIGTERM', signalExitHandlers.sigterm);
289
+ this.initialized = true;
290
+ }
291
+ catch (err) {
292
+ // Rollback of partial init. Without this, a failed init would leave
293
+ // the events WebSocket open (auto-reconnecting with the failed run's
294
+ // token), the API storage factory holding stale run credentials and
295
+ // caches, an open event manager, and registered handlers — so an
296
+ // init retry would overlay another run's credentials and handlers
297
+ // on this same process. NOTE: charging state is intentionally NOT
298
+ // closed here — `charge()` requires a completed init, so a partial
299
+ // init cannot have admitted charges, and closing the manager would
300
+ // also reject charges made after a successful retry.
301
+ try {
302
+ if (signalExitHandlers.sigint) {
303
+ process.removeListener('SIGINT', signalExitHandlers.sigint);
304
+ signalExitHandlers.sigint = undefined;
305
+ }
306
+ if (signalExitHandlers.sigterm) {
307
+ process.removeListener('SIGTERM', signalExitHandlers.sigterm);
308
+ signalExitHandlers.sigterm = undefined;
309
+ }
310
+ if (gracefulShutdownHandlers.aborting) {
311
+ this.off('aborting', gracefulShutdownHandlers.aborting);
312
+ gracefulShutdownHandlers.aborting = undefined;
313
+ }
314
+ if (gracefulShutdownHandlers.migrating) {
315
+ this.off('migrating', gracefulShutdownHandlers.migrating);
316
+ gracefulShutdownHandlers.migrating = undefined;
317
+ }
318
+ if (eventManager) {
319
+ await eventManager.close().catch(() => undefined);
320
+ }
321
+ (0, events_client_js_1.disconnectEvents)();
322
+ (0, patch_storage_js_1.disconnectStorage)();
323
+ if (apifyClientAssigned) {
324
+ apifyClientInstance = prevApifyClientInstance;
325
+ }
326
+ }
327
+ finally {
328
+ this._lifecycleStarted = false;
329
+ }
330
+ throw err;
331
+ }
332
+ }
333
+ static async init(options = {}) {
334
+ return Actor.getDefaultInstance().init(options);
335
+ }
336
+ /**
337
+ * Runs the main user function that performs the job of the Actor
338
+ * and terminates the process when the user function finishes.
339
+ *
340
+ * @param userFunc - User function to be executed
341
+ * @param options - Main options (exit, init, gracefulShutdown, etc.)
342
+ */
343
+ static async main(userFunc, options) {
344
+ if (!userFunc || typeof userFunc !== 'function') {
345
+ throw new Error(`First parameter for Actor.main() must be a function (was '${userFunc === null ? 'null' : typeof userFunc}').`);
346
+ }
347
+ await Actor.init(options);
348
+ try {
349
+ const result = await userFunc();
350
+ await Actor._shutdownActor(options?.timeoutSecs ?? 30);
351
+ await new Promise((resolve) => process.stdout.write('', () => resolve()));
352
+ process.exit(options?.exitCode ?? 0);
353
+ return result;
354
+ }
355
+ catch (err) {
356
+ const cause = err.cause;
357
+ const causeStr = cause ? ` (cause: ${cause.message ?? String(cause)})` : '';
358
+ console.error(`[Scrapely] Actor failed: ${err.message}${causeStr}`);
359
+ // Still flush pending charges on failure paths before hard exit.
360
+ try {
361
+ await Actor._shutdownActor(options?.timeoutSecs ?? 30);
362
+ }
363
+ catch (shutdownErr) {
364
+ console.warn(`[Scrapely] Shutdown after failure failed: ${shutdownErr.message}`);
365
+ }
366
+ await new Promise((resolve) => process.stderr.write('', () => resolve()));
367
+ process.exit(options?.exitCode ?? 1);
368
+ throw err;
369
+ }
370
+ }
371
+ /**
372
+ * Gracefully exits the Actor run.
373
+ *
374
+ * @param options - Exit options
375
+ */
376
+ static async exit(messageOrOptions, options = {}) {
377
+ const opts = typeof messageOrOptions === 'string'
378
+ ? { ...options, statusMessage: messageOrOptions }
379
+ : { ...messageOrOptions, ...options };
380
+ const code = opts.exitCode ?? 0;
381
+ const timeoutSecs = opts.timeoutSecs ?? 30;
382
+ const shouldExit = opts.exit ?? true;
383
+ if (opts.statusMessage)
384
+ console.log(`[Scrapely] ${opts.statusMessage}`);
385
+ Actor._ensureActorInit('exit');
386
+ await Actor._shutdownActor(timeoutSecs);
387
+ if (shouldExit) {
388
+ await new Promise((resolve) => process.stdout.write('', () => resolve()));
389
+ process.exit(code);
390
+ }
391
+ }
392
+ static async fail(messageOrOptions, options = {}) {
393
+ const opts = typeof messageOrOptions === 'string'
394
+ ? { exitCode: 1, statusMessage: messageOrOptions, ...options }
395
+ : { exitCode: 1, ...messageOrOptions, ...options };
396
+ console.error(`[Scrapely] Actor failed: ${opts.statusMessage ?? 'Unknown error'}`);
397
+ await new Promise((resolve) => process.stderr.write('', () => resolve()));
398
+ await Actor.exit(opts);
399
+ }
400
+ static async getInput() {
401
+ Actor._ensureActorInit('getInput');
402
+ // Check for metamorph input first (Apify-compatible)
403
+ // When an actor is metamorphed, the input is stored under INPUT-METAMORPH-{n}
404
+ const metamorphCount = process.env.ACTOR_METAMORPH_COUNT;
405
+ if (metamorphCount) {
406
+ const count = parseInt(metamorphCount, 10);
407
+ if (count > 0) {
408
+ const { KeyValueStore } = await import('crawlee');
409
+ const store = await KeyValueStore.open();
410
+ const metamorphKey = `INPUT-METAMORPH-${count}`;
411
+ const metamorphInput = await store.getValue(metamorphKey);
412
+ if (metamorphInput !== null && metamorphInput !== undefined) {
413
+ return metamorphInput;
414
+ }
415
+ // Fall through to regular INPUT if metamorph key not found
416
+ }
417
+ }
418
+ let input = null;
419
+ const raw = process.env.ACTOR_INPUT;
420
+ if (raw) {
421
+ // Try base64 decode first (server passes input as base64)
422
+ try {
423
+ const decoded = Buffer.from(raw, 'base64').toString('utf-8');
424
+ input = JSON.parse(decoded);
425
+ }
426
+ catch {
427
+ // If base64 decode fails, try direct JSON parse
428
+ input = JSON.parse(raw);
429
+ }
430
+ }
431
+ else {
432
+ const { KeyValueStore } = await import('crawlee');
433
+ const store = await KeyValueStore.open();
434
+ const inputKey = Actor._actorConfig().get('inputKey', 'INPUT');
435
+ input = await store.getValue(inputKey);
436
+ }
437
+ // Decrypt secret inputs after loading (must happen here, not in init(),
438
+ // because the input needs to be available first)
439
+ if (input && typeof input === 'object' && !Array.isArray(input)) {
440
+ return (0, secrets_js_1.decryptSecretInputs)(input);
441
+ }
442
+ return input;
443
+ }
444
+ static async getInputOrThrow() {
445
+ const input = await Actor.getInput();
446
+ if (input == null) {
447
+ throw new Error('Input does not exist');
448
+ }
449
+ return input;
450
+ }
451
+ /**
452
+ * Synchronously get the actor input (cached from first access).
453
+ * This matches Apify's Actor.input behavior.
454
+ */
455
+ static get input() {
456
+ if (cachedInput !== undefined)
457
+ return cachedInput ?? undefined;
458
+ const raw = process.env.ACTOR_INPUT;
459
+ if (!raw) {
460
+ cachedInput = null;
461
+ return undefined;
462
+ }
463
+ try {
464
+ // Try base64 decode first (server passes input as base64)
465
+ const decoded = Buffer.from(raw, 'base64').toString('utf-8');
466
+ cachedInput = JSON.parse(decoded);
467
+ }
468
+ catch {
469
+ try {
470
+ // Fallback to direct JSON parse
471
+ cachedInput = JSON.parse(raw);
472
+ }
473
+ catch {
474
+ cachedInput = null;
475
+ return undefined;
476
+ }
477
+ }
478
+ // Decrypt secret inputs after loading
479
+ if (cachedInput && typeof cachedInput === 'object' && !Array.isArray(cachedInput)) {
480
+ cachedInput = (0, secrets_js_1.decryptSecretInputs)(cachedInput);
481
+ }
482
+ return cachedInput ?? undefined;
483
+ }
484
+ static async pushData(item, eventName) {
485
+ return Actor.getDefaultInstance().pushData(item, eventName);
486
+ }
487
+ async pushData(item, eventName) {
488
+ Actor._ensureActorInit('pushData');
489
+ // Exact synthetic names only — prefix would block real priced events like apify-foo.
490
+ if (eventName === 'apify-default-dataset-item' ||
491
+ eventName === 'scrapely-default-dataset-item') {
492
+ throw new Error(`Cannot charge for synthetic event '${eventName}' manually`);
493
+ }
494
+ const { Dataset } = await import('crawlee');
495
+ const dataset = await Dataset.open();
496
+ if (eventName === undefined && !this.usesPushDataInterception(dataset)) {
497
+ await dataset.pushData(item);
498
+ return;
499
+ }
500
+ if (this.usesPushDataInterception(dataset)) {
501
+ return this.pushDataViaInterceptedClient(dataset, item, eventName);
502
+ }
503
+ return this.pushDataWithExplicitCharging(dataset, item, eventName);
504
+ }
505
+ usesPushDataInterception(dataset) {
506
+ return Boolean(dataset.client[charging_client_js_1.USES_PUSH_DATA_INTERCEPTION]);
507
+ }
508
+ async pushDataViaInterceptedClient(dataset, item, eventName) {
509
+ const context = { eventName };
510
+ await charging_client_js_1.pushDataChargingContext.run(context, async () => {
511
+ await dataset.pushData(item);
512
+ });
513
+ return (context.chargeResult ?? {
514
+ eventChargeLimitReached: false,
515
+ chargedCount: 0,
516
+ chargeableWithinLimit: {},
517
+ });
518
+ }
519
+ async pushDataWithExplicitCharging(dataset, items, explicitEventName) {
520
+ if (!this.initialized && explicitEventName === undefined) {
521
+ await dataset.pushData(items);
522
+ return;
523
+ }
524
+ const isDefaultDataset = dataset.id === this.config.get('defaultDatasetId');
525
+ return (0, charging_js_1.pushDataAndCharge)({
526
+ chargingManager: this.chargingManager,
527
+ items,
528
+ eventName: explicitEventName,
529
+ isDefaultDataset,
530
+ pushFn: async (limitedItems) => dataset.pushData(limitedItems),
531
+ });
532
+ }
533
+ static async getValue(key) {
534
+ Actor._ensureActorInit('getValue');
535
+ const { KeyValueStore } = await import('crawlee');
536
+ const store = await KeyValueStore.open();
537
+ return store.getValue(key);
538
+ }
539
+ /**
540
+ * Stores or deletes a value in the default KeyValueStore associated with the current Actor run.
541
+ *
542
+ * This is just a convenient shortcut for KeyValueStore.setValue.
543
+ *
544
+ * **Example:**
545
+ * ```typescript
546
+ * // Store an object (serialized to JSON)
547
+ * await Actor.setValue('OUTPUT', { foo: 'bar' });
548
+ *
549
+ * // Delete a record
550
+ * await Actor.setValue('my-key', null);
551
+ * ```
552
+ *
553
+ * @param key - Unique record key
554
+ * @param value - Record data. If null, the record is deleted. If no contentType is specified,
555
+ * value can be any JSON-serializable object. If contentType is set, value must be String or Buffer.
556
+ * @param options - Record options
557
+ */
558
+ static async setValue(key, value, options) {
559
+ Actor._ensureActorInit('setValue');
560
+ const { KeyValueStore } = await import('crawlee');
561
+ const store = await KeyValueStore.open();
562
+ await store.setValue(key, value, options);
563
+ }
564
+ /**
565
+ * Helper to resolve storage identifier to a string ID or name.
566
+ * Supports: string, { id: '...' }, { name: '...' }, { alias: '...' }
567
+ */
568
+ static _resolveStorageIdentifier(identifier, storageType, actorConfig = Actor._actorConfig()) {
569
+ if (!identifier)
570
+ return identifier;
571
+ if (typeof identifier === 'string') {
572
+ return identifier;
573
+ }
574
+ if ('id' in identifier) {
575
+ return identifier.id;
576
+ }
577
+ if ('name' in identifier) {
578
+ return identifier.name;
579
+ }
580
+ if ('alias' in identifier) {
581
+ // Try to resolve alias from ACTOR_STORAGES_JSON
582
+ const raw = actorConfig.get('actorStoragesJson');
583
+ if (raw) {
584
+ try {
585
+ let parsed;
586
+ try {
587
+ parsed = JSON.parse(raw);
588
+ }
589
+ catch {
590
+ const decoded = Buffer.from(raw, 'base64').toString('utf-8');
591
+ parsed = JSON.parse(decoded);
592
+ }
593
+ const id = parsed?.[storageType]?.[identifier.alias];
594
+ if (typeof id === 'string' && id)
595
+ return id;
596
+ }
597
+ catch {
598
+ // Fall through to return alias as name
599
+ }
600
+ }
601
+ // Running outside platform: fall back to alias behaving like a name
602
+ return identifier.alias;
603
+ }
604
+ return null;
605
+ }
606
+ static _buildStorageOpenOptions(actorConfig, options) {
607
+ if (!options)
608
+ return undefined;
609
+ const config = options.config ?? actorConfig;
610
+ const storageOptions = { config };
611
+ if (options.forceCloud) {
612
+ const factory = (0, patch_storage_js_1.getApiStorageFactory)();
613
+ if (factory) {
614
+ storageOptions.storageClient = factory;
615
+ }
616
+ else {
617
+ const apiUrl = config.get('apiBaseUrl');
618
+ const token = config.get('token');
619
+ const runId = config.get('actorRunId');
620
+ if (apiUrl && token && runId) {
621
+ storageOptions.storageClient = new api_storage_client_js_1.ApiStorageClientFactory({ apiUrl, token, runId });
622
+ }
623
+ else {
624
+ console.warn('[Scrapely] forceCloud was requested but no API storage credentials are '
625
+ + 'available (apiBaseUrl/token/actorRunId) and no storage factory is '
626
+ + 'initialized — falling back to the default (local) storage client.');
627
+ }
628
+ }
629
+ }
630
+ return storageOptions;
631
+ }
632
+ static async _openDataset(actorConfig, nameOrIdOrOptions, openOptions) {
633
+ const { Dataset } = await import('crawlee');
634
+ const resolved = Actor._resolveStorageIdentifier(nameOrIdOrOptions, 'datasets', actorConfig);
635
+ const storageOpts = Actor._buildStorageOpenOptions(actorConfig, openOptions);
636
+ return Dataset.open(resolved, storageOpts);
637
+ }
638
+ static async _openKeyValueStore(actorConfig, nameOrIdOrOptions, openOptions) {
639
+ const { KeyValueStore } = await import('crawlee');
640
+ const resolved = Actor._resolveStorageIdentifier(nameOrIdOrOptions, 'keyValueStores', actorConfig);
641
+ const storageOpts = Actor._buildStorageOpenOptions(actorConfig, openOptions);
642
+ return KeyValueStore.open(resolved, storageOpts);
643
+ }
644
+ static async _openRequestQueue(actorConfig, nameOrIdOrOptions, openOptions) {
645
+ const { RequestQueue } = await import('crawlee');
646
+ const resolved = Actor._resolveStorageIdentifier(nameOrIdOrOptions, 'requestQueues', actorConfig);
647
+ const storageOpts = Actor._buildStorageOpenOptions(actorConfig, openOptions);
648
+ return RequestQueue.open(resolved, storageOpts);
649
+ }
650
+ static async openDataset(nameOrIdOrOptions, openOptions) {
651
+ Actor._ensureActorInit('openDataset');
652
+ return Actor._openDataset(Actor._actorConfig(), nameOrIdOrOptions, openOptions);
653
+ }
654
+ static async openKeyValueStore(nameOrIdOrOptions, openOptions) {
655
+ Actor._ensureActorInit('openKeyValueStore');
656
+ return Actor._openKeyValueStore(Actor._actorConfig(), nameOrIdOrOptions, openOptions);
657
+ }
658
+ static async openRequestQueue(nameOrIdOrOptions, openOptions) {
659
+ Actor._ensureActorInit('openRequestQueue');
660
+ return Actor._openRequestQueue(Actor._actorConfig(), nameOrIdOrOptions, openOptions);
661
+ }
662
+ static isAtHome() {
663
+ return !!Actor._actorConfig().get('isAtHome');
664
+ }
665
+ /**
666
+ * Returns true when the Actor run was started in standby mode
667
+ * (Python SDK `Actor.is_standby` parity; detected via `metaOrigin === 'STANDBY'`).
668
+ */
669
+ static isStandby() {
670
+ return Actor._actorConfig().get('metaOrigin') === 'STANDBY';
671
+ }
672
+ /**
673
+ * Returns a new {@apilink ApifyEnv} object which contains information parsed from all the environment variables.
674
+ *
675
+ * For the list of the environment variables, see the Actor documentation.
676
+ * If some variables are not defined or are invalid, the corresponding value in the resulting object will be null.
677
+ *
678
+ * @returns Parsed environment variables with proper types
679
+ */
680
+ static getEnv() {
681
+ const env = process.env;
682
+ // Helper to parse dates
683
+ const parseDate = (value) => {
684
+ if (!value)
685
+ return null;
686
+ const unix = Date.parse(value);
687
+ return unix > 0 ? new Date(unix) : null;
688
+ };
689
+ // Helper to parse integers
690
+ const parseIntOrNull = (value) => {
691
+ if (!value)
692
+ return null;
693
+ const parsed = parseInt(value, 10);
694
+ return isNaN(parsed) ? null : parsed;
695
+ };
696
+ // Helper to parse floats
697
+ const parseFloatOrNull = (value) => {
698
+ if (!value)
699
+ return null;
700
+ const parsed = parseFloat(value);
701
+ return isNaN(parsed) ? null : parsed;
702
+ };
703
+ // Helper for disableOutdatedWarning (1 or null)
704
+ const parseDisableWarning = (value) => {
705
+ return value === '1' ? 1 : null;
706
+ };
707
+ return {
708
+ // Apify-compatible fields (in alphabetical order as per Apify interface)
709
+ actorId: env.ACTOR_ID || null,
710
+ actorBuildId: env.ACTOR_BUILD_ID || null,
711
+ actorBuildNumber: env.ACTOR_BUILD_NUMBER || null,
712
+ actorBuildTags: env.ACTOR_BUILD_TAGS || null,
713
+ actorEventsWsUrl: env.ACTOR_EVENTS_WS_URL || env.ACTOR_EVENTS_WEBSOCKET_URL || null,
714
+ actorFullName: env.ACTOR_FULL_NAME || null,
715
+ actorMaxPaidDatasetItems: parseIntOrNull(env.ACTOR_MAX_PAID_DATASET_ITEMS),
716
+ actorPermissionLevel: env.ACTOR_PERMISSION_LEVEL || null,
717
+ actorRunId: env.ACTOR_RUN_ID || null,
718
+ actorTaskId: env.ACTOR_TASK_ID || null,
719
+ apiBaseUrl: env.APIFY_API_BASE_URL || env.SCRAPELY_API_URL || null,
720
+ apiPublicBaseUrl: env.APIFY_API_PUBLIC_BASE_URL || null,
721
+ chromeExecutablePath: env.APIFY_CHROME_EXECUTABLE_PATH || null,
722
+ containerPort: parseIntOrNull(env.ACTOR_WEB_SERVER_PORT),
723
+ containerUrl: env.ACTOR_WEB_SERVER_URL || null,
724
+ dedicatedCpus: env.ACTOR_DEDICATED_CPUS || null,
725
+ defaultDatasetId: env.ACTOR_DEFAULT_DATASET_ID || null,
726
+ defaultKeyValueStoreId: env.ACTOR_DEFAULT_KEY_VALUE_STORE_ID || null,
727
+ defaultRequestQueueId: env.ACTOR_DEFAULT_REQUEST_QUEUE_ID || null,
728
+ disableOutdatedWarning: parseDisableWarning(env.APIFY_DISABLE_OUTDATED_WARNING),
729
+ fact: env.APIFY_FACT || null,
730
+ headless: env.APIFY_HEADLESS || null,
731
+ inputKey: env.ACTOR_INPUT_KEY || null,
732
+ inputSecretsPrivateKeyFile: env.APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE || null,
733
+ inputSecretsPrivateKeyPassphrase: env.APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE || null,
734
+ isAtHome: env.APIFY_IS_AT_HOME || env.SCRAPELY_IS_AT_HOME || null,
735
+ localStorageDir: env.CRAWLEE_STORAGE_DIR || env.APIFY_LOCAL_STORAGE_DIR || null,
736
+ logFormat: env.APIFY_LOG_FORMAT || null,
737
+ logLevel: env.APIFY_LOG_LEVEL || null,
738
+ memoryMbytes: parseIntOrNull(env.ACTOR_MEMORY_MBYTES),
739
+ metaOrigin: env.APIFY_META_ORIGIN || env.ACTOR_META_ORIGIN || null,
740
+ proxyHostname: env.APIFY_PROXY_HOSTNAME || null,
741
+ proxyPassword: env.APIFY_PROXY_PASSWORD || null,
742
+ proxyPort: env.APIFY_PROXY_PORT || null,
743
+ proxyStatusUrl: env.APIFY_PROXY_STATUS_URL || null,
744
+ sdkLatestVersion: null, // Not available from env
745
+ startedAt: parseDate(env.ACTOR_STARTED_AT),
746
+ systemInfoIntervalMillis: env.APIFY_SYSTEM_INFO_INTERVAL_MILLIS || null,
747
+ timeoutAt: parseDate(env.ACTOR_TIMEOUT_AT),
748
+ token: env.APIFY_TOKEN || env.SCRAPELY_TOKEN || null,
749
+ userId: env.APIFY_USER_ID || null,
750
+ userIsPaying: env.APIFY_USER_IS_PAYING || null,
751
+ workflowKey: env.ACTOR_WORKFLOW_KEY || null,
752
+ // Scrapely-specific extensions
753
+ maxTotalChargeUsd: parseFloatOrNull(env.ACTOR_MAX_TOTAL_CHARGE_USD),
754
+ userPlan: env.SCRAPELY_USER_PLAN || null,
755
+ restartOnError: env.ACTOR_RESTART_ON_ERROR || null,
756
+ };
757
+ }
758
+ /**
759
+ * Full technical name of the actor (owner-username/actor-name).
760
+ */
761
+ static get fullName() {
762
+ return process.env.ACTOR_FULL_NAME || undefined;
763
+ }
764
+ /**
765
+ * ID of the Actor task, or undefined if running outside a task.
766
+ */
767
+ static get taskId() {
768
+ return Actor._actorConfig().get('actorTaskId') || undefined;
769
+ }
770
+ /**
771
+ * ID of the Actor build used in the run.
772
+ */
773
+ static get buildId() {
774
+ return process.env.ACTOR_BUILD_ID || undefined;
775
+ }
776
+ /**
777
+ * Build number of the Actor build used in the run.
778
+ */
779
+ static get buildNumber() {
780
+ return process.env.ACTOR_BUILD_NUMBER || undefined;
781
+ }
782
+ /**
783
+ * Comma-separated list of tags of the Actor build used in the run.
784
+ */
785
+ static get buildTags() {
786
+ return (process.env.ACTOR_BUILD_TAGS || '')
787
+ .split(',')
788
+ .map(t => t.trim())
789
+ .filter(t => t.length > 0);
790
+ }
791
+ /**
792
+ * Date when the Actor was started (ISO 8601 UTC).
793
+ */
794
+ static get startedAt() {
795
+ return process.env.ACTOR_STARTED_AT || undefined;
796
+ }
797
+ /**
798
+ * Date when the Actor will time out (ISO 8601 UTC).
799
+ */
800
+ static get timeoutAt() {
801
+ return process.env.ACTOR_TIMEOUT_AT || undefined;
802
+ }
803
+ /**
804
+ * API token of the user who started the Actor.
805
+ */
806
+ static get token() {
807
+ return Actor._actorConfig().get('token') || undefined;
808
+ }
809
+ /**
810
+ * ID of the user who started the Actor.
811
+ */
812
+ static get userId() {
813
+ return Actor._actorConfig().get('userId') || undefined;
814
+ }
815
+ /**
816
+ * Whether the user who started the Actor is a paying user.
817
+ */
818
+ static get isPaying() {
819
+ return Actor._actorConfig().get('userIsPaying') === true;
820
+ }
821
+ /**
822
+ * Public URL of the Apify API. May be used to interact with the platform programmatically.
823
+ */
824
+ static get apiPublicBaseUrl() {
825
+ return Actor._actorConfig().get('apiPublicBaseUrl') || undefined;
826
+ }
827
+ /**
828
+ * Get a single environment variable value by name.
829
+ * This is an Apify SDK-compatible alias for process.env[key].
830
+ */
831
+ static getenv(key) {
832
+ return process.env[key];
833
+ }
834
+ on(event, listener) {
835
+ this.config.getEventManager().on(event, listener);
836
+ }
837
+ off(event, listener) {
838
+ this.config.getEventManager().off(event, listener);
839
+ }
840
+ static on(event, listener) {
841
+ Actor.getDefaultInstance().on(event, listener);
842
+ }
843
+ static off(event, listener) {
844
+ Actor.getDefaultInstance().off(event, listener);
845
+ }
846
+ /**
847
+ * Closes the event manager, disconnects platform events, and flushes storage.
848
+ * Called automatically at the end of {@link Actor.main} and {@link Actor.exit}.
849
+ */
850
+ static _fatalShutdownStarted = false;
851
+ /** One-shot bounded reconciliation on fatal process errors. */
852
+ static installFatalChargeHandlers() {
853
+ if (Actor._fatalHandlersInstalled)
854
+ return;
855
+ Actor._fatalHandlersInstalled = true;
856
+ const runOnce = (reason, err) => {
857
+ if (Actor._fatalShutdownStarted)
858
+ return;
859
+ Actor._fatalShutdownStarted = true;
860
+ if (err)
861
+ console.error(`[Scrapely] Fatal ${reason}:`, err);
862
+ else
863
+ console.warn(`[Scrapely] Fatal ${reason} — attempting bounded charge flush`);
864
+ // Independent hard-exit timer: a hung flush must not leave a
865
+ // corrupted process alive. unref so the timer alone keeps nothing running.
866
+ const hardExit = setTimeout(() => process.exit(1), 10_000);
867
+ hardExit.unref?.();
868
+ void Actor._shutdownActor(5)
869
+ .catch(() => undefined)
870
+ .finally(() => {
871
+ // Registering an uncaughtException listener SUPPRESSES Node's
872
+ // default termination — a fatal error MUST terminate the process
873
+ // so the platform can retry the run from a clean state.
874
+ process.exit(1);
875
+ });
876
+ };
877
+ process.once('uncaughtException', (err) => runOnce('uncaughtException', err));
878
+ process.once('unhandledRejection', (err) => runOnce('unhandledRejection', err));
879
+ }
880
+ static async _shutdownActor(timeoutSecs = 30) {
881
+ const actor = Actor.getDefaultInstance();
882
+ // Tear down whenever init() started installing lifecycle state — including
883
+ // a partially initialized Actor whose init() failed. The public
884
+ // `initialized` flag stays false in that case, but the events WebSocket,
885
+ // storage factory, event manager, and handlers exist and must not leak
886
+ // into a retry, so that flag must NOT be used to skip this cleanup.
887
+ if (!actor._lifecycleStarted) {
888
+ return;
889
+ }
890
+ if (gracefulShutdownHandlers.aborting) {
891
+ Actor.off('aborting', gracefulShutdownHandlers.aborting);
892
+ }
893
+ if (gracefulShutdownHandlers.migrating) {
894
+ Actor.off('migrating', gracefulShutdownHandlers.migrating);
895
+ }
896
+ if (signalExitHandlers.sigint) {
897
+ process.removeListener('SIGINT', signalExitHandlers.sigint);
898
+ }
899
+ if (signalExitHandlers.sigterm) {
900
+ process.removeListener('SIGTERM', signalExitHandlers.sigterm);
901
+ }
902
+ // Best-effort: drain unresolved PPE charge intents under a hard deadline.
903
+ // timeoutSecs bounds the flush so hung network cannot block exit forever.
904
+ const flushDeadlineMs = Math.min(Math.max(1, timeoutSecs) * 1000, 30_000);
905
+ // Only a fully initialized run can have admitted charges; a partial init
906
+ // has none, and closing the manager here would also reject charges made
907
+ // after a successful init retry on the same instance.
908
+ if (actor.initialized) {
909
+ try {
910
+ const cm = actor.chargingManager;
911
+ // F6: reject NEW charges before the final flush so nothing can be
912
+ // admitted after it while teardown is closing storages/credentials.
913
+ if (cm && typeof cm.close === 'function') {
914
+ cm.close();
915
+ }
916
+ if (cm && typeof cm.flushPending === 'function') {
917
+ await Promise.race([
918
+ cm.flushPending(64, flushDeadlineMs),
919
+ new Promise((resolve) => setTimeout(resolve, flushDeadlineMs)),
920
+ ]);
921
+ }
922
+ }
923
+ catch (err) {
924
+ console.warn(`[Scrapely] ChargingManager.flushPending failed during shutdown: ${err.message}`);
925
+ }
926
+ }
927
+ const eventManager = Actor._actorConfig().getEventManager();
928
+ await eventManager.close();
929
+ (0, events_client_js_1.disconnectEvents)();
930
+ (0, patch_storage_js_1.disconnectStorage)();
931
+ // Brief grace for in-flight I/O after flush (bounded).
932
+ await new Promise((resolve) => {
933
+ setTimeout(resolve, Math.min(1000, flushDeadlineMs));
934
+ });
935
+ }
936
+ /**
937
+ * Sets the status message for the current Actor run.
938
+ *
939
+ * @param message - The status message to set
940
+ * @param options - Optional settings for the status message
941
+ * @returns The updated ActorRun object, or an empty object if running locally
942
+ */
943
+ static async setStatusMessage(message, options) {
944
+ const { isStatusMessageTerminal, level } = options || {};
945
+ Actor._ensureActorInit('setStatusMessage');
946
+ const loggedStatusMessage = `[Status message]: ${message}`;
947
+ switch (level) {
948
+ case 'DEBUG':
949
+ console.debug(loggedStatusMessage);
950
+ break;
951
+ case 'WARNING':
952
+ console.warn(loggedStatusMessage);
953
+ break;
954
+ case 'ERROR':
955
+ console.error(loggedStatusMessage);
956
+ break;
957
+ default:
958
+ console.log(loggedStatusMessage);
959
+ break;
960
+ }
961
+ const config = Actor._actorConfig();
962
+ const client = config.getStorageClient();
963
+ if (typeof client.setStatusMessage === 'function') {
964
+ await Promise.race([
965
+ client.setStatusMessage(message, { isStatusMessageTerminal, level }),
966
+ new Promise((_, reject) => setTimeout(() => reject(new Error('Setting status message timed out after 1s')), 1000)),
967
+ ]).catch((err) => {
968
+ console.warn(`[Scrapely] ${err.message}`);
969
+ });
970
+ }
971
+ const apiUrl = config.get('apiBaseUrl');
972
+ const token = config.get('token');
973
+ const runId = config.get('actorRunId');
974
+ if (!apiUrl || !token || !runId) {
975
+ return {};
976
+ }
977
+ const response = await fetch(`${apiUrl}/v2/actor-runs/${runId}`, {
978
+ method: 'PUT',
979
+ headers: {
980
+ 'Content-Type': 'application/json',
981
+ Authorization: `Bearer ${token}`,
982
+ },
983
+ body: JSON.stringify({
984
+ statusMessage: message,
985
+ isStatusMessageTerminal,
986
+ }),
987
+ });
988
+ if (!response.ok) {
989
+ const error = await response.text();
990
+ throw new Error(`Failed to update status message: ${response.status} ${error}`);
991
+ }
992
+ const body = (await response.json());
993
+ return (body.data ?? {});
994
+ }
995
+ /**
996
+ * Creates a proxy configuration and returns a promise resolving to an instance
997
+ * of the ProxyConfiguration class that is already initialized.
998
+ *
999
+ * Configures connection to a proxy server with the provided options. Proxy servers
1000
+ * are used to prevent target websites from blocking your crawlers based on IP address
1001
+ * rate limits or blacklists. Setting proxy configuration in your crawlers automatically
1002
+ * configures them to use the selected proxies for all connections.
1003
+ *
1004
+ * **Example usage:**
1005
+ * ```javascript
1006
+ * const proxyConfiguration = await Actor.createProxyConfiguration({
1007
+ * proxyUrls: ['http://user:pass@proxy1.example.com:8080'],
1008
+ * });
1009
+ *
1010
+ * const crawler = new CheerioCrawler({
1011
+ * proxyConfiguration,
1012
+ * requestHandler({ proxyInfo }) {
1013
+ * const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
1014
+ * },
1015
+ * });
1016
+ * ```
1017
+ *
1018
+ * For compatibility with existing Actor Input UI (Input Schema), this function returns
1019
+ * `undefined` when `{ useApifyProxy: false }` is passed.
1020
+ *
1021
+ * @param options - Proxy configuration options
1022
+ * @returns Promise resolving to ProxyConfiguration instance, or undefined
1023
+ */
1024
+ static async createProxyConfiguration(options = {}) {
1025
+ return (0, proxy_configuration_js_1.createProxyConfiguration)(options);
1026
+ }
1027
+ /**
1028
+ * Creates a Scrapely remote browser configuration (location and
1029
+ * fingerprint options). Pass it to browser crawlers next to
1030
+ * `proxyConfiguration`:
1031
+ *
1032
+ * ```javascript
1033
+ * const browserConfiguration = await Actor.createBrowserConfiguration({ countryCode: 'US', os: 'windows' });
1034
+ * const crawler = new PlaywrightCrawler({ browserConfiguration, requestHandler });
1035
+ * ```
1036
+ *
1037
+ * Ignored outside the Scrapely remote browser (local runs), so the same
1038
+ * code runs locally and on the platform.
1039
+ */
1040
+ static async createBrowserConfiguration(options = {}) {
1041
+ return (0, browser_configuration_js_1.createBrowserConfiguration)(options);
1042
+ }
1043
+ /** @internal */
1044
+ static _instance;
1045
+ /**
1046
+ * Get the default Actor instance (singleton pattern for Apify SDK compatibility).
1047
+ * @returns The default Actor instance
1048
+ */
1049
+ static getDefaultInstance() {
1050
+ Actor._instance ??= new Actor();
1051
+ return Actor._instance;
1052
+ }
1053
+ /**
1054
+ * Default Configuration instance (Apify-compatible static accessor).
1055
+ * Used after Actor.init() without new Actor(), e.g. Actor.config.get('standbyPort').
1056
+ */
1057
+ static get config() {
1058
+ return Actor.getDefaultInstance().config;
1059
+ }
1060
+ /**
1061
+ * Default API client instance for the Scrapely platform.
1062
+ */
1063
+ static get apifyClient() {
1064
+ return Actor.getDefaultInstance().apifyClient;
1065
+ }
1066
+ /**
1067
+ * Returns a new Apify API client pointed at the Scrapely platform.
1068
+ */
1069
+ newClient(options) {
1070
+ const config = this.config;
1071
+ const clientOptions = {
1072
+ token: options?.token ?? config.get('token') ?? '',
1073
+ // Fall back to the client's own default when the configuration has no value,
1074
+ // instead of forcing an empty string (which would produce a relative `/v2` base URL).
1075
+ baseUrl: options?.baseUrl ?? config.get('apiBaseUrl'),
1076
+ // Public URLs (getRecordPublicUrl, createItemsPublicUrl) must not be minted
1077
+ // against a different host than the API the client talks to. Follow an explicit
1078
+ // per-call base URL override, then the configured public base URL.
1079
+ publicBaseUrl: options?.baseUrl ?? config.get('apiPublicBaseUrl'),
1080
+ };
1081
+ if (options?.token || options?.baseUrl) {
1082
+ return (0, charging_client_js_1.createChargingClient)(clientOptions, this);
1083
+ }
1084
+ if (apifyClientInstance) {
1085
+ return apifyClientInstance;
1086
+ }
1087
+ apifyClientInstance = (0, charging_client_js_1.createChargingClient)(clientOptions, this);
1088
+ return apifyClientInstance;
1089
+ }
1090
+ static newClient(options) {
1091
+ return Actor.getDefaultInstance().newClient(options);
1092
+ }
1093
+ /** Apify-compatible instance API — delegates to {@link Actor.charge}. */
1094
+ async charge(options, count) {
1095
+ return Actor.charge(options, count);
1096
+ }
1097
+ /**
1098
+ * Charge for a custom pricing event (pay-per-event Actors).
1099
+ */
1100
+ static async charge(options, count) {
1101
+ Actor._ensureActorInit('charge');
1102
+ const chargeOptions = typeof options === 'string' ? { eventName: options, count: count ?? 1 } : options;
1103
+ return Actor.getDefaultInstance().chargingManager.charge(chargeOptions);
1104
+ }
1105
+ static getChargingManager() {
1106
+ return Actor.getDefaultInstance().chargingManager;
1107
+ }
1108
+ getChargingManager() {
1109
+ return this.chargingManager;
1110
+ }
1111
+ /**
1112
+ * Metamorph the current actor run into a run of another actor.
1113
+ * This follows Apify's metamorph behavior:
1114
+ * 1. Stores new input under INPUT-METAMORPH-{n} key in the default KVS
1115
+ * 2. Signals the server to swap the container to the target actor's image
1116
+ * 3. Exits the current process (the container will be replaced)
1117
+ *
1118
+ * @param targetActorId - The ID or username/actorName of the target actor
1119
+ * @param input - Optional input for the target actor (uses current input if not provided)
1120
+ * @param options - Optional build tag and content type for the target actor
1121
+ */
1122
+ static async metamorph(targetActorId, input, options) {
1123
+ const config = Actor._actorConfig();
1124
+ const apiUrl = config.get('apiBaseUrl');
1125
+ const token = config.get('token');
1126
+ const runId = config.get('actorRunId');
1127
+ if (!apiUrl || !token || !runId) {
1128
+ throw new Error('Metamorph requires apiBaseUrl, token, and actorRunId in configuration');
1129
+ }
1130
+ // Wire format (server contract, Python SDK parity): the server copies the
1131
+ // current INPUT to INPUT-METAMORPH-<n> itself and tracks the metamorph
1132
+ // count in run stats; the new input (if any) is read from the request
1133
+ // body's `input` field and written to INPUT. `inputKey` is NOT part of
1134
+ // the contract — the old client-side KVS write made the server overwrite
1135
+ // our value with the stale INPUT, silently losing the new input.
1136
+ const body = { targetActorId };
1137
+ if (options?.build) {
1138
+ body.build = options.build;
1139
+ }
1140
+ if (input !== undefined) {
1141
+ body.input = input;
1142
+ }
1143
+ console.log(`[Scrapely] Metamorphosing to actor ${targetActorId}`);
1144
+ // Signal the server to perform the metamorph
1145
+ try {
1146
+ const response = await fetch(`${apiUrl}/v2/actor-runs/${runId}/metamorph`, {
1147
+ method: 'POST',
1148
+ headers: {
1149
+ 'Content-Type': 'application/json',
1150
+ 'Authorization': `Bearer ${token}`,
1151
+ },
1152
+ body: JSON.stringify(body),
1153
+ });
1154
+ if (!response.ok) {
1155
+ const error = await response.text();
1156
+ throw new Error(`Metamorph request failed: ${response.status} ${error}`);
1157
+ }
1158
+ console.log('[Scrapely] Metamorph request accepted, exiting...');
1159
+ }
1160
+ catch (err) {
1161
+ console.error(`[Scrapely] Metamorph failed: ${err.message}`);
1162
+ throw err;
1163
+ }
1164
+ // Flush pending PPE intents before hard exit (same class of loss as SIGTERM).
1165
+ try {
1166
+ await Actor._shutdownActor(5);
1167
+ }
1168
+ catch (err) {
1169
+ console.warn(`[Scrapely] flushPending during metamorph failed: ${err.message}`);
1170
+ }
1171
+ // Exit so the worker can restart the container with the target actor.
1172
+ process.exit(0);
1173
+ }
1174
+ /**
1175
+ * Persist runtime state across server migrations.
1176
+ *
1177
+ * Long-running Actor jobs may need to migrate between servers.
1178
+ * Without state persistence, your job's progress is lost during
1179
+ * migration, causing it to restart from the beginning on the new server.
1180
+ *
1181
+ * This method automatically saves your state to the key-value store
1182
+ * whenever a `persistState` event occurs (every 60 seconds and on exit),
1183
+ * and restores it when the Actor restarts after a migration.
1184
+ *
1185
+ * The state must be JSON-serializable (values must survive JSON.stringify/parse).
1186
+ * For short-running Actors (less than a few minutes), state persistence is
1187
+ * typically unnecessary.
1188
+ *
1189
+ * @param nameOrInitialState - Either the state key name (string) or default state object
1190
+ * @param defaultValue - Default state used when no previously persisted state exists (when first param is string)
1191
+ * @param options - Options for state persistence
1192
+ * @returns The current state object (mutate it directly; it saves on persistState events)
1193
+ */
1194
+ static async useState(nameOrInitialState, defaultValue = {}, options) {
1195
+ Actor._ensureActorInit('useState');
1196
+ let stateKey;
1197
+ let initialState;
1198
+ if (typeof nameOrInitialState === 'string') {
1199
+ stateKey = nameOrInitialState;
1200
+ initialState = defaultValue;
1201
+ }
1202
+ else if (nameOrInitialState !== undefined) {
1203
+ // Legacy: useState(initialState) stores under STATE
1204
+ stateKey = 'STATE';
1205
+ initialState = nameOrInitialState;
1206
+ }
1207
+ else {
1208
+ stateKey = 'APIFY_GLOBAL_STATE';
1209
+ initialState = defaultValue;
1210
+ }
1211
+ const { KeyValueStore } = await import('crawlee');
1212
+ const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
1213
+ config: options?.config ?? Actor._actorConfig(),
1214
+ });
1215
+ return kvStore.getAutoSavedValue(stateKey, initialState);
1216
+ }
1217
+ /**
1218
+ * Reboot the Actor run.
1219
+ *
1220
+ * This method is designed to be called in the `migrating` event handler
1221
+ * to speed up the migration process and ensure state consistency.
1222
+ *
1223
+ * After calling this, the Actor container will be stopped and restarted
1224
+ * on a new server. Any state persisted via Actor.useState() will be
1225
+ * automatically restored.
1226
+ *
1227
+ * This is an Apify-compatible method.
1228
+ *
1229
+ * @param options - Reboot options
1230
+ */
1231
+ static async reboot(options) {
1232
+ console.log('[Scrapely] Rebooting...');
1233
+ Actor._ensureActorInit('reboot');
1234
+ // Emit persistState to ensure state is saved before reboot
1235
+ Actor._actorConfig().getEventManager().emit('persistState', { isMigrating: true });
1236
+ // Wait a moment for state to be persisted
1237
+ await new Promise((resolve) => setTimeout(resolve, 1000));
1238
+ // Call the reboot endpoint to signal the worker
1239
+ const config = Actor._actorConfig();
1240
+ const apiUrl = config.get('apiBaseUrl') || 'http://localhost:3000';
1241
+ const token = config.get('token') || '';
1242
+ const runId = config.get('actorRunId') || '';
1243
+ try {
1244
+ const response = await fetch(`${apiUrl}/v2/actor-runs/${runId}/reboot`, {
1245
+ method: 'POST',
1246
+ headers: {
1247
+ 'Authorization': `Bearer ${token}`,
1248
+ 'Content-Type': 'application/json',
1249
+ },
1250
+ });
1251
+ if (!response.ok) {
1252
+ const error = await response.text();
1253
+ throw new Error(`Reboot request failed: ${response.status} ${error}`);
1254
+ }
1255
+ console.log(`[Scrapely] Reboot request accepted (${response.status})`);
1256
+ // Wait briefly for the reboot signal to be processed, then exit
1257
+ const sleepMillis = options?.customAfterSleepMillis ?? Actor._actorConfig().get('metamorphAfterSleepMillis', 1000);
1258
+ await new Promise((resolve) => setTimeout(resolve, sleepMillis));
1259
+ }
1260
+ catch (err) {
1261
+ console.error(`[Scrapely] Failed to send reboot request: ${err.message}`);
1262
+ throw err;
1263
+ }
1264
+ // Flush pending PPE intents before hard exit (same class of loss as SIGTERM/metamorph).
1265
+ try {
1266
+ await Actor._shutdownActor(5);
1267
+ }
1268
+ catch (err) {
1269
+ console.warn(`[Scrapely] flushPending during reboot failed: ${err.message}`);
1270
+ }
1271
+ // Exit so the worker can restart the container (Apify-compatible).
1272
+ process.exit(0);
1273
+ }
1274
+ /**
1275
+ * Abort an Actor run on the Scrapely platform.
1276
+ *
1277
+ * @param runId - The ID of the run to abort. Defaults to the current run.
1278
+ * @param options - Abort options
1279
+ * @returns The ActorRun object with details about the aborted run
1280
+ */
1281
+ static async abort(runId, options = {}) {
1282
+ const targetRunId = runId ?? Actor._actorConfig().get('actorRunId');
1283
+ if (!targetRunId) {
1284
+ throw new Error('Actor.abort() requires a run ID');
1285
+ }
1286
+ const { token, statusMessage, ...rest } = options;
1287
+ const client = token ? Actor.newClient({ token }) : Actor.apifyClient;
1288
+ if (statusMessage) {
1289
+ await client.run(targetRunId).update({
1290
+ statusMessage,
1291
+ isStatusMessageTerminal: true,
1292
+ });
1293
+ }
1294
+ return client.run(targetRunId).abort(rest);
1295
+ }
1296
+ static async addWebhook(options) {
1297
+ if (!Actor.isAtHome()) {
1298
+ console.warn('[Scrapely] Actor.addWebhook() is only supported on the platform. Ignoring.');
1299
+ return undefined;
1300
+ }
1301
+ const runId = Actor._actorConfig().get('actorRunId');
1302
+ if (!runId) {
1303
+ throw new Error('Environment variable ACTOR_RUN_ID is not set!');
1304
+ }
1305
+ return Actor.apifyClient.webhooks().create({
1306
+ ...options,
1307
+ eventTypes: options.eventTypes,
1308
+ isAdHoc: true,
1309
+ condition: {
1310
+ actorRunId: runId,
1311
+ },
1312
+ });
1313
+ }
1314
+ /**
1315
+ * Runs an Actor on the platform and waits for it to finish (Apify-compatible instance API).
1316
+ */
1317
+ async call(actorId, input, options = {}) {
1318
+ return Actor.call(actorId, input, options);
1319
+ }
1320
+ static async call(actorId, input, options = {}) {
1321
+ const timeout = resolveCallTimeout(options.timeout);
1322
+ const { token, ...rest } = options;
1323
+ const client = token ? Actor.newClient({ token }) : Actor.apifyClient;
1324
+ return client.actor(actorId).call(input, { ...rest, timeout });
1325
+ }
1326
+ /**
1327
+ * Runs an Actor task on the platform and waits for it to finish (Apify-compatible instance API).
1328
+ */
1329
+ async callTask(taskId, input, options = {}) {
1330
+ return Actor.callTask(taskId, input, options);
1331
+ }
1332
+ static async callTask(taskId, input, options = {}) {
1333
+ const timeout = resolveCallTimeout(options.timeout);
1334
+ const { token, ...rest } = options;
1335
+ const client = token ? Actor.newClient({ token }) : Actor.apifyClient;
1336
+ return client.task(taskId).call(input, { ...rest, timeout });
1337
+ }
1338
+ static async start(actorId, input, options = {}) {
1339
+ const timeout = resolveCallTimeout(options.timeout);
1340
+ const { token, ...rest } = options;
1341
+ const client = token ? Actor.newClient({ token }) : Actor.apifyClient;
1342
+ return client.actor(actorId).start(input, { ...rest, timeout });
1343
+ }
1344
+ // -------------------------------------------------------------------------
1345
+ // Apify-compatible instance API (mirrors static methods on Actor class)
1346
+ // -------------------------------------------------------------------------
1347
+ async main(userFunc, options) {
1348
+ return Actor.main(userFunc, options);
1349
+ }
1350
+ async exit(messageOrOptions, options = {}) {
1351
+ return Actor.exit(messageOrOptions, options);
1352
+ }
1353
+ async fail(messageOrOptions, options = {}) {
1354
+ return Actor.fail(messageOrOptions, options);
1355
+ }
1356
+ async getInput() {
1357
+ this._ensureInit('getInput');
1358
+ return Actor.getInput();
1359
+ }
1360
+ async getInputOrThrow() {
1361
+ this._ensureInit('getInputOrThrow');
1362
+ return Actor.getInputOrThrow();
1363
+ }
1364
+ async getValue(key) {
1365
+ this._ensureInit('getValue');
1366
+ return Actor.getValue(key);
1367
+ }
1368
+ async setValue(key, value, options) {
1369
+ this._ensureInit('setValue');
1370
+ return Actor.setValue(key, value, options);
1371
+ }
1372
+ async openDataset(nameOrIdOrOptions, openOptions) {
1373
+ this._ensureInit('openDataset');
1374
+ return Actor._openDataset(this.config, nameOrIdOrOptions, {
1375
+ ...openOptions,
1376
+ config: openOptions?.config ?? this.config,
1377
+ });
1378
+ }
1379
+ async openKeyValueStore(nameOrIdOrOptions, openOptions) {
1380
+ this._ensureInit('openKeyValueStore');
1381
+ return Actor._openKeyValueStore(this.config, nameOrIdOrOptions, {
1382
+ ...openOptions,
1383
+ config: openOptions?.config ?? this.config,
1384
+ });
1385
+ }
1386
+ async openRequestQueue(nameOrIdOrOptions, openOptions) {
1387
+ this._ensureInit('openRequestQueue');
1388
+ return Actor._openRequestQueue(this.config, nameOrIdOrOptions, {
1389
+ ...openOptions,
1390
+ config: openOptions?.config ?? this.config,
1391
+ });
1392
+ }
1393
+ isAtHome() {
1394
+ return !!this.config.get('isAtHome');
1395
+ }
1396
+ /** Instance mirror of {@link Actor.isStandby} (Python SDK `Actor.is_standby` parity). */
1397
+ isStandby() {
1398
+ return this.config.get('metaOrigin') === 'STANDBY';
1399
+ }
1400
+ getEnv() {
1401
+ return Actor.getEnv();
1402
+ }
1403
+ async setStatusMessage(message, options) {
1404
+ this._ensureInit('setStatusMessage');
1405
+ return Actor.setStatusMessage(message, options);
1406
+ }
1407
+ async createProxyConfiguration(options = {}) {
1408
+ return Actor.createProxyConfiguration(options);
1409
+ }
1410
+ async createBrowserConfiguration(options = {}) {
1411
+ return Actor.createBrowserConfiguration(options);
1412
+ }
1413
+ async metamorph(targetActorId, input, options) {
1414
+ return Actor.metamorph(targetActorId, input, options);
1415
+ }
1416
+ async reboot(options) {
1417
+ return Actor.reboot(options);
1418
+ }
1419
+ async abort(runId, options = {}) {
1420
+ return Actor.abort(runId, options);
1421
+ }
1422
+ async addWebhook(options) {
1423
+ return Actor.addWebhook(options);
1424
+ }
1425
+ async start(actorId, input, options = {}) {
1426
+ return Actor.start(actorId, input, options);
1427
+ }
1428
+ async useState(nameOrInitialState, defaultValue = {}, options) {
1429
+ this._ensureInit('useState');
1430
+ return Actor.useState(nameOrInitialState, defaultValue, {
1431
+ ...options,
1432
+ config: options?.config ?? this.config,
1433
+ });
1434
+ }
1435
+ /**
1436
+ * Apify v2-compatible utility namespace.
1437
+ * Provides helper functions like enqueueLinks that accept the Apify v2 API signature.
1438
+ */
1439
+ static utils = utils_js_1.utils;
1440
+ }
1441
+ exports.Actor = Actor;