scrapely 1.2.33 → 1.2.34

package/dist/storage/rq-lock-script.d.ts

@@ -5,7 +5,7 @@ import type Redis from 'ioredis';
  * which ioredis delivers as [string[], number]. The ZCARD is taken after the
  * script's expired-lock purge and after this call's own locks are added, so it
  * equals the number of active locks held by ANY client — exactly what the
- * Apify `queueHasLockedRequests` spec describes ("locked by any client, either
+ * `queueHasLockedRequests` flag means ("locked by any client, either
  * the one calling the endpoint or a different one"). Callers get the flag from
  * this same atomic reply instead of re-reading the whole locked + pending sets.
  */
@@ -43,7 +43,7 @@ export declare function matchesLockOwner(owner: string, clientKey: string, runId
  * Returns the number of unlocked requests. One round-trip, no race window
  * between ZREM and HDEL, and the whole owners hash never leaves the server.
  */
-export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n  expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n  local requestId = flat[i]\n  local owner = flat[i + 1]\n  -- Per Apify spec: in an Actor run, unlock both (1) requests locked by this\n  -- specific run (clientKey:runId) and (2) requests locked by the same\n  -- clientKey without run context (clientKey), but NOT requests locked by the\n  -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n  if owner == expectedOwner or owner == clientKey\n      or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then\n    redis.call('ZREM', lockedKey, requestId)\n    redis.call('HDEL', lockOwnersKey, requestId)\n    if forefront == '1' then\n      redis.call('ZADD', pendingKey, 0, requestId)\n    end\n    unlockedCount = unlockedCount + 1\n  end\nend\n\nreturn unlockedCount\n";
+export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n  expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n  local requestId = flat[i]\n  local owner = flat[i + 1]\n  -- In an Actor run, unlock both (1) requests locked by this\n  -- specific run (clientKey:runId) and (2) requests locked by the same\n  -- clientKey without run context (clientKey), but NOT requests locked by the\n  -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n  if owner == expectedOwner or owner == clientKey\n      or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then\n    redis.call('ZREM', lockedKey, requestId)\n    redis.call('HDEL', lockOwnersKey, requestId)\n    if forefront == '1' then\n      redis.call('ZADD', pendingKey, 0, requestId)\n    end\n    unlockedCount = unlockedCount + 1\n  end\nend\n\nreturn unlockedCount\n";
 export interface UnlockRequestsKeys {
     lockedKey: string;
     lockOwnersKey: string;

package/dist/storage/rq-lock-script.js

@@ -10,7 +10,7 @@ exports.unlockRequestsAtomic = unlockRequestsAtomic;
  * which ioredis delivers as [string[], number]. The ZCARD is taken after the
  * script's expired-lock purge and after this call's own locks are added, so it
  * equals the number of active locks held by ANY client — exactly what the
- * Apify `queueHasLockedRequests` spec describes ("locked by any client, either
+ * `queueHasLockedRequests` flag means ("locked by any client, either
  * the one calling the endpoint or a different one"). Callers get the flag from
  * this same atomic reply instead of re-reading the whole locked + pending sets.
  */
@@ -104,7 +104,7 @@ local unlockedCount = 0
 for i = 1, #flat, 2 do
   local requestId = flat[i]
   local owner = flat[i + 1]
-  -- Per Apify spec: in an Actor run, unlock both (1) requests locked by this
+  -- In an Actor run, unlock both (1) requests locked by this
   -- specific run (clientKey:runId) and (2) requests locked by the same
   -- clientKey without run context (clientKey), but NOT requests locked by the
   -- same clientKey in DIFFERENT runs (clientKey:otherRunId).

package/dist/storage/run-metadata.js

@@ -4,7 +4,7 @@ exports.RunMetadataManager = void 0;
 const billing_tracker_js_1 = require("./billing-tracker.js");
 /**
  * Number of most-recent runs per user that are retained indefinitely (no TTL).
- * Mirrors Apify's model: the 10 latest runs are always accessible regardless of plan.
+ * The 10 latest runs are always accessible regardless of plan.
  * Older runs beyond this window fall back to the standard plan-based TTL.
  */
 const KEEP_N_RUNS = 10;

package/dist/storage/ttl-manager.d.ts

@@ -3,13 +3,13 @@ import type Redis from 'ioredis';
  * TTL configuration in seconds
  */
 export interface TtlConfig {
-    /** TTL for unnamed datasets (free users) - default 7 days */
+    /** TTL for unnamed datasets (free users) - default 48 hours (Free plan) */
     datasetFree: number;
-    /** TTL for unnamed datasets (paid users) - default 31 days */
+    /** TTL for unnamed datasets (paid users) - default 31 days (Starter, the shortest paid plan) */
     datasetPaid: number;
-    /** TTL for unnamed KV stores (free users) - default 7 days */
+    /** TTL for unnamed KV stores (free users) - default 48 hours (Free plan) */
     kvFree: number;
-    /** TTL for unnamed KV stores (paid users) - default 31 days */
+    /** TTL for unnamed KV stores (paid users) - default 31 days (Starter, the shortest paid plan) */
     kvPaid: number;
     /** Fallback TTL for request queues (safety net for orphaned RQs) - default 24 hours */
     requestQueueFallback: number;
@@ -21,7 +21,7 @@ export interface TtlConfig {
     logMaxSize: number;
 }
 /**
- * Default TTL configuration based on Apify's model
+ * Default TTL configuration
  */
 export declare const DEFAULT_TTL_CONFIG: TtlConfig;
 export type StorageType = 'dataset' | 'kv' | 'rq' | 'logs';
@@ -31,7 +31,8 @@ export type UserPlan = 'free' | 'paid';
  *
  * Retention policy:
  * - Named storage: Indefinite (billed), no TTL set
- * - Unnamed datasets/KV stores: 7 days (free) or 31 days (paid)
+ * - Unnamed datasets/KV stores: the plan's retention (Free 48 h, Starter 31 d,
+ *   Scale 60 d, Business 90 d); the defaults here are Free and Starter
  * - Request queues: Explicitly deleted after run, 24h fallback TTL for orphaned RQs
  * - Logs: Short TTL in Redis, persisted to S3
  */

package/dist/storage/ttl-manager.js

@@ -2,12 +2,12 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TtlManager = exports.DEFAULT_TTL_CONFIG = void 0;
 /**
- * Default TTL configuration based on Apify's model
+ * Default TTL configuration
  */
 exports.DEFAULT_TTL_CONFIG = {
-    datasetFree: 7 * 24 * 60 * 60, // 7 days
+    datasetFree: 48 * 60 * 60, // 48 hours
     datasetPaid: 31 * 24 * 60 * 60, // 31 days
-    kvFree: 7 * 24 * 60 * 60, // 7 days
+    kvFree: 48 * 60 * 60, // 48 hours
     kvPaid: 31 * 24 * 60 * 60, // 31 days
     requestQueueFallback: 24 * 60 * 60, // 24 hours
     logsFree: 12 * 60 * 60, // 12 hours
@@ -19,7 +19,8 @@ exports.DEFAULT_TTL_CONFIG = {
  *
  * Retention policy:
  * - Named storage: Indefinite (billed), no TTL set
- * - Unnamed datasets/KV stores: 7 days (free) or 31 days (paid)
+ * - Unnamed datasets/KV stores: the plan's retention (Free 48 h, Starter 31 d,
+ *   Scale 60 d, Business 90 d); the defaults here are Free and Starter
  * - Request queues: Explicitly deleted after run, 24h fallback TTL for orphaned RQs
  * - Logs: Short TTL in Redis, persisted to S3
  */

package/dist/storage-identifiers.d.ts

@@ -1,6 +1,5 @@
 /**
- * Storage identifier types (Apify-compatible names, so TypeScript actors importing
- * them from 'apify' still compile after the apify → scrapely rewrite).
+ * Storage identifier types, kept under these names for backward compatibility.
  */
 /** Identifies a storage by its alias from the Actor's schema storages. */
 export interface StorageAlias {

package/dist/storage-identifiers.js

@@ -1,6 +1,5 @@
 "use strict";
 /**
- * Storage identifier types (Apify-compatible names, so TypeScript actors importing
- * them from 'apify' still compile after the apify → scrapely rewrite).
+ * Storage identifier types, kept under these names for backward compatibility.
  */
 Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils.d.ts

@@ -1,16 +1,16 @@
 /**
- * Apify v2-compatible utility functions for scrapely.
- * These adapt the Apify v2 API to Crawlee v3 internally.
+ * v2-style utility functions for scrapely.
+ * These adapt the v2-style call signatures to Crawlee v3 internally.
  */
 import type { CheerioAPI } from 'cheerio';
 import type { RequestQueue } from 'crawlee';
 /**
- * Options for enqueueLinks (Apify v2 API compatible).
+ * Options for enqueueLinks (v2-style API).
  */
 export interface EnqueueLinksOptions {
-    /** Cheerio instance ($) - REQUIRED in Apify v2 */
+    /** Cheerio instance ($) - REQUIRED */
     $: CheerioAPI;
-    /** RequestQueue instance - REQUIRED in Apify v2 */
+    /** RequestQueue instance - REQUIRED */
     requestQueue: RequestQueue;
     /** CSS selector to find links. Default: 'a[href]' */
     selector?: string;
@@ -30,13 +30,13 @@ export interface EnqueueLinksOptions {
 /**
  * Enqueues links found on the page using a CSS selector.
  *
- * This is an Apify v2-compatible implementation that adapts to Crawlee v3 internally.
+ * This v2-style implementation adapts to Crawlee v3 internally.
  * It extracts URLs from the page using cheerio and adds them to the request queue.
  *
  * @example
  * ```typescript
  * // Inside a CheerioCrawler requestHandler:
- * await Apify.utils.enqueueLinks({
+ * await Actor.utils.enqueueLinks({
  *     $,
  *     requestQueue,
  *     selector: 'a[href*="/app/"]',
@@ -45,10 +45,43 @@ export interface EnqueueLinksOptions {
  * ```
  */
 export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<void>;
+/** Options of the v2-style `utils.requestAsBrowser`; other got options pass through. */
+export interface RequestAsBrowserOptions extends Record<string, any> {
+    url: string;
+    method?: string;
+    headers?: Record<string, string>;
+    payload?: string | Buffer;
+    proxyUrl?: string;
+    languageCode?: string;
+    countryCode?: string;
+    isMobile?: boolean;
+    json?: boolean;
+    ignoreSslErrors?: boolean;
+    useInsecureHttpParser?: boolean;
+    abortFunction?: unknown;
+}
+/**
+ * v2-style `utils.requestAsBrowser`: an HTTP request with browser-like headers,
+ * here through got-scraping (what v2 itself switched to). Resolves to the got
+ * response (`body`, `statusCode`, `headers`, `url`).
+ */
+export declare function requestAsBrowser(options: RequestAsBrowserOptions): Promise<any>;
+/** v2-style `utils.getRandomUserAgent`: a realistic desktop browser User-Agent. */
+export declare function getRandomUserAgent(): string;
 /**
- * Utility namespace providing Apify v2-compatible helper functions.
+ * Utility namespace providing v2-style helper functions.
  * These adapt to Crawlee v3 APIs internally.
  */
 export declare const utils: {
     enqueueLinks: typeof enqueueLinks;
+    requestAsBrowser: typeof requestAsBrowser;
+    getRandomUserAgent: typeof getRandomUserAgent;
+    sleep: (millis?: number) => any;
+    downloadListOfUrls: (options: any) => any;
+    extractUrls: (options: any) => any;
+    htmlToText: (html: any) => any;
+    isDocker: (forceReset?: boolean) => any;
+    readonly log: any;
+    readonly puppeteer: any;
+    readonly playwright: any;
 };

package/dist/utils.js

@@ -2,16 +2,18 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.utils = void 0;
 exports.enqueueLinks = enqueueLinks;
+exports.requestAsBrowser = requestAsBrowser;
+exports.getRandomUserAgent = getRandomUserAgent;
 /**
  * Enqueues links found on the page using a CSS selector.
  *
- * This is an Apify v2-compatible implementation that adapts to Crawlee v3 internally.
+ * This v2-style implementation adapts to Crawlee v3 internally.
  * It extracts URLs from the page using cheerio and adds them to the request queue.
  *
  * @example
  * ```typescript
  * // Inside a CheerioCrawler requestHandler:
- * await Apify.utils.enqueueLinks({
+ * await Actor.utils.enqueueLinks({
  *     $,
  *     requestQueue,
  *     selector: 'a[href*="/app/"]',
@@ -55,9 +57,53 @@ async function enqueueLinks(options) {
     }
 }
 /**
- * Utility namespace providing Apify v2-compatible helper functions.
+ * v2-style `utils.requestAsBrowser`: an HTTP request with browser-like headers,
+ * here through got-scraping (what v2 itself switched to). Resolves to the got
+ * response (`body`, `statusCode`, `headers`, `url`).
+ */
+async function requestAsBrowser(options) {
+    const { payload, languageCode, countryCode, isMobile, json, ignoreSslErrors, abortFunction: _abortFunction, ...gotOptions } = options;
+    const { gotScraping } = await import('@crawlee/utils');
+    const locale = languageCode ? `${languageCode}${countryCode ? `-${countryCode}` : ''}` : undefined;
+    return gotScraping({
+        ...(payload !== undefined ? { body: payload } : {}),
+        ...(json ? { responseType: 'json' } : {}),
+        ...(ignoreSslErrors ? { https: { rejectUnauthorized: false } } : {}),
+        headerGeneratorOptions: {
+            ...(locale ? { locales: [locale] } : {}),
+            ...(isMobile ? { devices: ['mobile'] } : {}),
+        },
+        ...gotOptions,
+    });
+}
+let userAgents;
+/** v2-style `utils.getRandomUserAgent`: a realistic desktop browser User-Agent. */
+function getRandomUserAgent() {
+    if (!userAgents) {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const { HeaderGenerator } = require('header-generator');
+        userAgents = new HeaderGenerator({ devices: ['desktop'] });
+    }
+    const headers = userAgents.getHeaders();
+    return headers['user-agent'] ?? headers['User-Agent'];
+}
+// Loaded on first use: the SDK entry must not load Crawlee (see lazy-exports.ts).
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const crawleeUtils = () => require('@crawlee/utils');
+/**
+ * Utility namespace providing v2-style helper functions.
  * These adapt to Crawlee v3 APIs internally.
  */
 exports.utils = {
     enqueueLinks,
+    requestAsBrowser,
+    getRandomUserAgent,
+    sleep: (millis) => crawleeUtils().sleep(millis),
+    downloadListOfUrls: (options) => crawleeUtils().downloadListOfUrls(options),
+    extractUrls: (options) => crawleeUtils().extractUrls(options),
+    htmlToText: (html) => crawleeUtils().htmlToText(html),
+    isDocker: (forceReset) => crawleeUtils().isDocker(forceReset),
+    get log() { return require('@apify/log').default; },
+    get puppeteer() { return require('@crawlee/puppeteer').puppeteerUtils; },
+    get playwright() { return require('@crawlee/playwright').playwrightUtils; },
 };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.33",
+    "version": "1.2.34",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -21,12 +21,17 @@
         "dist",
         "LICENSE",
         "LICENSE-APACHE",
-        "NOTICE"
+        "NOTICE",
+        "CHANGELOG.md"
     ],
     "private": false,
+    "homepage": "https://scrape.ly/docs/sdk-node",
     "repository": {
         "type": "git",
-        "url": "git+https://github.com/SandeepDev1/scrapely.git"
+        "url": "git+https://github.com/scrapely-app/scrapely.git"
+    },
+    "bugs": {
+        "url": "https://github.com/scrapely-app/scrapely/issues"
     },
     "publishConfig": {
         "registry": "https://registry.npmjs.org",
@@ -37,6 +42,9 @@
         "test:charging": "npx tsx --test tests/charging-pending.test.ts",
         "test:charge-limit-guard": "npx tsx --test tests/charge-limit-guard.test.ts",
         "test:charge-multi-event": "npx tsx --test tests/charge-multi-event.test.ts",
+        "test:charge-multi-event-void": "npx tsx --test tests/charge-multi-event-void.test.ts",
+        "test:charge-edge-cases": "npx tsx --test tests/charge-edge-cases.test.ts",
+        "test:charge-pending-persist": "npx tsx --test tests/charge-pending-persist.test.ts",
         "test:charge-platform-budget": "npx tsx --test tests/charge-platform-budget.test.ts",
         "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
         "test:push-idempotency": "npx tsx --test tests/push-idempotency.test.ts",
@@ -46,6 +54,7 @@
         "test:browser-pool": "npx tsx --test tests/patch-browser-pool.test.ts",
         "test:browser-config": "npx tsx --test tests/browser-configuration.test.ts",
         "build": "tsc",
+        "prepack": "npm run build",
         "setup:gvisor": "bash scripts/setup-gvisor.sh",
         "setup:server": "bash scripts/setup-server.sh",
         "prod": "bash scripts/deploy-prod.sh",
@@ -54,7 +63,7 @@
         "test:local-run": "npx tsx --test tests/local-run.test.ts",
         "test:input-from-kv": "npx tsx --test tests/input-from-kv.test.ts",
         "test:init-rollback": "npx tsx --test tests/init-rollback.test.ts",
-        "test:apify-parity": "npx tsx --test tests/apify-parity.test.ts",
+        "test:parity": "npx tsx --test tests/apify-parity.test.ts",
         "test:run-control": "npx tsx --test tests/run-control.test.ts",
         "test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts",
         "test:terminal-status": "npx tsx --test tests/terminal-status-message.test.ts",
@@ -62,16 +71,17 @@
         "test:proxy-input-aliases": "npx tsx --test tests/proxy-input-aliases.test.ts",
         "test:charge-retries": "npx tsx --test tests/charge-retries.test.ts",
         "test:solver": "npx tsx --test tests/solver.test.ts",
-        "test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts"
+        "test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts",
+        "test:v2-utils": "npx tsx --test tests/v2-utils.test.ts"
     },
-    "author": "SandeepDev1",
+    "author": "TeraShift GmbH (Scrape.ly)",
     "license": "ISC AND Apache-2.0",
     "dependencies": {
         "@aws-sdk/client-s3": "^3.700.0",
         "@crawlee/browser": "^3.16.0",
         "@crawlee/browser-pool": "^3.16.0",
         "@crawlee/playwright": "^3.16.0",
-        "basic-ftp": "^6.0.1",
+        "basic-ftp": "^6.2.3",
         "crawlee": "^3.16.0",
         "dockerode": "^4.0.9",
         "ioredis": "^6.0.0",
@@ -93,14 +103,14 @@
         "typescript": "^5.7.0"
     },
     "overrides": {
-        "adm-zip": "^0.6.0",
+        "adm-zip": "^0.6.1",
         "tar-fs": "2.1.5",
         "yargs": "18.1.0",
-        "axios": "^1.18.0",
+        "axios": "^1.20.0",
         "brace-expansion": "5.0.9",
         "fast-xml-builder": "^1.2.1",
         "file-type": "^21.3.3",
-        "follow-redirects": "^1.16.0",
+        "follow-redirects": "^1.16.1",
         "@crawlee/core": {
             "ow": "0.28.2"
         },
@@ -116,6 +126,7 @@
         "uuid": "^11.1.1",
         "@inquirer/external-editor": "3.0.3",
         "nanoid": "5.1.16",
-        "browserslist": "4.28.7"
+        "browserslist": "4.28.7",
+        "basic-ftp": "^6.2.3"
     }
 }