scrapely 1.2.3 → 1.2.4

package/dist/utils.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.utils = void 0;
+exports.enqueueLinks = enqueueLinks;
+/**
+ * Enqueues links found on the page using a CSS selector.
+ *
+ * This is an Apify v2-compatible implementation that adapts to Crawlee v3 internally.
+ * It extracts URLs from the page using cheerio and adds them to the request queue.
+ *
+ * @example
+ * ```typescript
+ * // Inside a CheerioCrawler requestHandler:
+ * await Apify.utils.enqueueLinks({
+ *     $,
+ *     requestQueue,
+ *     selector: 'a[href*="/app/"]',
+ *     baseUrl: new URL(request.url).origin,
+ * });
+ * ```
+ */
+async function enqueueLinks(options) {
+    const { $, requestQueue, selector = 'a[href]', baseUrl, userData = {}, transformRequestFunction } = options;
+    // Collect all matching URLs
+    const urls = [];
+    $(selector).each((_, element) => {
+        const href = $(element).attr('href');
+        if (href) {
+            // Resolve relative URLs if baseUrl is provided
+            let fullUrl;
+            try {
+                fullUrl = baseUrl ? new URL(href, baseUrl).href : href;
+                urls.push(fullUrl);
+            }
+            catch {
+                // Skip invalid URLs
+            }
+        }
+    });
+    // Deduplicate URLs
+    const uniqueUrls = [...new Set(urls)];
+    // Add each URL to the request queue
+    for (const url of uniqueUrls) {
+        const request = {
+            url,
+            userData: { ...userData },
+        };
+        // Apply transformation if provided
+        const transformed = transformRequestFunction
+            ? transformRequestFunction(request)
+            : request;
+        if (transformed) {
+            await requestQueue.addRequest(transformed);
+        }
+    }
+}
+/**
+ * Utility namespace providing Apify v2-compatible helper functions.
+ * These adapt to Crawlee v3 APIs internally.
+ */
+exports.utils = {
+    enqueueLinks,
+};

package/dist/validator.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Scrapely Actor Package Validator
+ * Validates actor package format
+ */
+import { ScrapelyManifest } from './manifest.js';
+export interface ValidationResult {
+    valid: boolean;
+    errors: string[];
+    manifest?: ScrapelyManifest;
+    packageJson?: Record<string, unknown>;
+}
+/**
+ * Validate an actor package directory
+ * Checks for: package.json, package.json, entrypoint file
+ */
+export declare function validatePackage(packageDir: string): ValidationResult;
+/**
+ * Get normalized manifest with defaults applied
+ */
+export declare function getNormalizedManifest(manifest: ScrapelyManifest): Required<Pick<ScrapelyManifest, 'name' | 'version' | 'entrypoint' | 'maxMemory' | 'maxTimeout'>> & Omit<ScrapelyManifest, 'maxMemory' | 'maxTimeout'>;
+/**
+ * CLI entry point for validation
+ */
+export declare function runValidator(args: string[]): Promise<void>;

package/dist/validator.js

@@ -0,0 +1,158 @@
+"use strict";
+/**
+ * Scrapely Actor Package Validator
+ * Validates actor package format
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validatePackage = validatePackage;
+exports.getNormalizedManifest = getNormalizedManifest;
+exports.runValidator = runValidator;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const manifest_js_1 = require("./manifest.js");
+/**
+ * Validate an actor package directory
+ * Checks for: package.json, package.json, entrypoint file
+ */
+function validatePackage(packageDir) {
+    const errors = [];
+    let manifest;
+    let packageJson;
+    // Check directory exists
+    if (!fs.existsSync(packageDir)) {
+        return { valid: false, errors: [`Directory not found: ${packageDir}`] };
+    }
+    if (!fs.statSync(packageDir).isDirectory()) {
+        return { valid: false, errors: [`Not a directory: ${packageDir}`] };
+    }
+    // Check package.json
+    const manifestPath = path.join(packageDir, 'package.json');
+    if (!fs.existsSync(manifestPath)) {
+        errors.push('Missing package.json');
+    }
+    else {
+        try {
+            const content = fs.readFileSync(manifestPath, 'utf-8');
+            const parsed = JSON.parse(content);
+            const manifestErrors = (0, manifest_js_1.validateManifest)(parsed);
+            if (manifestErrors.length > 0) {
+                errors.push(...manifestErrors.map(e => `package.json: ${e}`));
+            }
+            else {
+                manifest = parsed;
+            }
+        }
+        catch (err) {
+            errors.push(`package.json: Invalid JSON - ${err.message}`);
+        }
+    }
+    // Check package.json
+    const packageJsonPath = path.join(packageDir, 'package.json');
+    if (!fs.existsSync(packageJsonPath)) {
+        errors.push('Missing package.json');
+    }
+    else {
+        try {
+            const content = fs.readFileSync(packageJsonPath, 'utf-8');
+            packageJson = JSON.parse(content);
+        }
+        catch (err) {
+            errors.push(`package.json: Invalid JSON - ${err.message}`);
+        }
+    }
+    // Check entrypoint file exists
+    if (manifest?.entrypoint) {
+        const entrypointPath = path.join(packageDir, manifest.entrypoint);
+        if (!fs.existsSync(entrypointPath)) {
+            errors.push(`Entrypoint not found: ${manifest.entrypoint}`);
+        }
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+        manifest,
+        packageJson,
+    };
+}
+/**
+ * Get normalized manifest with defaults applied
+ */
+function getNormalizedManifest(manifest) {
+    return {
+        ...manifest,
+        maxMemory: manifest.maxMemory ?? manifest_js_1.DEFAULT_MAX_MEMORY,
+        maxTimeout: manifest.maxTimeout ?? manifest_js_1.DEFAULT_MAX_TIMEOUT,
+    };
+}
+/**
+ * CLI entry point for validation
+ */
+async function runValidator(args) {
+    const packageDir = args[0] || process.cwd();
+    console.log(`[Scrapely] Validating package: ${packageDir}\n`);
+    const result = validatePackage(packageDir);
+    if (result.valid) {
+        console.log('✓ Package is valid\n');
+        if (result.manifest) {
+            const normalized = getNormalizedManifest(result.manifest);
+            console.log('Manifest:');
+            console.log(`  Name: ${normalized.name}`);
+            console.log(`  Version: ${normalized.version}`);
+            console.log(`  Entrypoint: ${normalized.entrypoint}`);
+            console.log(`  Max Memory: ${normalized.maxMemory} MB`);
+            console.log(`  Max Timeout: ${normalized.maxTimeout} seconds`);
+            if (normalized.description) {
+                console.log(`  Description: ${normalized.description}`);
+            }
+            if (normalized.author) {
+                console.log(`  Author: ${normalized.author}`);
+            }
+        }
+        process.exit(0);
+    }
+    else {
+        console.log('✗ Package validation failed\n');
+        console.log('Errors:');
+        for (const error of result.errors) {
+            console.log(`  - ${error}`);
+        }
+        process.exit(1);
+    }
+}
+// Run if called directly (CommonJS compatible)
+if (require.main === module) {
+    runValidator(process.argv.slice(2));
+}

package/package.json

@@ -1,11 +1,92 @@
 {
-  "name": "scrapely",
-  "version": "1.2.3",
-  "description": "Reserved package name.",
-  "license": "UNLICENSED",
-  "private": false,
-  "files": [],
-  "publishConfig": {
-    "access": "public"
-  }
+    "name": "scrapely",
+    "version": "1.2.4",
+    "description": "Apify-like scraping platform using Evomi browser infrastructure",
+    "main": "dist/index.js",
+    "types": "dist/index.d.ts",
+    "files": [
+        "dist"
+    ],
+    "private": false,
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/SandeepDev1/scrapely.git"
+    },
+    "publishConfig": {
+        "registry": "https://registry.npmjs.org",
+        "access": "public"
+    },
+    "scripts": {
+        "test": "npx tsx tests/test-mvp.ts",
+        "test:charging": "npx tsx --test tests/charging-pending.test.ts",
+        "test:charging-client": "npx tsx --test tests/charging-client.test.ts",
+        "test:proxy": "npx tsx --test tests/proxy-configuration.test.ts",
+        "test:browser-pool": "npx tsx --test tests/patch-browser-pool.test.ts",
+        "test:browser-config": "npx tsx --test tests/browser-configuration.test.ts",
+        "build": "tsc",
+        "setup:gvisor": "bash scripts/setup-gvisor.sh",
+        "setup:server": "bash scripts/setup-server.sh",
+        "prod": "bash scripts/deploy-prod.sh",
+        "prod:update": "bash scripts/deploy-prod.sh --update-only",
+        "migrate": "npx tsx scripts/run-migration.ts",
+        "test:local-run": "npx tsx --test tests/local-run.test.ts",
+        "test:init-rollback": "npx tsx --test tests/init-rollback.test.ts",
+        "test:apify-parity": "npx tsx --test tests/apify-parity.test.ts",
+        "test:run-control": "npx tsx --test tests/run-control.test.ts",
+        "test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts"
+    },
+    "author": "SandeepDev1",
+    "license": "ISC",
+    "dependencies": {
+        "@aws-sdk/client-s3": "^3.700.0",
+        "@crawlee/browser": "^3.16.0",
+        "@crawlee/browser-pool": "^3.16.0",
+        "@crawlee/playwright": "^3.16.0",
+        "basic-ftp": "^6.0.1",
+        "crawlee": "^3.16.0",
+        "dockerode": "^4.0.9",
+        "ioredis": "^6.0.0",
+        "ip-address": "10.3.1",
+        "lodash": "^4.18.1",
+        "mysql2": "^3.23.1",
+        "playwright": "^1.58.2",
+        "scrapely-client": "^2.25.2",
+        "tar-fs": "2.1.5",
+        "ws": "^8.18.0"
+    },
+    "devDependencies": {
+        "@types/dockerode": "^4.0.1",
+        "@types/node": "^25.5.0",
+        "@types/ws": "^8.5.10",
+        "dotenv": "^17.3.1",
+        "puppeteer-core": "^24.43.1",
+        "tsx": "^4.21.0",
+        "typescript": "^5.7.0"
+    },
+    "overrides": {
+        "adm-zip": "^0.6.0",
+        "tar-fs": "2.1.5",
+        "yargs": "18.1.0",
+        "axios": "^1.18.0",
+        "brace-expansion": "5.0.9",
+        "fast-xml-builder": "^1.2.1",
+        "file-type": "^21.3.3",
+        "follow-redirects": "^1.16.0",
+        "@crawlee/core": {
+            "ow": "0.28.2"
+        },
+        "@crawlee/utils": {
+            "ow": "0.28.2"
+        },
+        "header-generator": {
+            "ow": "0.28.2"
+        },
+        "scrapely-client": {
+            "ow": "0.28.2"
+        },
+        "uuid": "^11.1.1",
+        "@inquirer/external-editor": "3.0.3",
+        "nanoid": "5.1.16",
+        "browserslist": "4.28.7"
+    }
 }