scrapfly-sdk 0.7.1 → 0.7.3

package/esm/main.d.ts

@@ -6,4 +6,5 @@ export * as errors from './errors.js';
 export { ExtractionResult, ScrapeResult, ScreenshotResult } from './result.js';
 export type { AccountData, ConfigData, ContextData, ResultData, ScreenshotMetadata } from './result.js';
 export { log } from './logger.js';
+export { createServer, verifySignature } from './webhookserver.js';
 //# sourceMappingURL=main.d.ts.map

package/esm/main.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EACL,MAAM,IAAI,YAAY,EACtB,YAAY,IAAI,mBAAmB,EACnC,YAAY,EACZ,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,MAAM,IAAI,gBAAgB,EAAE,OAAO,IAAI,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/E,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACxG,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EACL,MAAM,IAAI,YAAY,EACtB,YAAY,IAAI,mBAAmB,EACnC,YAAY,EACZ,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,MAAM,IAAI,gBAAgB,EAAE,OAAO,IAAI,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/E,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACxG,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAElC,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC"}

package/esm/main.js

@@ -5,3 +5,4 @@ export { ExtractionConfig } from './extractionconfig.js';
 export * as errors from './errors.js';
 export { ExtractionResult, ScrapeResult, ScreenshotResult } from './result.js';
 export { log } from './logger.js';
+export { createServer, verifySignature } from './webhookserver.js';

package/esm/webhookserver.d.ts

@@ -0,0 +1,8 @@
+import { Buffer } from 'node:buffer';
+export declare enum ResourceType {
+    SCRAPE = "scrape",
+    PING = "ping"
+}
+export declare function verifySignature(body: Buffer, signature: string, signingSecret: string): boolean;
+export declare function createServer(callback: (data: any, resourceType: string, request: any) => void, app?: any): Promise<any>;
+//# sourceMappingURL=webhookserver.d.ts.map

package/esm/webhookserver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"webhookserver.d.ts","sourceRoot":"","sources":["../src/webhookserver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAIrC,oBAAY,YAAY;IACtB,MAAM,WAAW;IACjB,IAAI,SAAS;CACd;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAM/F;AAGD,wBAAsB,YAAY,CAChC,QAAQ,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,IAAI,EACjE,GAAG,CAAC,EAAE,GAAG,GACR,OAAO,CAAC,GAAG,CAAC,CA4Cd"}

package/esm/webhookserver.js

@@ -0,0 +1,56 @@
+import { Buffer } from 'node:buffer';
+import crypto from 'node:crypto';
+import { log } from './logger.js';
+export var ResourceType;
+(function (ResourceType) {
+    ResourceType["SCRAPE"] = "scrape";
+    ResourceType["PING"] = "ping";
+})(ResourceType || (ResourceType = {}));
+export function verifySignature(body, signature, signingSecret) {
+    const secret = Buffer.from(signingSecret, 'hex');
+    const hmac = crypto.createHmac('sha256', secret);
+    hmac.update(body);
+    const computed = hmac.digest('hex').toUpperCase();
+    return computed === signature;
+}
+export async function createServer(callback, app) {
+    let express;
+    try {
+        const expressModule = await import('express');
+        express = expressModule.default || expressModule;
+    }
+    catch (e) {
+        throw new Error('express is not installed, please install it with `npm install express`');
+    }
+    if (!app) {
+        app = express();
+    }
+    const jsonMiddleware = express.json({
+        verify: (req, _res, buf) => {
+            req.rawBody = buf;
+        },
+    });
+    app.post('/webhook', jsonMiddleware, (req, res) => {
+        try {
+            const resourceType = req.headers['x-scrapfly-webhook-resource-type'];
+            if (resourceType !== 'scrape' && resourceType !== 'ping') {
+                log.error(`Unsupported resource type: ${resourceType}`);
+                return res.status(400).send(`this webhook server only supports scrape and ping calls`);
+            }
+            const data = req.body;
+            try {
+                callback(data, resourceType, req);
+                res.status(200).send('');
+            }
+            catch (e) {
+                log.error('Callback error:', e);
+                res.status(500).send('');
+            }
+        }
+        catch (e) {
+            log.error('Webhook processing error:', e);
+            res.status(400).send('');
+        }
+    });
+    return app;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrapfly-sdk",
-  "version": "0.7.1",
+  "version": "0.7.3",
   "description": "SDK for Scrapfly.io API for web scraping, screenshotting and data extraction",
   "keywords": [
     "web scraping",
@@ -32,7 +32,8 @@
     "@deno/shim-deno": "~0.18.0"
   },
   "devDependencies": {
-    "@types/node": "^20.9.0"
+    "@types/node": "^20.9.0",
+    "@types/express": "^4.17.21"
   },
   "_generatedBy": "dnt@dev"
 }

package/script/main.d.ts

@@ -6,4 +6,5 @@ export * as errors from './errors.js';
 export { ExtractionResult, ScrapeResult, ScreenshotResult } from './result.js';
 export type { AccountData, ConfigData, ContextData, ResultData, ScreenshotMetadata } from './result.js';
 export { log } from './logger.js';
+export { createServer, verifySignature } from './webhookserver.js';
 //# sourceMappingURL=main.d.ts.map

package/script/main.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EACL,MAAM,IAAI,YAAY,EACtB,YAAY,IAAI,mBAAmB,EACnC,YAAY,EACZ,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,MAAM,IAAI,gBAAgB,EAAE,OAAO,IAAI,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/E,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACxG,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EACL,MAAM,IAAI,YAAY,EACtB,YAAY,IAAI,mBAAmB,EACnC,YAAY,EACZ,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,MAAM,IAAI,gBAAgB,EAAE,OAAO,IAAI,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/E,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACxG,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAElC,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC"}

package/script/main.js

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.log = exports.ScreenshotResult = exports.ScrapeResult = exports.ExtractionResult = exports.errors = exports.ExtractionConfig = exports.ScreenshotConfig = exports.ScreenshotOptions = exports.ScreenshotFormat = exports.ScreenshotFlags = exports.ScrapeConfig = exports.ScrapeFormatOptions = exports.ScrapeFormat = exports.ScrapflyClient = void 0;
+exports.verifySignature = exports.createServer = exports.log = exports.ScreenshotResult = exports.ScrapeResult = exports.ExtractionResult = exports.errors = exports.ExtractionConfig = exports.ScreenshotConfig = exports.ScreenshotOptions = exports.ScreenshotFormat = exports.ScreenshotFlags = exports.ScrapeConfig = exports.ScrapeFormatOptions = exports.ScrapeFormat = exports.ScrapflyClient = void 0;
 var client_js_1 = require("./client.js");
 Object.defineProperty(exports, "ScrapflyClient", { enumerable: true, get: function () { return client_js_1.ScrapflyClient; } });
 var scrapeconfig_js_1 = require("./scrapeconfig.js");
@@ -44,3 +44,6 @@ Object.defineProperty(exports, "ScrapeResult", { enumerable: true, get: function
 Object.defineProperty(exports, "ScreenshotResult", { enumerable: true, get: function () { return result_js_1.ScreenshotResult; } });
 var logger_js_1 = require("./logger.js");
 Object.defineProperty(exports, "log", { enumerable: true, get: function () { return logger_js_1.log; } });
+var webhookserver_js_1 = require("./webhookserver.js");
+Object.defineProperty(exports, "createServer", { enumerable: true, get: function () { return webhookserver_js_1.createServer; } });
+Object.defineProperty(exports, "verifySignature", { enumerable: true, get: function () { return webhookserver_js_1.verifySignature; } });

package/script/webhookserver.d.ts

@@ -0,0 +1,8 @@
+import { Buffer } from 'node:buffer';
+export declare enum ResourceType {
+    SCRAPE = "scrape",
+    PING = "ping"
+}
+export declare function verifySignature(body: Buffer, signature: string, signingSecret: string): boolean;
+export declare function createServer(callback: (data: any, resourceType: string, request: any) => void, app?: any): Promise<any>;
+//# sourceMappingURL=webhookserver.d.ts.map

package/script/webhookserver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"webhookserver.d.ts","sourceRoot":"","sources":["../src/webhookserver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAIrC,oBAAY,YAAY;IACtB,MAAM,WAAW;IACjB,IAAI,SAAS;CACd;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAM/F;AAGD,wBAAsB,YAAY,CAChC,QAAQ,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,IAAI,EACjE,GAAG,CAAC,EAAE,GAAG,GACR,OAAO,CAAC,GAAG,CAAC,CA4Cd"}

package/script/webhookserver.js

@@ -0,0 +1,87 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ResourceType = void 0;
+exports.verifySignature = verifySignature;
+exports.createServer = createServer;
+const node_buffer_1 = require("node:buffer");
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const logger_js_1 = require("./logger.js");
+var ResourceType;
+(function (ResourceType) {
+    ResourceType["SCRAPE"] = "scrape";
+    ResourceType["PING"] = "ping";
+})(ResourceType || (exports.ResourceType = ResourceType = {}));
+function verifySignature(body, signature, signingSecret) {
+    const secret = node_buffer_1.Buffer.from(signingSecret, 'hex');
+    const hmac = node_crypto_1.default.createHmac('sha256', secret);
+    hmac.update(body);
+    const computed = hmac.digest('hex').toUpperCase();
+    return computed === signature;
+}
+async function createServer(callback, app) {
+    let express;
+    try {
+        const expressModule = await Promise.resolve().then(() => __importStar(require('express')));
+        express = expressModule.default || expressModule;
+    }
+    catch (e) {
+        throw new Error('express is not installed, please install it with `npm install express`');
+    }
+    if (!app) {
+        app = express();
+    }
+    const jsonMiddleware = express.json({
+        verify: (req, _res, buf) => {
+            req.rawBody = buf;
+        },
+    });
+    app.post('/webhook', jsonMiddleware, (req, res) => {
+        try {
+            const resourceType = req.headers['x-scrapfly-webhook-resource-type'];
+            if (resourceType !== 'scrape' && resourceType !== 'ping') {
+                logger_js_1.log.error(`Unsupported resource type: ${resourceType}`);
+                return res.status(400).send(`this webhook server only supports scrape and ping calls`);
+            }
+            const data = req.body;
+            try {
+                callback(data, resourceType, req);
+                res.status(200).send('');
+            }
+            catch (e) {
+                logger_js_1.log.error('Callback error:', e);
+                res.status(500).send('');
+            }
+        }
+        catch (e) {
+            logger_js_1.log.error('Webhook processing error:', e);
+            res.status(400).send('');
+        }
+    });
+    return app;
+}