scrapfly-sdk 0.6.2 → 0.6.4

package/README.md

@@ -24,8 +24,10 @@ The SDK is distributed through:
 3. Start scraping: 🚀
 
 ```javascript
-// node or bun:
+// node 
 import { ScrapflyClient, ScrapeConfig } from 'scrapfly-sdk';
+// bun
+import { ScrapflyClient, ScrapeConfig} from '@scrapfly/scrapfly-sdk';
 // deno: 
 import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk';
 

package/esm/client.d.ts

@@ -30,21 +30,21 @@ export declare class ScrapflyClient {
      */
     scrape(config: ScrapeConfig): Promise<ScrapeResult>;
     /**
-      Concurrently scrape multiple configs
-      This is a async generator call it like this:
+    Concurrently scrape multiple configs
+    This is a async generator call it like this:
   
-        const results = [];
-        const errors = [];
-        for await (const resultOrError of client.concurrentScrape(configs)) {
-            if (resultOrError instanceof Error) {
-                errors.push(resultOrError);
-            } else {
-                results.push(resultOrError);
-            }
-        }
+      const results = [];
+      const errors = [];
+      for await (const resultOrError of client.concurrentScrape(configs)) {
+          if (resultOrError instanceof Error) {
+              errors.push(resultOrError);
+          } else {
+              results.push(resultOrError);
+          }
+      }
   
-      @param concurrencyLimit: if not set it will be taken from your account info
-      */
+    @param concurrencyLimit: if not set it will be taken from your account info
+    */
     concurrentScrape(configs: ScrapeConfig[], concurrencyLimit?: number): AsyncGenerator<ScrapeResult | Error | undefined, void, undefined>;
     /**
      * Save screenshot response to a file

package/esm/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA+BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA6CzD;;;;;;;;;;;;;;;QAeI;IACG,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA4CzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}

package/esm/client.js

@@ -1,5 +1,5 @@
-import * as dntShim from "./_dnt.shims.js";
 import { path } from './deps.js';
+import { mkdir, writeFile } from './polyfill.js';
 import { fetchRetry } from './utils.js';
 import * as errors from './errors.js';
 import { ExtractionResult, ScrapeResult, ScreenshotResult } from './result.js';
@@ -127,14 +127,15 @@ export class ScrapflyClient {
             const url = new URL(this.HOST + '/account');
             const params = { key: this.key };
             url.search = new URLSearchParams(params).toString();
-            response = await this.fetch(new Request(url.toString(), {
+            response = await this.fetch({
+                url: url.toString(),
                 method: 'GET',
                 headers: {
                     'user-agent': this.ua,
                     'accept-encoding': 'gzip, deflate, br',
                     accept: 'application/json',
                 },
-            }));
+            });
         }
         catch (e) {
             log.error('error', e);
@@ -159,7 +160,8 @@ export class ScrapflyClient {
             const url = new URL(this.HOST + '/scrape');
             const params = config.toApiParams({ key: this.key });
             url.search = new URLSearchParams(params).toString();
-            response = await this.fetch(new Request(url.toString(), {
+            response = await this.fetch({
+                url: url.toString(),
                 method: config.method,
                 headers: {
                     'user-agent': this.ua,
@@ -170,7 +172,7 @@ export class ScrapflyClient {
                     accept: 'application/json',
                 },
                 body: config.body,
-            }));
+            });
         }
         catch (e) {
             log.error('error', e);
@@ -193,21 +195,21 @@ export class ScrapflyClient {
         return result;
     }
     /**
-      Concurrently scrape multiple configs
-      This is a async generator call it like this:
+    Concurrently scrape multiple configs
+    This is a async generator call it like this:
   
-        const results = [];
-        const errors = [];
-        for await (const resultOrError of client.concurrentScrape(configs)) {
-            if (resultOrError instanceof Error) {
-                errors.push(resultOrError);
-            } else {
-                results.push(resultOrError);
-            }
-        }
+      const results = [];
+      const errors = [];
+      for await (const resultOrError of client.concurrentScrape(configs)) {
+          if (resultOrError instanceof Error) {
+              errors.push(resultOrError);
+          } else {
+              results.push(resultOrError);
+          }
+      }
   
-      @param concurrencyLimit: if not set it will be taken from your account info
-      */
+    @param concurrencyLimit: if not set it will be taken from your account info
+    */
     async *concurrentScrape(configs, concurrencyLimit) {
         if (concurrencyLimit === undefined) {
             const account = await this.account();
@@ -250,7 +252,7 @@ export class ScrapflyClient {
         const extension_name = result.metadata.extension_name;
         let file_path;
         if (savePath) {
-            dntShim.Deno.mkdir(savePath, { recursive: true });
+            await mkdir(savePath, { recursive: true });
             file_path = path.join(savePath, `${name}.${extension_name}`);
         }
         else {
@@ -258,7 +260,7 @@ export class ScrapflyClient {
         }
         const content = new Uint8Array(result.image);
         // Use Deno's write file method
-        await dntShim.Deno.writeFile(file_path, content);
+        await writeFile(file_path, content);
     }
     /**
      * Turn scrapfly screenshot API response to ScreenshotResult or raise one of ScrapflyError
@@ -289,7 +291,8 @@ export class ScrapflyClient {
             const url = new URL(this.HOST + '/screenshot');
             const params = config.toApiParams({ key: this.key });
             url.search = new URLSearchParams(params).toString();
-            const req = new Request(url.toString(), {
+            response = await this.fetch({
+                url: url.toString(),
                 method: 'GET',
                 headers: {
                     'user-agent': this.ua,
@@ -297,7 +300,6 @@ export class ScrapflyClient {
                     accept: 'application/json',
                 },
             });
-            response = await this.fetch(req);
         }
         catch (e) {
             log.error('error', e);
@@ -345,12 +347,12 @@ export class ScrapflyClient {
             if (config.document_compression_format && config.document_compression_format) {
                 headers['content-encoding'] = config.document_compression_format;
             }
-            const req = new Request(url.toString(), {
+            response = await this.fetch({
+                url: url.toString(),
                 method: 'POST',
                 headers: headers,
                 body: config.body,
             });
-            response = await this.fetch(req);
         }
         catch (e) {
             log.error('error', e);

package/esm/polyfill.d.ts

@@ -0,0 +1,6 @@
+/// <reference types="node" />
+import * as dntShim from "./_dnt.shims.js";
+export declare const isDeno: boolean;
+export declare function mkdir(path: string | URL, options: dntShim.Deno.MkdirOptions): Promise<void>;
+export declare function writeFile(path: string | URL, data: Uint8Array, options?: dntShim.Deno.WriteFileOptions): Promise<void>;
+//# sourceMappingURL=polyfill.d.ts.map

package/esm/polyfill.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"polyfill.d.ts","sourceRoot":"","sources":["../src/polyfill.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,OAAO,MAAM,iBAAiB,CAAC;AAC3C,eAAO,MAAM,MAAM,SAAsC,CAAC;AAE1D,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjG;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,GAAG,GAAG,EAClB,IAAI,EAAE,UAAU,EAChB,OAAO,GAAE,OAAO,CAAC,IAAI,CAAC,gBAAqB,GAC1C,OAAO,CAAC,IAAI,CAAC,CAOf"}

package/esm/polyfill.js

@@ -0,0 +1,20 @@
+import * as dntShim from "./_dnt.shims.js";
+export const isDeno = typeof dntShim.Deno !== 'undefined';
+export async function mkdir(path, options) {
+    if (isDeno) {
+        await dntShim.Deno.mkdir(path, options);
+    }
+    else {
+        // @ts-ignore: type for Bun
+        await Bun.mkdir(path.toString(), options);
+    }
+}
+export async function writeFile(path, data, options = {}) {
+    if (isDeno) {
+        await dntShim.Deno.writeFile(path, data, options);
+    }
+    else {
+        // @ts-ignore: type for Bun
+        await Bun.write(path.toString(), data, options);
+    }
+}

package/esm/utils.d.ts

@@ -1,4 +1,10 @@
 /// <reference types="node" />
 export declare function urlsafe_b64encode(data: string): string;
-export declare function fetchRetry(config: Request, retries?: number, retryDelay?: number): Promise<Response>;
+export type RequestOptions = {
+    url: string;
+    method?: string;
+    headers?: any;
+    body?: string | Uint8Array;
+};
+export declare function fetchRetry(config: RequestOptions, retries?: number, retryDelay?: number): Promise<Response>;
 //# sourceMappingURL=utils.d.ts.map

package/esm/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAAA,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,wBAAsB,UAAU,CAC9B,MAAM,EAAE,OAAO,EACf,OAAO,GAAE,MAAU,EACnB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,QAAQ,CAAC,CA4BnB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAEA,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;CAC5B,CAAC;AAEF,wBAAsB,UAAU,CAC9B,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,MAAU,EACnB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,QAAQ,CAAC,CA+BnB"}

package/esm/utils.js

@@ -1,3 +1,4 @@
+import { log } from './logger.js';
 export function urlsafe_b64encode(data) {
     const encoder = new TextEncoder();
     const encoded = encoder.encode(data);
@@ -11,12 +12,15 @@ export async function fetchRetry(config, retries = 3, retryDelay = 1000) {
     let lastError = null;
     for (let attempt = 1; attempt <= retries; attempt++) {
         try {
-            // XXX: this breaks cloudflare workers as they don't support init options
-            const response = await fetch(config);
+            // XXX: note that cloudflare workers don't support init options
+            const { url, ...reqInit } = config;
+            const response = await fetch(new Request(url, reqInit));
             // retry 5xx status codes
             if (response.status >= 500 && response.status < 600) {
+                const _text = await response.text(); // consume response to prevent leak
                 lastError = new Error(`Fetch failed with status: ${response.status}`);
                 if (attempt < retries) {
+                    log.debug(`request failed ${response.status} (${_text}): retry ${attempt}/${retries} after ${retryDelay}ms`);
                     await new Promise((resolve) => setTimeout(resolve, retryDelay));
                 }
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrapfly-sdk",
-  "version": "0.6.2",
+  "version": "0.6.4",
   "description": "SDK for Scrapfly.io API for web scraping, screenshotting and data extraction",
   "keywords": [
     "web scraping",

package/script/client.d.ts

@@ -30,21 +30,21 @@ export declare class ScrapflyClient {
      */
     scrape(config: ScrapeConfig): Promise<ScrapeResult>;
     /**
-      Concurrently scrape multiple configs
-      This is a async generator call it like this:
+    Concurrently scrape multiple configs
+    This is a async generator call it like this:
   
-        const results = [];
-        const errors = [];
-        for await (const resultOrError of client.concurrentScrape(configs)) {
-            if (resultOrError instanceof Error) {
-                errors.push(resultOrError);
-            } else {
-                results.push(resultOrError);
-            }
-        }
+      const results = [];
+      const errors = [];
+      for await (const resultOrError of client.concurrentScrape(configs)) {
+          if (resultOrError instanceof Error) {
+              errors.push(resultOrError);
+          } else {
+              results.push(resultOrError);
+          }
+      }
   
-      @param concurrencyLimit: if not set it will be taken from your account info
-      */
+    @param concurrencyLimit: if not set it will be taken from your account info
+    */
     concurrentScrape(configs: ScrapeConfig[], concurrencyLimit?: number): AsyncGenerator<ScrapeResult | Error | undefined, void, undefined>;
     /**
      * Save screenshot response to a file

package/script/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA+BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA6CzD;;;;;;;;;;;;;;;QAeI;IACG,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA4CzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}

package/script/client.js

@@ -24,8 +24,8 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ScrapflyClient = void 0;
-const dntShim = __importStar(require("./_dnt.shims.js"));
 const deps_js_1 = require("./deps.js");
+const polyfill_js_1 = require("./polyfill.js");
 const utils_js_1 = require("./utils.js");
 const errors = __importStar(require("./errors.js"));
 const result_js_1 = require("./result.js");
@@ -153,14 +153,15 @@ class ScrapflyClient {
             const url = new URL(this.HOST + '/account');
             const params = { key: this.key };
             url.search = new URLSearchParams(params).toString();
-            response = await this.fetch(new Request(url.toString(), {
+            response = await this.fetch({
+                url: url.toString(),
                 method: 'GET',
                 headers: {
                     'user-agent': this.ua,
                     'accept-encoding': 'gzip, deflate, br',
                     accept: 'application/json',
                 },
-            }));
+            });
         }
         catch (e) {
             logger_js_1.log.error('error', e);
@@ -185,7 +186,8 @@ class ScrapflyClient {
             const url = new URL(this.HOST + '/scrape');
             const params = config.toApiParams({ key: this.key });
             url.search = new URLSearchParams(params).toString();
-            response = await this.fetch(new Request(url.toString(), {
+            response = await this.fetch({
+                url: url.toString(),
                 method: config.method,
                 headers: {
                     'user-agent': this.ua,
@@ -196,7 +198,7 @@ class ScrapflyClient {
                     accept: 'application/json',
                 },
                 body: config.body,
-            }));
+            });
         }
         catch (e) {
             logger_js_1.log.error('error', e);
@@ -219,21 +221,21 @@ class ScrapflyClient {
         return result;
     }
     /**
-      Concurrently scrape multiple configs
-      This is a async generator call it like this:
+    Concurrently scrape multiple configs
+    This is a async generator call it like this:
   
-        const results = [];
-        const errors = [];
-        for await (const resultOrError of client.concurrentScrape(configs)) {
-            if (resultOrError instanceof Error) {
-                errors.push(resultOrError);
-            } else {
-                results.push(resultOrError);
-            }
-        }
+      const results = [];
+      const errors = [];
+      for await (const resultOrError of client.concurrentScrape(configs)) {
+          if (resultOrError instanceof Error) {
+              errors.push(resultOrError);
+          } else {
+              results.push(resultOrError);
+          }
+      }
   
-      @param concurrencyLimit: if not set it will be taken from your account info
-      */
+    @param concurrencyLimit: if not set it will be taken from your account info
+    */
     async *concurrentScrape(configs, concurrencyLimit) {
         if (concurrencyLimit === undefined) {
             const account = await this.account();
@@ -276,7 +278,7 @@ class ScrapflyClient {
         const extension_name = result.metadata.extension_name;
         let file_path;
         if (savePath) {
-            dntShim.Deno.mkdir(savePath, { recursive: true });
+            await (0, polyfill_js_1.mkdir)(savePath, { recursive: true });
             file_path = deps_js_1.path.join(savePath, `${name}.${extension_name}`);
         }
         else {
@@ -284,7 +286,7 @@ class ScrapflyClient {
         }
         const content = new Uint8Array(result.image);
         // Use Deno's write file method
-        await dntShim.Deno.writeFile(file_path, content);
+        await (0, polyfill_js_1.writeFile)(file_path, content);
     }
     /**
      * Turn scrapfly screenshot API response to ScreenshotResult or raise one of ScrapflyError
@@ -315,7 +317,8 @@ class ScrapflyClient {
             const url = new URL(this.HOST + '/screenshot');
             const params = config.toApiParams({ key: this.key });
             url.search = new URLSearchParams(params).toString();
-            const req = new Request(url.toString(), {
+            response = await this.fetch({
+                url: url.toString(),
                 method: 'GET',
                 headers: {
                     'user-agent': this.ua,
@@ -323,7 +326,6 @@ class ScrapflyClient {
                     accept: 'application/json',
                 },
             });
-            response = await this.fetch(req);
         }
         catch (e) {
             logger_js_1.log.error('error', e);
@@ -371,12 +373,12 @@ class ScrapflyClient {
             if (config.document_compression_format && config.document_compression_format) {
                 headers['content-encoding'] = config.document_compression_format;
             }
-            const req = new Request(url.toString(), {
+            response = await this.fetch({
+                url: url.toString(),
                 method: 'POST',
                 headers: headers,
                 body: config.body,
             });
-            response = await this.fetch(req);
         }
         catch (e) {
             logger_js_1.log.error('error', e);

package/script/polyfill.d.ts

@@ -0,0 +1,6 @@
+/// <reference types="node" />
+import * as dntShim from "./_dnt.shims.js";
+export declare const isDeno: boolean;
+export declare function mkdir(path: string | URL, options: dntShim.Deno.MkdirOptions): Promise<void>;
+export declare function writeFile(path: string | URL, data: Uint8Array, options?: dntShim.Deno.WriteFileOptions): Promise<void>;
+//# sourceMappingURL=polyfill.d.ts.map

package/script/polyfill.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"polyfill.d.ts","sourceRoot":"","sources":["../src/polyfill.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,OAAO,MAAM,iBAAiB,CAAC;AAC3C,eAAO,MAAM,MAAM,SAAsC,CAAC;AAE1D,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjG;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,GAAG,GAAG,EAClB,IAAI,EAAE,UAAU,EAChB,OAAO,GAAE,OAAO,CAAC,IAAI,CAAC,gBAAqB,GAC1C,OAAO,CAAC,IAAI,CAAC,CAOf"}

package/script/polyfill.js

@@ -0,0 +1,48 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeFile = exports.mkdir = exports.isDeno = void 0;
+const dntShim = __importStar(require("./_dnt.shims.js"));
+exports.isDeno = typeof dntShim.Deno !== 'undefined';
+async function mkdir(path, options) {
+    if (exports.isDeno) {
+        await dntShim.Deno.mkdir(path, options);
+    }
+    else {
+        // @ts-ignore: type for Bun
+        await Bun.mkdir(path.toString(), options);
+    }
+}
+exports.mkdir = mkdir;
+async function writeFile(path, data, options = {}) {
+    if (exports.isDeno) {
+        await dntShim.Deno.writeFile(path, data, options);
+    }
+    else {
+        // @ts-ignore: type for Bun
+        await Bun.write(path.toString(), data, options);
+    }
+}
+exports.writeFile = writeFile;

package/script/utils.d.ts

@@ -1,4 +1,10 @@
 /// <reference types="node" />
 export declare function urlsafe_b64encode(data: string): string;
-export declare function fetchRetry(config: Request, retries?: number, retryDelay?: number): Promise<Response>;
+export type RequestOptions = {
+    url: string;
+    method?: string;
+    headers?: any;
+    body?: string | Uint8Array;
+};
+export declare function fetchRetry(config: RequestOptions, retries?: number, retryDelay?: number): Promise<Response>;
 //# sourceMappingURL=utils.d.ts.map

package/script/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAAA,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,wBAAsB,UAAU,CAC9B,MAAM,EAAE,OAAO,EACf,OAAO,GAAE,MAAU,EACnB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,QAAQ,CAAC,CA4BnB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAEA,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;CAC5B,CAAC;AAEF,wBAAsB,UAAU,CAC9B,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,MAAU,EACnB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,QAAQ,CAAC,CA+BnB"}

package/script/utils.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.fetchRetry = exports.urlsafe_b64encode = void 0;
+const logger_js_1 = require("./logger.js");
 function urlsafe_b64encode(data) {
     const encoder = new TextEncoder();
     const encoded = encoder.encode(data);
@@ -15,12 +16,15 @@ async function fetchRetry(config, retries = 3, retryDelay = 1000) {
     let lastError = null;
     for (let attempt = 1; attempt <= retries; attempt++) {
         try {
-            // XXX: this breaks cloudflare workers as they don't support init options
-            const response = await fetch(config);
+            // XXX: note that cloudflare workers don't support init options
+            const { url, ...reqInit } = config;
+            const response = await fetch(new Request(url, reqInit));
             // retry 5xx status codes
             if (response.status >= 500 && response.status < 600) {
+                const _text = await response.text(); // consume response to prevent leak
                 lastError = new Error(`Fetch failed with status: ${response.status}`);
                 if (attempt < retries) {
+                    logger_js_1.log.debug(`request failed ${response.status} (${_text}): retry ${attempt}/${retries} after ${retryDelay}ms`);
                     await new Promise((resolve) => setTimeout(resolve, retryDelay));
                 }
             }