webcrawlerapi-js 1.1.5 → 1.1.7

package/README.md

@@ -20,34 +20,36 @@ The package is regularly updated with new features and improvements to ensure th
 ## JavaScript Examples
 
 ### Synchronous Job
-In this mode, the promise will be resolved when all data is ready. Synchronous jobs are perfect for quick scraping tasks where you need immediate results and don't want to handle polling logic.
+In this mode, the promise will be resolved when all data is ready. Synchronous jobs are perfect for quick scraping tasks where you need immediate results and don't want to handle polling logic. However, for long running jobs we recommend to use async crawling or the webhook.
 
 ```javascript
 import webcrawlerapi from "webcrawlerapi-js"
 
 async function main() {
-    const client = new WebcrawlerClient(
-        "YOUR API ACCESS KEY HERE",
-    )
+    const client = new WebcrawlerClient("YOUR API ACCESS KEY HERE")
 
     try {
-        const syncJob = await client.crawl({
+        const crawlJob = await client.crawl({
             "items_limit": 10,
-            "url": "https://stripe.com/",
+            "url": "https://books.toscrape.com/",
             "scrape_type": "markdown"
         })
         
-        for (const item of syncJob.job_items) {
+        // Get job details
+        const job = await client.getJob(crawlJob.id);
+        console.log("Job status:", job.status);
+
+        // Output item content
+        for (const item of job.job_items) {
             const content = await item.getContent()
+            console.log("---- Title: " + item.title + "----")
             console.log(content.slice(0, 100))
         }
-        console.log(syncJob)
-        
     } catch (error) {
         if (error instanceof webcrawlerapi.WebcrawlerApiError) {
             console.error("Webcrawler error:", error.message)
         }
-        return
+        console.error(error)
     }
 }
 
@@ -61,27 +63,32 @@ In this mode, you get a job ID immediately and can poll for results. Asynchronou
 import webcrawlerapi from "webcrawlerapi-js"
 
 async function main() {
-    const client = new WebcrawlerClient(
-        "YOUR API ACCESS KEY HERE",
-    )
+    const client = new WebcrawlerClient("YOUR API ACCESS KEY HERE")
 
     try {
-        // Get job ID immediately
+        try {
         const jobWithId = await client.crawlAsync({
             "items_limit": 10,
-            "url": "https://stripe.com/",
+            "url": "https://books.toscrape.com/",
             "scrape_type": "markdown"
         })
-        
+
         // Poll for job completion
         const jobId = jobWithId.id
         let asyncJob = await client.getJob(jobId)
-        
+
         // Check job status
-        console.log("Job status:", asyncJob.status)
-        
+        console.log("Job id:", asyncJob.id)
+        console.log("Dashboard link:", `https://dash.webcrawlerapi.com/jobs/job/${asyncJob.id}`)
+
+        //wait until job is done
+        while (asyncJob.status !== "done") {
+            await new Promise(resolve => setTimeout(resolve, 1000));
+            asyncJob = await client.getJob(jobId)
+        }
+
         // Once complete, access results
-        if (asyncJob.status === "completed") {
+        if (asyncJob.status === "done") {
             for (const item of asyncJob.job_items) {
                 const content = await item.getContent()
                 console.log(content.slice(0, 100))
@@ -91,6 +98,8 @@ async function main() {
         if (error instanceof webcrawlerapi.WebcrawlerApiError) {
             console.error("Webcrawler error:", error.message)
         }
+        console.error(error)
+        return
     }
 }
 
@@ -102,87 +111,40 @@ main().catch(console.error)
 The SDK provides full TypeScript support with type safety and error handling. Our TypeScript integration ensures you catch potential issues at compile-time and provides excellent IDE support with autocompletion and inline documentation.
 
 ```typescript
-import { WebcrawlerClient, WebcrawlerApiError } from "webcrawlerapi-js"
+import {WebcrawlerClient, WebcrawlerApiError} from "webcrawlerapi-js";
 
 async function main() {
-    const client = new WebcrawlerClient(
-        "YOUR_API_KEY",
-        // Optional base URL, defaults to production API
-        "http://localhost:8080" 
-    )
+    const client = new WebcrawlerClient("YOUR_API_KEY")
 
     try {
         // Basic crawl with markdown
         const syncJob = await client.crawl({
-            "items_limit": 2,
+            "items_limit": 10,
             "url": "https://books.toscrape.com/",
-            "scrape_type": "markdown"
-        })
-
-        // Type-safe status check
-        if (syncJob.status === "done") {
-            for (const item of syncJob.job_items) {
-                if (item.markdown_content_url) {
-                    // Access typed properties
-                    console.log("Status code:", item.page_status_code)
-                    console.log("Content URL:", item.markdown_content_url)
-                }
-            }
+            "scrape_type": "markdown",
         }
-
-        // Using specialized scraper
-        const scrapeJob = await client.scrape({
-            "crawler_id": "webcrawler/url-to-md",
-            "input": {
-                "url": "https://books.toscrape.com/"
-            }
-        })
-
-        if (scrapeJob.status === "done") {
-            const content = scrapeJob.getContent()
-            console.log("Scraped content:", content)
+    )
+        // Type-safe status check
+        for (const item of syncJob.job_items) {
+            const content = await item.getContent()
+            console.log(content.slice(0,100))
         }
-
     } catch (error) {
-        // Type-safe error handling
         if (error instanceof WebcrawlerApiError) {
             console.error("API Error:", error.errorCode, error.message)
-            // errorCode is typed and can be used for specific error handling
-            if (error.errorCode === "invalid_request") {
-                // Handle invalid request
-            }
         }
+        console.log(error)
     }
 }
 
 main().catch(console.error)
 ```
 
-### Error Handling
-
-The SDK provides typed error handling through the `WebcrawlerApiError` class. Our error handling system is designed to provide detailed, actionable feedback that helps you quickly identify and resolve issues in your scraping pipeline.
-
-```typescript
-try {
-    const syncJob = await client.crawl({
-        "items_limit": 2,
-        "url": "https://books.toscrape.com/",
-        // This will cause an error due to missing scrape_type
-    })
-} catch (error) {
-    if (error instanceof WebcrawlerApiError) {
-        // Typed error properties
-        console.error("Error code:", error.errorCode)
-        console.error("Error message:", error.message)
-    }
-}
-```
-
 ## Response Structure
 
 Example of a job response. The response structure is carefully designed to provide comprehensive information about your scraping job, including detailed status updates, timing information, and resource usage metrics.
 
-```json
+```js
 {
   id: '49c4942b-b7d9-4d62-94b5-b54a3016ac51',
   url: 'https://stripe.com/',

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, Action } from "./model";
 import { JobStatus } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus };
@@ -12,8 +12,8 @@ export declare class WebcrawlerClient {
     scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
     scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
     getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
-    crawl(crawlRequest: CrawlRequest): Promise<Job>;
-    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
+    crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;
+    crawlAsync(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;
     private sendRequest;
 }

package/dist/@types/model.d.ts

@@ -26,6 +26,7 @@ export interface CrawlRequest {
     blacklist_regexp?: string;
     allow_subdomains?: boolean;
     webhook_url?: string;
+    actions?: Action[];
 }
 export interface Job {
     id: string;
@@ -61,3 +62,14 @@ export interface JobItem {
     cleaned_content_url?: string;
     getContent(): Promise<string | null>;
 }
+export interface Action {
+    type: string;
+}
+export interface UploadS3Action extends Action {
+    type: 'upload_s3';
+    path: string;
+    access_key_id: string;
+    secret_access_key: string;
+    bucket: string;
+    endpoint?: string;
+}

package/dist/api.js

@@ -23,6 +23,44 @@ const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
 __exportStar(require("./model"), exports);
+// Helper function to add getContent method to job items
+function addGetContentMethod(job) {
+    // Transform each job item to include getContent method
+    job.job_items = job.job_items.map(item => ({
+        ...item,
+        getContent: async function () {
+            if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
+                return null;
+            }
+            let contentUrl;
+            switch (job.scrape_type) {
+                case 'html':
+                    contentUrl = this.raw_content_url;
+                    break;
+                case 'cleaned':
+                    contentUrl = this.cleaned_content_url;
+                    break;
+                case 'markdown':
+                    contentUrl = this.markdown_content_url;
+                    break;
+            }
+            if (!contentUrl) {
+                return null;
+            }
+            const response = await fetch(contentUrl, {
+                headers: {
+                    'Accept-Encoding': 'gzip, deflate, br',
+                    'Accept': '*/*'
+                }
+            });
+            if (!response.ok) {
+                throw new Error(`Failed to fetch content: ${response.statusText}`);
+            }
+            return await response.text();
+        }
+    }));
+    return job;
+}
 class WebcrawlerClient {
     constructor(apiKey, basePath = BASE_PATH, apiVersion = "v1") {
         this.apiVersion = "v1";
@@ -92,8 +130,12 @@ class WebcrawlerClient {
         };
         return await this.sendRequest(url, requestOptions);
     }
-    async crawl(crawlRequest) {
+    async crawl(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
+        const requestBody = {
+            ...crawlRequest,
+            actions: actions ? (Array.isArray(actions) ? actions : [actions]) : undefined
+        };
         const requestOptions = {
             'method': 'POST',
             'headers': {
@@ -104,7 +146,7 @@ class WebcrawlerClient {
                 'Pragma': 'no-cache',
                 'Expires': '0'
             },
-            'body': JSON.stringify(crawlRequest),
+            'body': JSON.stringify(requestBody),
         };
         const jobIdResponse = await this.sendRequest(url, requestOptions);
         if (jobIdResponse.id === '') {
@@ -116,40 +158,6 @@ class WebcrawlerClient {
             const timestamp = new Date().getTime();
             const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
             if (job.status !== constants_1.JobStatus.IN_PROGRESS && job.status !== constants_1.JobStatus.NEW) {
-                // Transform each job item to include getContent method
-                job.job_items = job.job_items.map(item => ({
-                    ...item,
-                    getContent: async function () {
-                        if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
-                            return null;
-                        }
-                        let contentUrl;
-                        switch (job.scrape_type) {
-                            case 'html':
-                                contentUrl = this.raw_content_url;
-                                break;
-                            case 'cleaned':
-                                contentUrl = this.cleaned_content_url;
-                                break;
-                            case 'markdown':
-                                contentUrl = this.markdown_content_url;
-                                break;
-                        }
-                        if (!contentUrl) {
-                            return null;
-                        }
-                        const response = await fetch(contentUrl, {
-                            headers: {
-                                'Accept-Encoding': 'gzip, deflate, br',
-                                'Accept': '*/*'
-                            }
-                        });
-                        if (!response.ok) {
-                            throw new Error(`Failed to fetch content: ${response.statusText}`);
-                        }
-                        return await response.text();
-                    }
-                }));
                 return job;
             }
             if (job.recommended_pull_delay_ms > 0) {
@@ -158,15 +166,19 @@ class WebcrawlerClient {
         }
         throw new errors_1.WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
     }
-    async crawlAsync(crawlRequest) {
+    async crawlAsync(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
+        const requestBody = {
+            ...crawlRequest,
+            actions: actions ? (Array.isArray(actions) ? actions : [actions]) : undefined
+        };
         const requestOptions = {
             'method': 'POST',
             'headers': {
                 'Content-Type': 'application/json',
                 'Authorization': `Bearer ${this.apiKey}`
             },
-            'body': JSON.stringify(crawlRequest),
+            'body': JSON.stringify(requestBody),
         };
         return await this.sendRequest(url, requestOptions);
     }
@@ -183,7 +195,8 @@ class WebcrawlerClient {
                 'Expires': '0'
             }
         };
-        return await this.sendRequest(url, requestOptions);
+        const job = await this.sendRequest(url, requestOptions);
+        return addGetContentMethod(job);
     }
     async sendRequest(url, requestOptions) {
         let response;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.1.5",
+  "version": "1.1.7",
 
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",