webcrawlerapi-js 2.0.11 → 2.2.0

package/README.md

@@ -1,179 +1,77 @@
 # JS client for WebcrawlerAPI
 
-Official client for [WebcrawlerAPI](https://webcrawlerapi.com/). This SDK provides a robust and type-safe way to interact with WebcrawlerAPI's powerful web scraping capabilities, making it easy to extract structured data from any website.
+Webcrawler API for your AI agent. Get markdown of any website or page with a single call.
 
-WebcrawlerAPI allows you to extract data from any website with just a simple API call. The service handles all the complexities of web scraping, including JavaScript rendering, proxy rotation, and rate limiting, so you can focus on using the data.
-
-## Preparation
-1. Register to [dashboard|https://dash.webcrawlerapi.com/].
-2. Get an [Access Key](https://dash.webcrawlerapi.com/access).
-
-Our dashboard provides comprehensive analytics and monitoring tools to help you track your API usage and optimize your scraping operations.
+Official JavaScript/TypeScript client for [WebcrawlerAPI](https://webcrawlerapi.com/) — web scraping and crawling with JS rendering, proxy rotation, and AI extraction.
 
 ## Installation
-Install WebcrawlerAPI js package:
-
-`npm i webcrawlerapi-js`
 
-The package is regularly updated with new features and improvements to ensure the best possible scraping experience.
+```
+npm i webcrawlerapi-js
+```
 
-## JavaScript Examples
-Use `crawlRawMarkdown` when you want a crawl to finish and immediately return the combined `/job/{id}/markdown` content.
+Get your API key at [dash.webcrawlerapi.com/access](https://dash.webcrawlerapi.com/access).
 
-### Synchronous Job
-In this mode, the promise will be resolved when all data is ready. Synchronous jobs are perfect for quick scraping tasks where you need immediate results and don't want to handle polling logic. However, for long running jobs we recommend to use async crawling or the webhook.
+## Scrape a single page
 
 ```javascript
 import webcrawlerapi from "webcrawlerapi-js"
 
-async function main() {
-    const client = new webcrawlerapi.WebcrawlerClient("YOUR API ACCESS KEY HERE")
-
-    try {
-        const crawlJob = await client.crawl({
-            "items_limit": 10,
-            "url": "https://books.toscrape.com/",
-            "scrape_type": "markdown"
-        })
-        
-        // Get job details
-        const job = await client.getJob(crawlJob.id);
-        console.log("Job status:", job.status);
-
-        // Output item content
-        for (const item of job.job_items) {
-            const content = await item.getContent()
-            console.log("---- Title: " + item.title + "----")
-            console.log(content.slice(0, 100))
-        }
-    } catch (error) {
-        if (error instanceof webcrawlerapi.WebcrawlerApiError) {
-            console.error("Webcrawler error:", error.message)
-        }
-        console.error(error)
-    }
-}
+const client = new webcrawlerapi.WebcrawlerClient("YOUR_API_KEY")
+
+const result = await client.scrape({
+    url: "https://books.toscrape.com/",
+    output_formats: ["markdown"],
+})
 
-main().catch(console.error)
+console.log(result.markdown)
 ```
 
-### Asynchronous Job
-In this mode, you get a job ID immediately and can poll for results. Asynchronous jobs are ideal for large-scale scraping operations where you want to process data in batches or handle long-running tasks efficiently.
+## Crawl multiple pages
 
 ```javascript
 import webcrawlerapi from "webcrawlerapi-js"
 
-async function main() {
-    const client = new webcrawlerapi.WebcrawlerClient("YOUR API ACCESS KEY HERE")
-
-    try {
-        try {
-        const jobWithId = await client.crawlAsync({
-            "items_limit": 10,
-            "url": "https://books.toscrape.com/",
-            "scrape_type": "markdown"
-        })
-
-        // Poll for job completion
-        const jobId = jobWithId.id
-        let asyncJob = await client.getJob(jobId)
-
-        // Check job status
-        console.log("Job id:", asyncJob.id)
-        console.log("Dashboard link:", `https://dash.webcrawlerapi.com/jobs/job/${asyncJob.id}`)
-
-        //wait until job is done
-        while (asyncJob.status !== "done") {
-            await new Promise(resolve => setTimeout(resolve, 1000));
-            asyncJob = await client.getJob(jobId)
-        }
-
-        // Once complete, access results
-        if (asyncJob.status === "done") {
-            for (const item of asyncJob.job_items) {
-                const content = await item.getContent()
-                console.log(content.slice(0, 100))
-            }
-        }
-    } catch (error) {
-        if (error instanceof webcrawlerapi.WebcrawlerApiError) {
-            console.error("Webcrawler error:", error.message)
-        }
-        console.error(error)
-        return
-    }
-}
+const client = new webcrawlerapi.WebcrawlerClient("YOUR_API_KEY")
 
-main().catch(console.error)
-```
+const job = await client.crawl({
+    url: "https://books.toscrape.com/",
+    items_limit: 10,
+    output_formats: ["markdown"],
+})
 
-## TypeScript Usage
-
-The SDK provides full TypeScript support with type safety and error handling. Our TypeScript integration ensures you catch potential issues at compile-time and provides excellent IDE support with autocompletion and inline documentation.
-
-```typescript
-import {WebcrawlerClient, WebcrawlerApiError} from "webcrawlerapi-js";
-
-async function main() {
-    const client = new webcrawlerapi.WebcrawlerClient("YOUR_API_KEY")
-
-    try {
-        // Basic crawl with markdown
-        const syncJob = await client.crawl({
-            "items_limit": 10,
-            "url": "https://books.toscrape.com/",
-            "scrape_type": "markdown",
-        }
-    )
-        // Type-safe status check
-        for (const item of syncJob.job_items) {
-            const content = await item.getContent()
-            console.log(content.slice(0,100))
-        }
-    } catch (error) {
-        if (error instanceof WebcrawlerApiError) {
-            console.error("API Error:", error.errorCode, error.message)
-        }
-        console.log(error)
-    }
+for (const item of job.job_items) {
+    const content = await item.getContent()
+    console.log(item.title, content?.slice(0, 100))
 }
-
-main().catch(console.error)
 ```
 
-## Response Structure
-
-Example of a job response. The response structure is carefully designed to provide comprehensive information about your scraping job, including detailed status updates, timing information, and resource usage metrics.
-
-```js
-{
-  id: '49c4942b-b7d9-4d62-94b5-b54a3016ac51',
-  url: 'https://stripe.com/',
-  scrape_type: 'markdown',
-  whitelist_regexp: '',
-  blacklist_regexp: '',
-  items_limit: 10,
-  created_at: '2024-12-28T21:36:04.417Z',
-  finished_at: null,
-  updated_at: '2024-12-28T21:36:04.383Z',
-  webhook_url: '',
-  status: 'in_progress',
-  job_items: [
-    {
-      id: 'f26cefe1-09d1-4d4c-8b74-b65e075e230d',
-      job_id: '49c4942a-b7d9-4d62-94b5-b54a3016ac51',
-      original_url: 'https://stripe.com/',
-      page_status_code: 0,
-      status: 'new',
-      title: '',
-      last_error: '',
-      error_code: '',
-      created_at: '2024-12-28T21:36:04.468Z',
-      updated_at: '2024-12-28T21:36:04.435Z',
-      cost: 0,
-      referred_url: ''
-    }
-  ],
-  recommended_pull_delay_ms: 5000
-}
+## Run an agent
+
+```javascript
+import webcrawlerapi from "webcrawlerapi-js"
+
+const client = new webcrawlerapi.WebcrawlerClient("YOUR_API_KEY")
+
+const result = await client.runAgent({
+    prompt: "Visit the provided website and return the page title as JSON.",
+    urls: ["https://books.toscrape.com/"],
+    seed_urls_only: true,
+    output_schema: {
+        type: "object",
+        properties: {
+            title: { type: "string" },
+        },
+        required: ["title"],
+        additionalProperties: false,
+    },
+    max_spend_usd: 0.02,
+})
+
+console.log(result.status, result.data)
 ```
+
+## Documentation
+
+- [Scraping](docs/scraping.md) — single page, async mode, AI extraction, parameters, response fields
+- [Crawling](docs/crawling.md) — multi-page, async mode, combined markdown, parameters, job items

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action, JobMarkdownResponse } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action, JobMarkdownResponse, AgentRunRequest, AgentRun } from "./model";
 import { JobStatus, ErrorCode } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus, ErrorCode };
@@ -8,10 +8,13 @@ export declare class WebcrawlerClient {
     protected apiKey: string;
     protected basePath: string;
     constructor(apiKey: string, basePath?: string, apiVersion?: string);
+    private getDefaultHeaders;
     scrapeAsync(request: ScrapeRequest): Promise<ScrapeId>;
+    runAgentAsync(request: AgentRunRequest): Promise<AgentRun>;
+    getAgentJob(jobID: string): Promise<AgentRun>;
+    runAgent(request: AgentRunRequest, maxPolls?: number): Promise<AgentRun>;
     getScrape(scrapeId: string): Promise<ScrapeResponse | ScrapeResponseError>;
     scrape(request: ScrapeRequest, maxPolls?: number): Promise<ScrapeResponse | ScrapeResponseError>;
-    crawlRawMarkdown(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<string>;
     crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;
     crawlAsync(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;

package/dist/@types/model.d.ts

@@ -1,13 +1,19 @@
 export interface ScrapeRequest {
     url: string;
+    /** @deprecated Use output_formats instead */
     output_format?: string;
+    output_formats?: ('markdown' | 'cleaned' | 'html' | 'links')[];
     webhook_url?: string;
+    /** CSS selectors to clean from the output. */
     clean_selectors?: string;
     actions?: Action[];
+    /** A prompt to run on the scraped content, used to extract specific information or format the output (extra cost will be charged). */
     prompt?: string;
+    /** JSON schema to enforce structured output when using a prompt. Follows the OpenAI Structured Outputs format. */
     response_schema?: Record<string, any>;
-    respect_robots_txt?: boolean;
+    /** Extract only the main content of the page. Works best for articles, blog posts, news. */
     main_content_only?: boolean;
+    /** Maximum age of cached content in seconds. Set to 0 to always fetch fresh content. */
     max_age?: number;
 }
 export interface ScrapeId {
@@ -34,17 +40,27 @@ export interface JobId {
     id: string;
 }
 export interface CrawlRequest {
+    /** The seed URL where the crawler starts. Can be any valid URL. */
     url?: string;
-    crawler_id?: string;
+    /** The crawler will stop when it reaches this limit of pages for this job. */
     items_limit?: number;
+    /** @deprecated Use output_formats instead */
     scrape_type?: string;
+    output_formats?: ('links' | 'markdown' | 'cleaned' | 'html')[];
+    /** Regular expression to whitelist URLs. Only URLs that match the pattern will be crawled. */
     whitelist_regexp?: string;
+    /** Regular expression to blacklist URLs. URLs that match the pattern will be skipped. */
     blacklist_regexp?: string;
+    /** URL where the server will send a POST request once the job is completed. */
     webhook_url?: string;
     actions?: Action[];
+    /** If true, the crawler will respect the website's robots.txt and skip disallowed pages. Default is false. */
     respect_robots_txt?: boolean;
+    /** Extract only the main content of the page. Works best for articles, blog posts, news. */
     main_content_only?: boolean;
+    /** Maximum crawl depth from the starting URL. 0 = starting page only, 1 = starting page + directly linked pages, etc. No limit by default. */
     max_depth?: number;
+    /** Maximum age of cached content in seconds. Set to 0 to always fetch fresh content. */
     max_age?: number;
 }
 export interface Job {
@@ -52,7 +68,9 @@ export interface Job {
     org_id: string;
     url: string;
     status: string;
-    scrape_type: string;
+    /** @deprecated Use output_formats instead */
+    scrape_type?: string;
+    output_formats?: string[];
     whitelist_regexp: string;
     blacklist_regexp: string;
     items_limit: number;
@@ -83,6 +101,9 @@ export interface JobItem {
     cleaned_content_url?: string;
     error_code?: string;
     getContent(): Promise<string | null>;
+    getMarkdown(): Promise<string | null>;
+    getCleaned(): Promise<string | null>;
+    getHTML(): Promise<string | null>;
 }
 export interface Action {
     type: string;
@@ -98,3 +119,35 @@ export interface UploadS3Action extends Action {
 export interface JobMarkdownResponse {
     content_url: string;
 }
+export interface AgentRunRequest {
+    /** Natural-language task for the agent to complete. */
+    prompt: string;
+    /** Optional seed URLs the agent can use while running the task. */
+    urls?: string[];
+    /** If true, the agent only uses the provided seed URLs. */
+    seed_urls_only?: boolean;
+    /** Optional JSON schema for structured agent output. */
+    output_schema?: Record<string, any>;
+    /** Model to use for the run. Omit to use the API default. */
+    model?: string;
+    /** Maximum spend for this run in USD. Required by the API. */
+    max_spend_usd: number;
+}
+export interface AgentRun {
+    id: string;
+    status: string;
+    prompt: string;
+    model: string;
+    data?: any;
+    error_reason?: string;
+    balance_used_usd: number;
+    max_spend_usd?: number;
+    urls?: string[];
+    created_at: string;
+    updated_at: string;
+    success: boolean;
+}
+export interface AgentRunList {
+    items: AgentRun[];
+    total: number;
+}

package/dist/api.js

@@ -24,8 +24,47 @@ const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
 const DEFAULT_POLL_DELAY_SECONDS = 2;
+const AgentTerminalStatuses = ["done", "error", "canceled"];
 __exportStar(require("./model"), exports);
 const SCRAPE_VERSION = "v2";
+function resolveContentUrl(job, item) {
+    // Prefer output_formats if present and non-empty, using priority: markdown > cleaned > html
+    if (Array.isArray(job.output_formats) && job.output_formats.length > 0) {
+        const priority = ['markdown', 'cleaned', 'html'];
+        for (const fmt of priority) {
+            if (job.output_formats.includes(fmt)) {
+                switch (fmt) {
+                    case 'markdown': return item.markdown_content_url;
+                    case 'cleaned': return item.cleaned_content_url;
+                    case 'html': return item.raw_content_url;
+                }
+            }
+        }
+        return undefined;
+    }
+    // Fall back to scrape_type for backward compatibility
+    switch (job.scrape_type) {
+        case 'html': return item.raw_content_url;
+        case 'cleaned': return item.cleaned_content_url;
+        case 'markdown': return item.markdown_content_url;
+        default: return undefined;
+    }
+}
+async function fetchContentUrl(url) {
+    if (!url) {
+        return null;
+    }
+    const response = await fetch(url, {
+        headers: {
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Accept': '*/*'
+        }
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to fetch content: ${response.statusText}`);
+    }
+    return response.text();
+}
 function addGetContentMethod(job) {
     // Transform each job item to include getContent method
     job.job_items = job.job_items.map(item => ({
@@ -34,31 +73,17 @@ function addGetContentMethod(job) {
             if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
                 return null;
             }
-            let contentUrl;
-            switch (job.scrape_type) {
-                case 'html':
-                    contentUrl = this.raw_content_url;
-                    break;
-                case 'cleaned':
-                    contentUrl = this.cleaned_content_url;
-                    break;
-                case 'markdown':
-                    contentUrl = this.markdown_content_url;
-                    break;
-            }
-            if (!contentUrl) {
-                return null;
-            }
-            const response = await fetch(contentUrl, {
-                headers: {
-                    'Accept-Encoding': 'gzip, deflate, br',
-                    'Accept': '*/*'
-                }
-            });
-            if (!response.ok) {
-                throw new Error(`Failed to fetch content: ${response.statusText}`);
-            }
-            return await response.text();
+            const contentUrl = resolveContentUrl(job, this);
+            return fetchContentUrl(contentUrl);
+        },
+        getMarkdown: async function () {
+            return fetchContentUrl(this.markdown_content_url);
+        },
+        getCleaned: async function () {
+            return fetchContentUrl(this.cleaned_content_url);
+        },
+        getHTML: async function () {
+            return fetchContentUrl(this.raw_content_url);
         }
     }));
     return job;
@@ -70,6 +95,19 @@ class WebcrawlerClient {
         this.basePath = basePath;
         this.apiVersion = apiVersion;
     }
+    getDefaultHeaders(cacheControl = false) {
+        const headers = {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${this.apiKey}`,
+            "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+        };
+        if (cacheControl) {
+            headers['Cache-Control'] = 'no-cache, no-store, must-revalidate';
+            headers['Pragma'] = 'no-cache';
+            headers['Expires'] = '0';
+        }
+        return headers;
+    }
     async scrapeAsync(request) {
         const apiUrl = `${this.basePath}/${SCRAPE_VERSION}/scrape?async=true`;
         const requestOptions = {
@@ -84,6 +122,41 @@ class WebcrawlerClient {
         const response = await this.sendRequest(apiUrl, requestOptions);
         return { id: response.id };
     }
+    async runAgentAsync(request) {
+        const url = `${this.basePath}/${this.apiVersion}/agent`;
+        const requestOptions = {
+            'method': 'POST',
+            'headers': this.getDefaultHeaders(),
+            'body': JSON.stringify(request),
+        };
+        return await this.sendRequest(url, requestOptions);
+    }
+    async getAgentJob(jobID) {
+        const url = `${this.basePath}/${this.apiVersion}/agent/job/${jobID}`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': this.getDefaultHeaders(true)
+        };
+        return await this.sendRequest(url, requestOptions);
+    }
+    async runAgent(request, maxPolls = MaxPullRetries) {
+        const run = await this.runAgentAsync(request);
+        if (!run.id) {
+            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch agent job status', 0);
+        }
+        if (AgentTerminalStatuses.includes(run.status)) {
+            return run;
+        }
+        for (let i = 0; i < maxPolls; i++) {
+            await new Promise(resolve => setTimeout(resolve, DEFAULT_POLL_DELAY_SECONDS * 1000));
+            const timestamp = new Date().getTime();
+            const agentJob = await this.getAgentJob(`${run.id}?t=${timestamp}`);
+            if (AgentTerminalStatuses.includes(agentJob.status)) {
+                return agentJob;
+            }
+        }
+        throw new errors_1.WebcrawlerApiError('timeout', 'Agent run took too long, please retry or increase the number of polling retries', 0);
+    }
     async getScrape(scrapeId) {
         const url = `${this.basePath}/${SCRAPE_VERSION}/scrape/${scrapeId}`;
         const requestOptions = {
@@ -138,23 +211,10 @@ class WebcrawlerClient {
         // Return the last known state if maxPolls is reached
         return result;
     }
-    async crawlRawMarkdown(crawlRequest, actions) {
-        const markdownRequest = {
-            ...crawlRequest,
-            scrape_type: crawlRequest.scrape_type ?? 'markdown'
-        };
-        const job = await this.crawl(markdownRequest, actions);
-        if (job.status !== constants_1.JobStatus.DONE) {
-            throw new errors_1.WebcrawlerApiError('job_not_done', `Job finished with status ${job.status}`, 0);
-        }
-        if (job.scrape_type !== 'markdown') {
-            throw new errors_1.WebcrawlerApiError('invalid_scrape_type', 'CrawlRawMarkdown requires scrape_type to be markdown', 0);
-        }
-        return this.getJobMarkdownContent(job.id);
-    }
     async crawl(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
         const requestBody = {
+            output_formats: ['markdown'],
             ...crawlRequest,
             actions: actions ? (Array.isArray(actions) ? actions : [actions]) : undefined
         };
@@ -191,6 +251,7 @@ class WebcrawlerClient {
     async crawlAsync(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
         const requestBody = {
+            output_formats: ['markdown'],
             ...crawlRequest,
             actions: actions ? (Array.isArray(actions) ? actions : [actions]) : undefined
         };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "2.0.11",
+  "version": "2.2.0",
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",