webcrawlerapi-js 2.1.0 → 2.2.0

package/README.md

@@ -46,6 +46,31 @@ for (const item of job.job_items) {
 }
 ```
 
+## Run an agent
+
+```javascript
+import webcrawlerapi from "webcrawlerapi-js"
+
+const client = new webcrawlerapi.WebcrawlerClient("YOUR_API_KEY")
+
+const result = await client.runAgent({
+    prompt: "Visit the provided website and return the page title as JSON.",
+    urls: ["https://books.toscrape.com/"],
+    seed_urls_only: true,
+    output_schema: {
+        type: "object",
+        properties: {
+            title: { type: "string" },
+        },
+        required: ["title"],
+        additionalProperties: false,
+    },
+    max_spend_usd: 0.02,
+})
+
+console.log(result.status, result.data)
+```
+
 ## Documentation
 
 - [Scraping](docs/scraping.md) — single page, async mode, AI extraction, parameters, response fields

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action, JobMarkdownResponse } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action, JobMarkdownResponse, AgentRunRequest, AgentRun } from "./model";
 import { JobStatus, ErrorCode } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus, ErrorCode };
@@ -8,7 +8,11 @@ export declare class WebcrawlerClient {
     protected apiKey: string;
     protected basePath: string;
     constructor(apiKey: string, basePath?: string, apiVersion?: string);
+    private getDefaultHeaders;
     scrapeAsync(request: ScrapeRequest): Promise<ScrapeId>;
+    runAgentAsync(request: AgentRunRequest): Promise<AgentRun>;
+    getAgentJob(jobID: string): Promise<AgentRun>;
+    runAgent(request: AgentRunRequest, maxPolls?: number): Promise<AgentRun>;
     getScrape(scrapeId: string): Promise<ScrapeResponse | ScrapeResponseError>;
     scrape(request: ScrapeRequest, maxPolls?: number): Promise<ScrapeResponse | ScrapeResponseError>;
     crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;

package/dist/@types/model.d.ts

@@ -119,3 +119,35 @@ export interface UploadS3Action extends Action {
 export interface JobMarkdownResponse {
     content_url: string;
 }
+export interface AgentRunRequest {
+    /** Natural-language task for the agent to complete. */
+    prompt: string;
+    /** Optional seed URLs the agent can use while running the task. */
+    urls?: string[];
+    /** If true, the agent only uses the provided seed URLs. */
+    seed_urls_only?: boolean;
+    /** Optional JSON schema for structured agent output. */
+    output_schema?: Record<string, any>;
+    /** Model to use for the run. Omit to use the API default. */
+    model?: string;
+    /** Maximum spend for this run in USD. Required by the API. */
+    max_spend_usd: number;
+}
+export interface AgentRun {
+    id: string;
+    status: string;
+    prompt: string;
+    model: string;
+    data?: any;
+    error_reason?: string;
+    balance_used_usd: number;
+    max_spend_usd?: number;
+    urls?: string[];
+    created_at: string;
+    updated_at: string;
+    success: boolean;
+}
+export interface AgentRunList {
+    items: AgentRun[];
+    total: number;
+}

package/dist/api.js

@@ -24,6 +24,7 @@ const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
 const DEFAULT_POLL_DELAY_SECONDS = 2;
+const AgentTerminalStatuses = ["done", "error", "canceled"];
 __exportStar(require("./model"), exports);
 const SCRAPE_VERSION = "v2";
 function resolveContentUrl(job, item) {
@@ -94,6 +95,19 @@ class WebcrawlerClient {
         this.basePath = basePath;
         this.apiVersion = apiVersion;
     }
+    getDefaultHeaders(cacheControl = false) {
+        const headers = {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${this.apiKey}`,
+            "User-Agent": "WebcrawlerAPI-NodeJS-Client"
+        };
+        if (cacheControl) {
+            headers['Cache-Control'] = 'no-cache, no-store, must-revalidate';
+            headers['Pragma'] = 'no-cache';
+            headers['Expires'] = '0';
+        }
+        return headers;
+    }
     async scrapeAsync(request) {
         const apiUrl = `${this.basePath}/${SCRAPE_VERSION}/scrape?async=true`;
         const requestOptions = {
@@ -108,6 +122,41 @@ class WebcrawlerClient {
         const response = await this.sendRequest(apiUrl, requestOptions);
         return { id: response.id };
     }
+    async runAgentAsync(request) {
+        const url = `${this.basePath}/${this.apiVersion}/agent`;
+        const requestOptions = {
+            'method': 'POST',
+            'headers': this.getDefaultHeaders(),
+            'body': JSON.stringify(request),
+        };
+        return await this.sendRequest(url, requestOptions);
+    }
+    async getAgentJob(jobID) {
+        const url = `${this.basePath}/${this.apiVersion}/agent/job/${jobID}`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': this.getDefaultHeaders(true)
+        };
+        return await this.sendRequest(url, requestOptions);
+    }
+    async runAgent(request, maxPolls = MaxPullRetries) {
+        const run = await this.runAgentAsync(request);
+        if (!run.id) {
+            throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch agent job status', 0);
+        }
+        if (AgentTerminalStatuses.includes(run.status)) {
+            return run;
+        }
+        for (let i = 0; i < maxPolls; i++) {
+            await new Promise(resolve => setTimeout(resolve, DEFAULT_POLL_DELAY_SECONDS * 1000));
+            const timestamp = new Date().getTime();
+            const agentJob = await this.getAgentJob(`${run.id}?t=${timestamp}`);
+            if (AgentTerminalStatuses.includes(agentJob.status)) {
+                return agentJob;
+            }
+        }
+        throw new errors_1.WebcrawlerApiError('timeout', 'Agent run took too long, please retry or increase the number of polling retries', 0);
+    }
     async getScrape(scrapeId) {
         const url = `${this.basePath}/${SCRAPE_VERSION}/scrape/${scrapeId}`;
         const requestOptions = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",