webcrawlerapi-js 2.2.1 → 2.2.5

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action, JobMarkdownResponse, AgentRunRequest, AgentRun } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, JobMarkdownResponse, MarkdownRequest, MarkdownResponse, AgentRunRequest, AgentRun } from "./model";
 import { JobStatus, ErrorCode } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus, ErrorCode };
@@ -15,10 +15,11 @@ export declare class WebcrawlerClient {
     runAgent(request: AgentRunRequest, maxPolls?: number): Promise<AgentRun>;
     getScrape(scrapeId: string): Promise<ScrapeResponse | ScrapeResponseError>;
     scrape(request: ScrapeRequest, maxPolls?: number): Promise<ScrapeResponse | ScrapeResponseError>;
-    crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;
-    crawlAsync(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<JobId>;
+    crawl(crawlRequest: CrawlRequest, maxPolls?: number): Promise<Job>;
+    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;
     getJobMarkdown(jobID: string): Promise<JobMarkdownResponse>;
     getJobMarkdownContent(jobID: string): Promise<string>;
+    markdown(request: MarkdownRequest): Promise<MarkdownResponse>;
     private sendRequest;
 }

package/dist/@types/model.d.ts

@@ -53,7 +53,6 @@ export interface CrawlRequest {
     blacklist_regexp?: string;
     /** URL where the server will send a POST request once the job is completed. */
     webhook_url?: string;
-    actions?: Action[];
     /** If true, the crawler will respect the website's robots.txt and skip disallowed pages. Default is false. */
     respect_robots_txt?: boolean;
     /** Extract only the main content of the page. Works best for articles, blog posts, news. */
@@ -62,6 +61,8 @@ export interface CrawlRequest {
     max_depth?: number;
     /** Maximum age of cached content in seconds. Set to 0 to always fetch fresh content. */
     max_age?: number;
+    /** If true, keeps query parameters in crawled URLs instead of stripping them. Default is false. */
+    keep_query_params?: boolean;
 }
 export interface Job {
     id: string;
@@ -119,6 +120,15 @@ export interface UploadS3Action extends Action {
 export interface JobMarkdownResponse {
     content_url: string;
 }
+export interface MarkdownRequest {
+    /** The URL of the webpage to extract the article markdown from. */
+    url: string;
+}
+export interface MarkdownResponse {
+    success: boolean;
+    /** Cleaned article markdown (main content only). */
+    markdown?: string;
+}
 export interface AgentRunRequest {
     /** Natural-language task for the agent to complete. */
     prompt: string;

package/dist/api.js

@@ -211,12 +211,11 @@ class WebcrawlerClient {
         // Return the last known state if maxPolls is reached
         return result;
     }
-    async crawl(crawlRequest, actions) {
+    async crawl(crawlRequest, maxPolls = MaxPullRetries) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
         const requestBody = {
             output_formats: ['markdown'],
-            ...crawlRequest,
-            actions: actions ? (Array.isArray(actions) ? actions : [actions]) : undefined
+            ...crawlRequest
         };
         const requestOptions = {
             'method': 'POST',
@@ -235,7 +234,7 @@ class WebcrawlerClient {
             throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
         }
         let delayIntervalMs = initialPullDelayMs;
-        for (let i = 0; i < MaxPullRetries; i++) {
+        for (let i = 0; i < maxPolls; i++) {
             await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
             const timestamp = new Date().getTime();
             const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
@@ -248,12 +247,11 @@ class WebcrawlerClient {
         }
         throw new errors_1.WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
     }
-    async crawlAsync(crawlRequest, actions) {
+    async crawlAsync(crawlRequest) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
         const requestBody = {
             output_formats: ['markdown'],
-            ...crawlRequest,
-            actions: actions ? (Array.isArray(actions) ? actions : [actions]) : undefined
+            ...crawlRequest
         };
         const requestOptions = {
             'method': 'POST',
@@ -330,6 +328,15 @@ class WebcrawlerClient {
         }
         return response.text();
     }
+    async markdown(request) {
+        const url = `${this.basePath}/markdown`;
+        const requestOptions = {
+            'method': 'POST',
+            'headers': this.getDefaultHeaders(),
+            'body': JSON.stringify(request),
+        };
+        return this.sendRequest(url, requestOptions);
+    }
     async sendRequest(url, requestOptions) {
         let response;
         try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "2.2.1",
+  "version": "2.2.5",
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",