webcrawlerapi-js 2.2.0 → 2.2.2

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action, JobMarkdownResponse, AgentRunRequest, AgentRun } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, JobMarkdownResponse, AgentRunRequest, AgentRun } from "./model";
 import { JobStatus, ErrorCode } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus, ErrorCode };
@@ -15,8 +15,8 @@ export declare class WebcrawlerClient {
     runAgent(request: AgentRunRequest, maxPolls?: number): Promise<AgentRun>;
     getScrape(scrapeId: string): Promise<ScrapeResponse | ScrapeResponseError>;
     scrape(request: ScrapeRequest, maxPolls?: number): Promise<ScrapeResponse | ScrapeResponseError>;
-    crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;
-    crawlAsync(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<JobId>;
+    crawl(crawlRequest: CrawlRequest, maxPolls?: number): Promise<Job>;
+    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;
     getJobMarkdown(jobID: string): Promise<JobMarkdownResponse>;
     getJobMarkdownContent(jobID: string): Promise<string>;

package/dist/@types/model.d.ts

@@ -53,7 +53,6 @@ export interface CrawlRequest {
     blacklist_regexp?: string;
     /** URL where the server will send a POST request once the job is completed. */
     webhook_url?: string;
-    actions?: Action[];
     /** If true, the crawler will respect the website's robots.txt and skip disallowed pages. Default is false. */
     respect_robots_txt?: boolean;
     /** Extract only the main content of the page. Works best for articles, blog posts, news. */
@@ -62,6 +61,8 @@ export interface CrawlRequest {
     max_depth?: number;
     /** Maximum age of cached content in seconds. Set to 0 to always fetch fresh content. */
     max_age?: number;
+    /** If true, keeps query parameters in crawled URLs instead of stripping them. Default is false. */
+    keep_query_params?: boolean;
 }
 export interface Job {
     id: string;
@@ -95,7 +96,7 @@ export interface JobItem {
     updated_at: string;
     cost: number;
     referred_url: string;
-    link?: string;
+    links?: string;
     depth?: number;
     raw_content_url?: string;
     cleaned_content_url?: string;

package/dist/api.js

@@ -211,12 +211,11 @@ class WebcrawlerClient {
         // Return the last known state if maxPolls is reached
         return result;
     }
-    async crawl(crawlRequest, actions) {
+    async crawl(crawlRequest, maxPolls = MaxPullRetries) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
         const requestBody = {
             output_formats: ['markdown'],
-            ...crawlRequest,
-            actions: actions ? (Array.isArray(actions) ? actions : [actions]) : undefined
+            ...crawlRequest
         };
         const requestOptions = {
             'method': 'POST',
@@ -235,7 +234,7 @@ class WebcrawlerClient {
             throw new errors_1.WebcrawlerApiError('invalid_response', 'Failed to fetch job status', 0);
         }
         let delayIntervalMs = initialPullDelayMs;
-        for (let i = 0; i < MaxPullRetries; i++) {
+        for (let i = 0; i < maxPolls; i++) {
             await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
             const timestamp = new Date().getTime();
             const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
@@ -248,12 +247,11 @@ class WebcrawlerClient {
         }
         throw new errors_1.WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
     }
-    async crawlAsync(crawlRequest, actions) {
+    async crawlAsync(crawlRequest) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
         const requestBody = {
             output_formats: ['markdown'],
-            ...crawlRequest,
-            actions: actions ? (Array.isArray(actions) ? actions : [actions]) : undefined
+            ...crawlRequest
         };
         const requestOptions = {
             'method': 'POST',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "2.2.0",
+  "version": "2.2.2",
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",