webcrawlerapi-js 1.1.4 → 1.1.6

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, Action } from "./model";
 import { JobStatus } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus };
@@ -12,8 +12,8 @@ export declare class WebcrawlerClient {
     scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
     scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
     getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
-    crawl(crawlRequest: CrawlRequest): Promise<Job>;
-    crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
+    crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;
+    crawlAsync(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;
     private sendRequest;
 }

package/dist/@types/model.d.ts

@@ -18,7 +18,7 @@ export interface ScrapeResponse {
     recommended_pull_delay_ms: number;
 }
 export interface CrawlRequest {
-    url: string;
+    url?: string;
     crawler_id?: string;
     items_limit?: number;
     scrape_type?: string;
@@ -26,6 +26,7 @@ export interface CrawlRequest {
     blacklist_regexp?: string;
     allow_subdomains?: boolean;
     webhook_url?: string;
+    actions?: Action[];
 }
 export interface Job {
     id: string;
@@ -61,3 +62,14 @@ export interface JobItem {
     cleaned_content_url?: string;
     getContent(): Promise<string | null>;
 }
+export interface Action {
+    type: string;
+}
+export interface UploadS3Action extends Action {
+    type: 'upload_s3';
+    path: string;
+    access_key_id: string;
+    secret_access_key: string;
+    bucket: string;
+    endpoint?: string;
+}

package/dist/api.js

@@ -92,8 +92,12 @@ class WebcrawlerClient {
         };
         return await this.sendRequest(url, requestOptions);
     }
-    async crawl(crawlRequest) {
+    async crawl(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
+        const requestBody = {
+            ...crawlRequest,
+            actions: actions ? (Array.isArray(actions) ? actions : [actions]) : undefined
+        };
         const requestOptions = {
             'method': 'POST',
             'headers': {
@@ -104,7 +108,7 @@ class WebcrawlerClient {
                 'Pragma': 'no-cache',
                 'Expires': '0'
             },
-            'body': JSON.stringify(crawlRequest),
+            'body': JSON.stringify(requestBody),
         };
         const jobIdResponse = await this.sendRequest(url, requestOptions);
         if (jobIdResponse.id === '') {
@@ -158,15 +162,19 @@ class WebcrawlerClient {
         }
         throw new errors_1.WebcrawlerApiError('timeout', 'Crawling took too long, please retry or increase the number of polling retries', 0);
     }
-    async crawlAsync(crawlRequest) {
+    async crawlAsync(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
+        const requestBody = {
+            ...crawlRequest,
+            actions: actions ? (Array.isArray(actions) ? actions : [actions]) : undefined
+        };
         const requestOptions = {
             'method': 'POST',
             'headers': {
                 'Content-Type': 'application/json',
                 'Authorization': `Bearer ${this.apiKey}`
             },
-            'body': JSON.stringify(crawlRequest),
+            'body': JSON.stringify(requestBody),
         };
         return await this.sendRequest(url, requestOptions);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.1.4",
+  "version": "1.1.6",
 
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",