webcrawlerapi-js 2.0.9 → 2.0.10

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action, JobMarkdownResponse } from "./model";
 import { JobStatus, ErrorCode } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus, ErrorCode };
@@ -15,6 +15,7 @@ export declare class WebcrawlerClient {
     crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;
     crawlAsync(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;
-    getJobMarkdown(jobID: string): Promise<string>;
+    getJobMarkdown(jobID: string): Promise<JobMarkdownResponse>;
+    getJobMarkdownContent(jobID: string): Promise<string>;
     private sendRequest;
 }

package/dist/@types/model.d.ts

@@ -94,3 +94,6 @@ export interface UploadS3Action extends Action {
     bucket: string;
     endpoint?: string;
 }
+export interface JobMarkdownResponse {
+    content_url: string;
+}

package/dist/api.js

@@ -150,7 +150,7 @@ class WebcrawlerClient {
         if (job.scrape_type !== 'markdown') {
             throw new errors_1.WebcrawlerApiError('invalid_scrape_type', 'CrawlRawMarkdown requires scrape_type to be markdown', 0);
         }
-        return this.getJobMarkdown(job.id);
+        return this.getJobMarkdownContent(job.id);
     }
     async crawl(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
@@ -233,6 +233,21 @@ class WebcrawlerClient {
                 'Expires': '0'
             }
         };
+        return this.sendRequest(url, requestOptions);
+    }
+    async getJobMarkdownContent(jobID) {
+        const url = `${this.basePath}/${this.apiVersion}/job/${jobID}/markdown/content`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                'Cache-Control': 'no-cache, no-store, must-revalidate',
+                'Pragma': 'no-cache',
+                'Expires': '0'
+            }
+        };
         let response;
         try {
             response = await fetch(url, requestOptions);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "2.0.9",
+  "version": "2.0.10",
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",