webcrawlerapi-js 2.0.3 → 2.0.5

package/README.md

@@ -18,6 +18,7 @@ Install WebcrawlerAPI js package:
 The package is regularly updated with new features and improvements to ensure the best possible scraping experience.
 
 ## JavaScript Examples
+Use `crawlRawMarkdown` when you want a crawl to finish and immediately return the combined `/job/{id}/markdown` content.
 
 ### Synchronous Job
 In this mode, the promise will be resolved when all data is ready. Synchronous jobs are perfect for quick scraping tasks where you need immediate results and don't want to handle polling logic. However, for long running jobs we recommend to use async crawling or the webhook.
@@ -176,4 +177,4 @@ Example of a job response. The response structure is carefully designed to provi
   ],
   recommended_pull_delay_ms: 5000
 }
-```
+```

package/dist/@types/api.d.ts

@@ -11,8 +11,10 @@ export declare class WebcrawlerClient {
     scrapeAsync(request: ScrapeRequest): Promise<ScrapeId>;
     getScrape(scrapeId: string): Promise<ScrapeResponse | ScrapeResponseError>;
     scrape(request: ScrapeRequest, maxPolls?: number): Promise<ScrapeResponse | ScrapeResponseError>;
+    crawlRawMarkdown(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<string>;
     crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;
     crawlAsync(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;
+    getJobMarkdown(jobID: string): Promise<string>;
     private sendRequest;
 }

package/dist/@types/model.d.ts

@@ -42,6 +42,7 @@ export interface CrawlRequest {
     actions?: Action[];
     respect_robots_txt?: boolean;
     main_content_only?: boolean;
+    max_depth?: number;
 }
 export interface Job {
     id: string;
@@ -53,6 +54,7 @@ export interface Job {
     blacklist_regexp: string;
     allow_subdomains: boolean;
     items_limit: number;
+    max_depth?: number;
     created_at: string;
     finished_at: string;
     updated_at: string;
@@ -73,6 +75,7 @@ export interface JobItem {
     updated_at: string;
     cost: number;
     referred_url: string;
+    depth?: number;
     raw_content_url?: string;
     cleaned_content_url?: string;
     getContent(): Promise<string | null>;

package/dist/api.js

@@ -138,6 +138,20 @@ class WebcrawlerClient {
         // Return the last known state if maxPolls is reached
         return result;
     }
+    async crawlRawMarkdown(crawlRequest, actions) {
+        const markdownRequest = {
+            ...crawlRequest,
+            scrape_type: crawlRequest.scrape_type ?? 'markdown'
+        };
+        const job = await this.crawl(markdownRequest, actions);
+        if (job.status !== constants_1.JobStatus.DONE) {
+            throw new errors_1.WebcrawlerApiError('job_not_done', `Job finished with status ${job.status}`, 0);
+        }
+        if (job.scrape_type !== 'markdown') {
+            throw new errors_1.WebcrawlerApiError('invalid_scrape_type', 'CrawlRawMarkdown requires scrape_type to be markdown', 0);
+        }
+        return this.getJobMarkdown(job.id);
+    }
     async crawl(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
         const requestBody = {
@@ -206,6 +220,40 @@ class WebcrawlerClient {
         const job = await this.sendRequest(url, requestOptions);
         return addGetContentMethod(job);
     }
+    async getJobMarkdown(jobID) {
+        const url = `${this.basePath}/${this.apiVersion}/job/${jobID}/markdown`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                'Cache-Control': 'no-cache, no-store, must-revalidate',
+                'Pragma': 'no-cache',
+                'Expires': '0'
+            }
+        };
+        let response;
+        try {
+            response = await fetch(url, requestOptions);
+        }
+        catch (e) {
+            throw new errors_1.WebcrawlerApiError('network_error', `Failed to send request: ${e}`, 0);
+        }
+        if (!response.ok) {
+            try {
+                const errorData = await response.json();
+                throw (0, errors_1.createErrorFromResponse)(response, errorData);
+            }
+            catch (e) {
+                if (e instanceof errors_1.WebcrawlerApiError) {
+                    throw e;
+                }
+                throw new errors_1.WebcrawlerApiError('unknown_error', `Request failed with status ${response.status} ${response.statusText}`, response.status);
+            }
+        }
+        return response.text();
+    }
     async sendRequest(url, requestOptions) {
         let response;
         try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "2.0.3",
+  "version": "2.0.5",
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",
@@ -28,7 +28,6 @@
     "type": "git",
     "url": "https://github.com/WebCrawlerAPI/webcrawlerapi-js-sdk"
   },
-  "dependencies": {},
   "scripts": {
     "build": "tsc --outDir dist/"
   }