webcrawlerapi-js 2.0.7 → 2.0.10

package/README.md

@@ -27,7 +27,7 @@ In this mode, the promise will be resolved when all data is ready. Synchronous j
 import webcrawlerapi from "webcrawlerapi-js"
 
 async function main() {
-    const client = new WebcrawlerClient("YOUR API ACCESS KEY HERE")
+    const client = new webcrawlerapi.WebcrawlerClient("YOUR API ACCESS KEY HERE")
 
     try {
         const crawlJob = await client.crawl({
@@ -64,7 +64,7 @@ In this mode, you get a job ID immediately and can poll for results. Asynchronou
 import webcrawlerapi from "webcrawlerapi-js"
 
 async function main() {
-    const client = new WebcrawlerClient("YOUR API ACCESS KEY HERE")
+    const client = new webcrawlerapi.WebcrawlerClient("YOUR API ACCESS KEY HERE")
 
     try {
         try {
@@ -115,7 +115,7 @@ The SDK provides full TypeScript support with type safety and error handling. Ou
 import {WebcrawlerClient, WebcrawlerApiError} from "webcrawlerapi-js";
 
 async function main() {
-    const client = new WebcrawlerClient("YOUR_API_KEY")
+    const client = new webcrawlerapi.WebcrawlerClient("YOUR_API_KEY")
 
     try {
         // Basic crawl with markdown

package/dist/@types/api.d.ts

@@ -1,4 +1,4 @@
-import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action } from "./model";
+import { CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse, ScrapeResponseError, ScrapeId, Action, JobMarkdownResponse } from "./model";
 import { JobStatus, ErrorCode } from "./constants";
 import { WebcrawlerApiError } from "./errors";
 export { WebcrawlerApiError, JobStatus, ErrorCode };
@@ -15,6 +15,7 @@ export declare class WebcrawlerClient {
     crawl(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<Job>;
     crawlAsync(crawlRequest: CrawlRequest, actions?: Action | Action[]): Promise<JobId>;
     getJob(jobID: string): Promise<Job>;
-    getJobMarkdown(jobID: string): Promise<string>;
+    getJobMarkdown(jobID: string): Promise<JobMarkdownResponse>;
+    getJobMarkdownContent(jobID: string): Promise<string>;
     private sendRequest;
 }

package/dist/@types/model.d.ts

@@ -5,6 +5,7 @@ export interface ScrapeRequest {
     clean_selectors?: string;
     actions?: Action[];
     prompt?: string;
+    response_schema?: Record<string, any>;
     respect_robots_txt?: boolean;
     main_content_only?: boolean;
     max_age?: number;
@@ -21,6 +22,7 @@ export interface ScrapeResponse {
     page_status_code: number;
     page_title?: string;
     structured_data?: any;
+    links?: string[];
 }
 export interface ScrapeResponseError {
     success: boolean;
@@ -78,6 +80,7 @@ export interface JobItem {
     depth?: number;
     raw_content_url?: string;
     cleaned_content_url?: string;
+    error_code?: string;
     getContent(): Promise<string | null>;
 }
 export interface Action {
@@ -91,3 +94,6 @@ export interface UploadS3Action extends Action {
     bucket: string;
     endpoint?: string;
 }
+export interface JobMarkdownResponse {
+    content_url: string;
+}

package/dist/api.js

@@ -150,7 +150,7 @@ class WebcrawlerClient {
         if (job.scrape_type !== 'markdown') {
             throw new errors_1.WebcrawlerApiError('invalid_scrape_type', 'CrawlRawMarkdown requires scrape_type to be markdown', 0);
         }
-        return this.getJobMarkdown(job.id);
+        return this.getJobMarkdownContent(job.id);
     }
     async crawl(crawlRequest, actions) {
         const url = `${this.basePath}/${this.apiVersion}/crawl`;
@@ -233,6 +233,21 @@ class WebcrawlerClient {
                 'Expires': '0'
             }
         };
+        return this.sendRequest(url, requestOptions);
+    }
+    async getJobMarkdownContent(jobID) {
+        const url = `${this.basePath}/${this.apiVersion}/job/${jobID}/markdown/content`;
+        const requestOptions = {
+            'method': 'GET',
+            'headers': {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`,
+                "User-Agent": "WebcrawlerAPI-NodeJS-Client",
+                'Cache-Control': 'no-cache, no-store, must-revalidate',
+                'Pragma': 'no-cache',
+                'Expires': '0'
+            }
+        };
         let response;
         try {
             response = await fetch(url, requestOptions);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "2.0.7",
+  "version": "2.0.10",
   "description": "JS client for WebcrawlerAPI",
   "main": "./dist/api.js",
   "types": "./dist/@types/api.d.ts",