webcrawlerapi-js 1.0.8 → 1.0.9

package/README.md

@@ -16,10 +16,10 @@ Install WebcrawlerAPI js package:
 ## Request example
 
 ```javascript
-import webcrawlerapi from "webcrawlerapi-js";
+import { WebcrawlerClient } from "webcrawlerapi-js"
 
 async function main() {
-    const client = new webcrawlerapi.WebcrawlerClient(
+    const client = new WebcrawlerClient(
         "YOUR API ACCESS KEY HERE",
     )
     // sync way - promise will be resolved with the all the data
@@ -79,6 +79,7 @@ main().catch(console.error);
       status: 'new',
       title: '',
       last_error: '',
+      error_code: '',
       created_at: '2024-12-28T21:36:04.468Z',
       updated_at: '2024-12-28T21:36:04.435Z',
       cost: 0,

package/api.ts

@@ -1,4 +1,5 @@
 import {CrawlRequest, Job, JobId, ScrapeRequest, ScrapeResponse} from "./model";
+import { JobStatus } from "./constants";
 
 const BASE_PATH = "https://api.webcrawlerapi.com"
 const initialPullDelayMs = 2000
@@ -67,7 +68,7 @@ export class WebcrawlerClient {
             if (scrapeRequest.debug) {
                 console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
             }
-            if (scrapeResult.status !== 'in_progress' && scrapeResult.status !== 'new') {
+            if (scrapeResult.status !== JobStatus.IN_PROGRESS && scrapeResult.status !== JobStatus.NEW) {
                 return scrapeResult;
             }
             if (scrapeResult.recommended_pull_delay_ms > 0) {
@@ -139,12 +140,12 @@ export class WebcrawlerClient {
             await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
             const timestamp = new Date().getTime();
             const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
-            if (job.status !== 'in_progress' && job.status !== 'new') {
+            if (job.status !== JobStatus.IN_PROGRESS && job.status !== JobStatus.NEW) {
                 // Transform each job item to include getContent method
                 job.job_items = job.job_items.map(item => ({
                     ...item,
                     getContent: async function(): Promise<string | null> {
-                        if (this.status !== 'done') {
+                        if (job.status !== JobStatus.DONE || this.status !== JobStatus.DONE) {
                             return null;
                         }
 

package/constants.ts

@@ -0,0 +1,14 @@
+export const JobStatus = {
+    IN_PROGRESS: 'in_progress',
+    NEW: 'new',
+    DONE: 'done'
+} as const;
+
+export const HttpStatus = {
+    OK: 200,
+    BAD_REQUEST: 400,
+    UNAUTHORIZED: 401,
+    FORBIDDEN: 403,
+    NOT_FOUND: 404,
+    INTERNAL_SERVER_ERROR: 500
+} as const; 

package/dist/api.js

@@ -10,6 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WebcrawlerClient = void 0;
+const constants_1 = require("./constants");
 const BASE_PATH = "https://api.webcrawlerapi.com";
 const initialPullDelayMs = 2000;
 const MaxPullRetries = 100;
@@ -67,7 +68,7 @@ class WebcrawlerClient {
                 if (scrapeRequest.debug) {
                     console.log(`Scrape result: ${JSON.stringify(scrapeResult)}`);
                 }
-                if (scrapeResult.status !== 'in_progress' && scrapeResult.status !== 'new') {
+                if (scrapeResult.status !== constants_1.JobStatus.IN_PROGRESS && scrapeResult.status !== constants_1.JobStatus.NEW) {
                     return scrapeResult;
                 }
                 if (scrapeResult.recommended_pull_delay_ms > 0) {
@@ -134,11 +135,11 @@ class WebcrawlerClient {
                 yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
                 const timestamp = new Date().getTime();
                 const job = yield this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
-                if (job.status !== 'in_progress' && job.status !== 'new') {
+                if (job.status !== constants_1.JobStatus.IN_PROGRESS && job.status !== constants_1.JobStatus.NEW) {
                     // Transform each job item to include getContent method
                     job.job_items = job.job_items.map(item => (Object.assign(Object.assign({}, item), { getContent: function () {
                             return __awaiter(this, void 0, void 0, function* () {
-                                if (this.status !== 'done') {
+                                if (job.status !== constants_1.JobStatus.DONE || this.status !== constants_1.JobStatus.DONE) {
                                     return null;
                                 }
                                 let contentUrl;

package/dist/constants.d.ts

@@ -0,0 +1,13 @@
+export declare const JobStatus: {
+    readonly IN_PROGRESS: "in_progress";
+    readonly NEW: "new";
+    readonly DONE: "done";
+};
+export declare const HttpStatus: {
+    readonly OK: 200;
+    readonly BAD_REQUEST: 400;
+    readonly UNAUTHORIZED: 401;
+    readonly FORBIDDEN: 403;
+    readonly NOT_FOUND: 404;
+    readonly INTERNAL_SERVER_ERROR: 500;
+};

package/dist/constants.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpStatus = exports.JobStatus = void 0;
+exports.JobStatus = {
+    IN_PROGRESS: 'in_progress',
+    NEW: 'new',
+    DONE: 'done'
+};
+exports.HttpStatus = {
+    OK: 200,
+    BAD_REQUEST: 400,
+    UNAUTHORIZED: 401,
+    FORBIDDEN: 403,
+    NOT_FOUND: 404,
+    INTERNAL_SERVER_ERROR: 500
+};

package/package.json

@@ -1,9 +1,13 @@
 {
   "name": "webcrawlerapi-js",
-  "version": "1.0.8",
+  "version": "1.0.9",
+
   "description": "JS client for WecrawlerAPI",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
+  "scripts": {
+    "build": "tsc --outDir dist/"
+  },
   "keywords": [
     "scraping",
     "scraper",
@@ -19,8 +23,5 @@
   },
   "author": "Andrew <sdk@webcrawlerapi.com>",
   "license": "MIT",
-  "dependencies": {},
-  "scripts": {
-    "build": "tsc --outDir dist/"
-  }
-}
+  "dependencies": {}
+}