webcrawlerapi-js 1.0.4 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/README.md +62 -37
  2. package/api.ts +12 -8
  3. package/dist/api.d.ts +2 -2
  4. package/dist/api.js +14 -11
  5. package/package.json +2 -2
package/README.md CHANGED
@@ -1,60 +1,85 @@
1
- # JS client for WebcrawlerAPI scrapers
1
+ # JS client for WebcrawlerAPI
2
2
 
3
- Official client for [WebcrawlerAPI](https://webcrawlerapi.com/) scrapers.
3
+ Official client for [WebcrawlerAPI](https://webcrawlerapi.com/).
4
4
 
5
- ## Installation
6
- Install WebcrawlerAPI js package:
7
-
8
- `npm i webcrawlerapi-js`
5
+ WebcrawlerAPI allows you to extract data from any website with just a simple API call.
9
6
 
10
7
  ## Preparation
11
8
  1. Register to [dashboard|https://dash.webcrawlerapi.com/].
12
9
  2. Get an [Access Key](https://dash.webcrawlerapi.com/access).
13
10
 
11
+ ## Installation
12
+ Install WebcrawlerAPI js package:
13
+
14
+ `npm i webcrawlerapi-js`
15
+
14
16
  ## Request example
15
17
 
16
18
  ```javascript
17
- const webcrawlerapi = require('webcrawlerapi-js');
19
+ import webcrawlerapi from "webcrawlerapi-js";
18
20
 
19
21
  async function main() {
20
22
  const client = new webcrawlerapi.WebcrawlerClient(
21
- "YOUR API KEY HERE"
23
+ "YOUR API ACCESS KEY HERE",
24
+ )
25
+ // sync way - promise will be resolved with the all the data
26
+ const syncJob = await client.crawl({
27
+ "items_limit": 10,
28
+ "url": "https://stripe.com/",
29
+ "scrape_type": "markdown"
30
+ }
22
31
  )
23
- const response = await client.scrape({
24
- input: {
25
- "url": "https://www.funda.nl/detail/koop/heerhugowaard/huis-govert-flinckplantsoen-1/89968455/"
26
- },
27
- crawler_id: "webcrawler/funda",
28
- })
29
- console.log(response)
32
+ console.log(syncJob);
33
+
34
+ // or async - get the job id and then poll the job status and get the data
35
+
36
+ const jobWithId = await client.crawlAsync({
37
+ "items_limit": 10,
38
+ "url": "https://stripe.com/",
39
+ "scrape_type": "markdown"
40
+ }
41
+ )
42
+ // wait for job to complete
43
+ const jobId = jobWithId.id;
44
+ let asyncJob = await client.getJob(jobId);
45
+
46
+ console.log(asyncJob);
30
47
  }
31
48
 
32
49
  main().catch(console.error);
33
50
  ```
34
51
 
35
52
  ## Response example
36
- ```json
53
+ ```javascript
37
54
  {
38
- city: 'Heerhugowaard',
39
- price: 325000,
40
- images: [ 'https://cloud.funda.nl/valentina_media/191/215/183_2160.jpg' ],
41
- status: 'inonderhandeling',
42
- videos: [],
43
- address: 'Govert Flinckplantsoen 1',
44
- country: 'Nederland',
45
- province: 'Noord-Holland',
46
- plot_area: '183 m²',
47
- post_code: '1701NH',
48
- description: 'De woning is met liefde en zorg 53 jaar bewoond door...',
49
- living_area: 127,
50
- house_number: 1,
51
- energie_label: 'd',
52
- property_type: 'woonhuis',
53
- publication_date: '2024-05-28T00:00:00',
54
- number_of_bedrooms: 4,
55
- coordinates_latitude: 52.67685,
56
- year_of_construction: 1971,
57
- coordinates_longitude: 4.8560443,
58
- house_number_extension: ''
55
+ id: '49c4942b-b7d9-4d62-94b5-b54a3016ac51',
56
+ org_id: 'clxsnorta00075wuuqxgzzvxm',
57
+ url: 'https://stripe.com/',
58
+ scrape_type: 'markdown',
59
+ whitelist_regexp: '',
60
+ blacklist_regexp: '',
61
+ allow_subdomains: false,
62
+ items_limit: 10,
63
+ created_at: '2024-12-28T21:36:04.417Z',
64
+ finished_at: null,
65
+ updated_at: '2024-12-28T21:36:04.383Z',
66
+ webhook_url: '',
67
+ status: 'in_progress',
68
+ job_items: [
69
+ {
70
+ id: 'f26cefe1-09d1-4d4c-8b74-b65e075e230d',
71
+ job_id: '49c4942a-b7d9-4d62-94b5-b54a3016ac51',
72
+ original_url: 'https://stripe.com/',
73
+ page_status_code: 0,
74
+ status: 'new',
75
+ title: '',
76
+ last_error: '',
77
+ created_at: '2024-12-28T21:36:04.468Z',
78
+ updated_at: '2024-12-28T21:36:04.435Z',
79
+ cost: 0,
80
+ referred_url: ''
81
+ }
82
+ ],
83
+ recommended_pull_delay_ms: 5000
59
84
  }
60
85
  ```
package/api.ts CHANGED
@@ -22,7 +22,8 @@ export class WebcrawlerClient {
22
22
  'method': 'POST',
23
23
  'headers': {
24
24
  'Content-Type': 'application/json',
25
- 'Authorization': `Bearer ${this.apiKey}`
25
+ 'Authorization': `Bearer ${this.apiKey}`,
26
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client"
26
27
  },
27
28
  'body': JSON.stringify(scrapeRequest),
28
29
  };
@@ -41,7 +42,7 @@ export class WebcrawlerClient {
41
42
  }
42
43
  }
43
44
 
44
- public async scrapeWithMeta(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse> {
45
+ public async scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<ScrapeResponse> {
45
46
  const url = `${this.basePath}/${this.apiVersion}/scrape`;
46
47
 
47
48
  const requestOptions = {
@@ -60,7 +61,7 @@ export class WebcrawlerClient {
60
61
  }
61
62
 
62
63
  let delayIntervalMs = initialPullDelayMs;
63
- for (let i = 0; i < MaxPullRetries; i++) {
64
+ for (let i = 0; i < maxPollingRetries; i++) {
64
65
  await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
65
66
  const scrapeResult = await this.getScrapeResult(jobIdResponse.id);
66
67
  if (scrapeRequest.debug) {
@@ -73,11 +74,11 @@ export class WebcrawlerClient {
73
74
  delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
74
75
  }
75
76
  }
76
- throw new Error("Scraping took too long, please retry");
77
+ throw new Error("Scraping took too long, please retry or increase the number of polling retries");
77
78
  }
78
79
 
79
- public async scrape(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse> {
80
- const scrapeResult = await this.scrapeWithMeta(scrapeRequest);
80
+ public async scrape(scrapeRequest: ScrapeRequest, maxPollingRetries: number = MaxPullRetries): Promise<any> {
81
+ const scrapeResult = await this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
81
82
  return scrapeResult.structured_data;
82
83
  }
83
84
 
@@ -87,6 +88,7 @@ export class WebcrawlerClient {
87
88
  'method': 'GET',
88
89
  'headers': {
89
90
  'Authorization': `Bearer ${this.apiKey}`,
91
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client"
90
92
  },
91
93
  };
92
94
  const response = await fetch(url, requestOptions);
@@ -113,7 +115,8 @@ export class WebcrawlerClient {
113
115
  'method': 'POST',
114
116
  'headers': {
115
117
  'Content-Type': 'application/json',
116
- 'Authorization': `Bearer ${this.apiKey}`
118
+ 'Authorization': `Bearer ${this.apiKey}`,
119
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client"
117
120
  },
118
121
  'body': JSON.stringify(crawlRequest),
119
122
  };
@@ -158,7 +161,8 @@ export class WebcrawlerClient {
158
161
  'method': 'GET',
159
162
  'headers': {
160
163
  'Content-Type': 'application/json',
161
- 'Authorization': `Bearer ${this.apiKey}`
164
+ 'Authorization': `Bearer ${this.apiKey}`,
165
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client"
162
166
  }
163
167
  }
164
168
  const response = await fetch(url, requestOptions);
package/dist/api.d.ts CHANGED
@@ -5,8 +5,8 @@ export declare class WebcrawlerClient {
5
5
  protected basePath: string;
6
6
  constructor(apiKey: string, basePath?: string, apiVersion?: string);
7
7
  scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
8
- scrapeWithMeta(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse>;
9
- scrape(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse>;
8
+ scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
9
+ scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
10
10
  getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
11
11
  crawl(crawlRequest: CrawlRequest): Promise<Job>;
12
12
  crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
package/dist/api.js CHANGED
@@ -27,7 +27,8 @@ class WebcrawlerClient {
27
27
  'method': 'POST',
28
28
  'headers': {
29
29
  'Content-Type': 'application/json',
30
- 'Authorization': `Bearer ${this.apiKey}`
30
+ 'Authorization': `Bearer ${this.apiKey}`,
31
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client"
31
32
  },
32
33
  'body': JSON.stringify(scrapeRequest),
33
34
  };
@@ -44,8 +45,8 @@ class WebcrawlerClient {
44
45
  }
45
46
  });
46
47
  }
47
- scrapeWithMeta(scrapeRequest) {
48
- return __awaiter(this, void 0, void 0, function* () {
48
+ scrapeWithMeta(scrapeRequest_1) {
49
+ return __awaiter(this, arguments, void 0, function* (scrapeRequest, maxPollingRetries = MaxPullRetries) {
49
50
  const url = `${this.basePath}/${this.apiVersion}/scrape`;
50
51
  const requestOptions = {
51
52
  'method': 'POST',
@@ -60,7 +61,7 @@ class WebcrawlerClient {
60
61
  throw new Error("Failed to fetch job status");
61
62
  }
62
63
  let delayIntervalMs = initialPullDelayMs;
63
- for (let i = 0; i < MaxPullRetries; i++) {
64
+ for (let i = 0; i < maxPollingRetries; i++) {
64
65
  yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
65
66
  const scrapeResult = yield this.getScrapeResult(jobIdResponse.id);
66
67
  if (scrapeRequest.debug) {
@@ -73,12 +74,12 @@ class WebcrawlerClient {
73
74
  delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
74
75
  }
75
76
  }
76
- throw new Error("Scraping took too long, please retry");
77
+ throw new Error("Scraping took too long, please retry or increase the number of polling retries");
77
78
  });
78
79
  }
79
- scrape(scrapeRequest) {
80
- return __awaiter(this, void 0, void 0, function* () {
81
- const scrapeResult = yield this.scrapeWithMeta(scrapeRequest);
80
+ scrape(scrapeRequest_1) {
81
+ return __awaiter(this, arguments, void 0, function* (scrapeRequest, maxPollingRetries = MaxPullRetries) {
82
+ const scrapeResult = yield this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
82
83
  return scrapeResult.structured_data;
83
84
  });
84
85
  }
@@ -89,6 +90,7 @@ class WebcrawlerClient {
89
90
  'method': 'GET',
90
91
  'headers': {
91
92
  'Authorization': `Bearer ${this.apiKey}`,
93
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client"
92
94
  },
93
95
  };
94
96
  const response = yield fetch(url, requestOptions);
@@ -111,7 +113,8 @@ class WebcrawlerClient {
111
113
  'method': 'POST',
112
114
  'headers': {
113
115
  'Content-Type': 'application/json',
114
- 'Authorization': `Bearer ${this.apiKey}`
116
+ 'Authorization': `Bearer ${this.apiKey}`,
117
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client"
115
118
  },
116
119
  'body': JSON.stringify(crawlRequest),
117
120
  };
@@ -149,12 +152,12 @@ class WebcrawlerClient {
149
152
  getJob(jobID) {
150
153
  return __awaiter(this, void 0, void 0, function* () {
151
154
  const url = `${this.basePath}/${this.apiVersion}/job/${jobID}`;
152
- console.log(url);
153
155
  const requestOptions = {
154
156
  'method': 'GET',
155
157
  'headers': {
156
158
  'Content-Type': 'application/json',
157
- 'Authorization': `Bearer ${this.apiKey}`
159
+ 'Authorization': `Bearer ${this.apiKey}`,
160
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client"
158
161
  }
159
162
  };
160
163
  const response = yield fetch(url, requestOptions);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "webcrawlerapi-js",
3
- "version": "1.0.4",
3
+ "version": "1.0.6",
4
4
  "description": "JS client for WecrawlerAPI",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",
@@ -17,7 +17,7 @@
17
17
  "@types/node": "^22.4.0",
18
18
  "typescript": "^5.5.4"
19
19
  },
20
- "author": "Andrew <support@webcrawlerapi.com>",
20
+ "author": "Andrew <sdk@webcrawlerapi.com>",
21
21
  "license": "MIT",
22
22
  "dependencies": {},
23
23
  "scripts": {