webcrawlerapi-js 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +2 -2
- package/dist/api.js +14 -10
- package/package.json +1 -1
package/dist/api.d.ts
CHANGED
|
@@ -5,8 +5,8 @@ export declare class WebcrawlerClient {
|
|
|
5
5
|
protected basePath: string;
|
|
6
6
|
constructor(apiKey: string, basePath?: string, apiVersion?: string);
|
|
7
7
|
scrapeAsync(scrapeRequest: ScrapeRequest): Promise<JobId>;
|
|
8
|
-
scrapeWithMeta(scrapeRequest: ScrapeRequest): Promise<ScrapeResponse>;
|
|
9
|
-
scrape(scrapeRequest: ScrapeRequest): Promise<
|
|
8
|
+
scrapeWithMeta(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<ScrapeResponse>;
|
|
9
|
+
scrape(scrapeRequest: ScrapeRequest, maxPollingRetries?: number): Promise<any>;
|
|
10
10
|
getScrapeResult(scrapeID: string): Promise<ScrapeResponse>;
|
|
11
11
|
crawl(crawlRequest: CrawlRequest): Promise<Job>;
|
|
12
12
|
crawlAsync(crawlRequest: CrawlRequest): Promise<JobId>;
|
package/dist/api.js
CHANGED
|
@@ -27,7 +27,8 @@ class WebcrawlerClient {
|
|
|
27
27
|
'method': 'POST',
|
|
28
28
|
'headers': {
|
|
29
29
|
'Content-Type': 'application/json',
|
|
30
|
-
'Authorization': `Bearer ${this.apiKey}
|
|
30
|
+
'Authorization': `Bearer ${this.apiKey}`,
|
|
31
|
+
"User-Agent": "WebcrawlerAPI-NodeJS-Client"
|
|
31
32
|
},
|
|
32
33
|
'body': JSON.stringify(scrapeRequest),
|
|
33
34
|
};
|
|
@@ -44,8 +45,8 @@ class WebcrawlerClient {
|
|
|
44
45
|
}
|
|
45
46
|
});
|
|
46
47
|
}
|
|
47
|
-
scrapeWithMeta(
|
|
48
|
-
return __awaiter(this,
|
|
48
|
+
scrapeWithMeta(scrapeRequest_1) {
|
|
49
|
+
return __awaiter(this, arguments, void 0, function* (scrapeRequest, maxPollingRetries = MaxPullRetries) {
|
|
49
50
|
const url = `${this.basePath}/${this.apiVersion}/scrape`;
|
|
50
51
|
const requestOptions = {
|
|
51
52
|
'method': 'POST',
|
|
@@ -60,7 +61,7 @@ class WebcrawlerClient {
|
|
|
60
61
|
throw new Error("Failed to fetch job status");
|
|
61
62
|
}
|
|
62
63
|
let delayIntervalMs = initialPullDelayMs;
|
|
63
|
-
for (let i = 0; i <
|
|
64
|
+
for (let i = 0; i < maxPollingRetries; i++) {
|
|
64
65
|
yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
|
|
65
66
|
const scrapeResult = yield this.getScrapeResult(jobIdResponse.id);
|
|
66
67
|
if (scrapeRequest.debug) {
|
|
@@ -73,12 +74,12 @@ class WebcrawlerClient {
|
|
|
73
74
|
delayIntervalMs = scrapeResult.recommended_pull_delay_ms;
|
|
74
75
|
}
|
|
75
76
|
}
|
|
76
|
-
throw new Error("Scraping took too long, please retry");
|
|
77
|
+
throw new Error("Scraping took too long, please retry or increase the number of polling retries");
|
|
77
78
|
});
|
|
78
79
|
}
|
|
79
|
-
scrape(
|
|
80
|
-
return __awaiter(this,
|
|
81
|
-
const scrapeResult = yield this.scrapeWithMeta(scrapeRequest);
|
|
80
|
+
scrape(scrapeRequest_1) {
|
|
81
|
+
return __awaiter(this, arguments, void 0, function* (scrapeRequest, maxPollingRetries = MaxPullRetries) {
|
|
82
|
+
const scrapeResult = yield this.scrapeWithMeta(scrapeRequest, maxPollingRetries);
|
|
82
83
|
return scrapeResult.structured_data;
|
|
83
84
|
});
|
|
84
85
|
}
|
|
@@ -89,6 +90,7 @@ class WebcrawlerClient {
|
|
|
89
90
|
'method': 'GET',
|
|
90
91
|
'headers': {
|
|
91
92
|
'Authorization': `Bearer ${this.apiKey}`,
|
|
93
|
+
"User-Agent": "WebcrawlerAPI-NodeJS-Client"
|
|
92
94
|
},
|
|
93
95
|
};
|
|
94
96
|
const response = yield fetch(url, requestOptions);
|
|
@@ -111,7 +113,8 @@ class WebcrawlerClient {
|
|
|
111
113
|
'method': 'POST',
|
|
112
114
|
'headers': {
|
|
113
115
|
'Content-Type': 'application/json',
|
|
114
|
-
'Authorization': `Bearer ${this.apiKey}
|
|
116
|
+
'Authorization': `Bearer ${this.apiKey}`,
|
|
117
|
+
"User-Agent": "WebcrawlerAPI-NodeJS-Client"
|
|
115
118
|
},
|
|
116
119
|
'body': JSON.stringify(crawlRequest),
|
|
117
120
|
};
|
|
@@ -153,7 +156,8 @@ class WebcrawlerClient {
|
|
|
153
156
|
'method': 'GET',
|
|
154
157
|
'headers': {
|
|
155
158
|
'Content-Type': 'application/json',
|
|
156
|
-
'Authorization': `Bearer ${this.apiKey}
|
|
159
|
+
'Authorization': `Bearer ${this.apiKey}`,
|
|
160
|
+
"User-Agent": "WebcrawlerAPI-NodeJS-Client"
|
|
157
161
|
}
|
|
158
162
|
};
|
|
159
163
|
const response = yield fetch(url, requestOptions);
|