webcrawlerapi-js 1.0.7 → 1.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/README.md +5 -0
  2. package/api.ts +55 -4
  3. package/dist/api.js +49 -4
  4. package/dist/model.d.ts +3 -0
  5. package/model.ts +3 -0
  6. package/package.json +1 -1
package/README.md CHANGED
@@ -29,6 +29,11 @@ async function main() {
29
29
  "scrape_type": "markdown"
30
30
  }
31
31
  )
32
+ for (const item of syncJob.job_items) {
33
+ item.getContent().then((content) => {
34
+ console.log(content.slice(0, 100));
35
+ })
36
+ }
32
37
  console.log(syncJob);
33
38
 
34
39
  // or async - get the job id and then poll the job status and get the data
package/api.ts CHANGED
@@ -87,8 +87,12 @@ export class WebcrawlerClient {
87
87
  const requestOptions = {
88
88
  'method': 'GET',
89
89
  'headers': {
90
+ 'Content-Type': 'application/json',
90
91
  'Authorization': `Bearer ${this.apiKey}`,
91
- "User-Agent": "WebcrawlerAPI-NodeJS-Client"
92
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client",
93
+ 'Cache-Control': 'no-cache, no-store, must-revalidate',
94
+ 'Pragma': 'no-cache',
95
+ 'Expires': '0'
92
96
  },
93
97
  };
94
98
  const response = await fetch(url, requestOptions);
@@ -116,7 +120,10 @@ export class WebcrawlerClient {
116
120
  'headers': {
117
121
  'Content-Type': 'application/json',
118
122
  'Authorization': `Bearer ${this.apiKey}`,
119
- "User-Agent": "WebcrawlerAPI-NodeJS-Client"
123
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client",
124
+ 'Cache-Control': 'no-cache, no-store, must-revalidate',
125
+ 'Pragma': 'no-cache',
126
+ 'Expires': '0'
120
127
  },
121
128
  'body': JSON.stringify(crawlRequest),
122
129
  };
@@ -130,14 +137,55 @@ export class WebcrawlerClient {
130
137
  let delayIntervalMs = initialPullDelayMs;
131
138
  for (let i = 0; i < MaxPullRetries; i++) {
132
139
  await new Promise(resolve => setTimeout(resolve, delayIntervalMs));
133
- const job = await this.getJob(jobIdResponse.id);
140
+ const timestamp = new Date().getTime();
141
+ const job = await this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
134
142
  if (job.status !== 'in_progress' && job.status !== 'new') {
143
+ // Transform each job item to include getContent method
144
+ job.job_items = job.job_items.map(item => ({
145
+ ...item,
146
+ getContent: async function(): Promise<string | null> {
147
+ if (this.status !== 'done') {
148
+ return null;
149
+ }
150
+
151
+ let contentUrl: string | undefined;
152
+ switch (job.scrape_type) {
153
+ case 'html':
154
+ contentUrl = this.raw_content_url;
155
+ break;
156
+ case 'cleaned':
157
+ contentUrl = this.cleaned_content_url;
158
+ break;
159
+ case 'markdown':
160
+ contentUrl = this.markdown_content_url;
161
+ break;
162
+ }
163
+
164
+ if (!contentUrl) {
165
+ return null;
166
+ }
167
+
168
+ const response = await fetch(contentUrl, {
169
+ headers: {
170
+ 'Accept-Encoding': 'gzip, deflate, br',
171
+ 'Accept': '*/*'
172
+ }
173
+ });
174
+
175
+ if (!response.ok) {
176
+ throw new Error(`Failed to fetch content: ${response.statusText}`);
177
+ }
178
+
179
+ return await response.text();
180
+ }
181
+ }));
135
182
  return job;
136
183
  }
137
184
  if (job.recommended_pull_delay_ms > 0) {
138
185
  delayIntervalMs = job.recommended_pull_delay_ms;
139
186
  }
140
187
  }
188
+ throw new Error("Crawling took too long, please retry or increase the number of polling retries");
141
189
  }
142
190
 
143
191
  public async crawlAsync(crawlRequest: CrawlRequest): Promise<JobId> {
@@ -162,7 +210,10 @@ export class WebcrawlerClient {
162
210
  'headers': {
163
211
  'Content-Type': 'application/json',
164
212
  'Authorization': `Bearer ${this.apiKey}`,
165
- "User-Agent": "WebcrawlerAPI-NodeJS-Client"
213
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client",
214
+ 'Cache-Control': 'no-cache, no-store, must-revalidate',
215
+ 'Pragma': 'no-cache',
216
+ 'Expires': '0'
166
217
  }
167
218
  }
168
219
  const response = await fetch(url, requestOptions);
package/dist/api.js CHANGED
@@ -89,8 +89,12 @@ class WebcrawlerClient {
89
89
  const requestOptions = {
90
90
  'method': 'GET',
91
91
  'headers': {
92
+ 'Content-Type': 'application/json',
92
93
  'Authorization': `Bearer ${this.apiKey}`,
93
- "User-Agent": "WebcrawlerAPI-NodeJS-Client"
94
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client",
95
+ 'Cache-Control': 'no-cache, no-store, must-revalidate',
96
+ 'Pragma': 'no-cache',
97
+ 'Expires': '0'
94
98
  },
95
99
  };
96
100
  const response = yield fetch(url, requestOptions);
@@ -114,7 +118,10 @@ class WebcrawlerClient {
114
118
  'headers': {
115
119
  'Content-Type': 'application/json',
116
120
  'Authorization': `Bearer ${this.apiKey}`,
117
- "User-Agent": "WebcrawlerAPI-NodeJS-Client"
121
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client",
122
+ 'Cache-Control': 'no-cache, no-store, must-revalidate',
123
+ 'Pragma': 'no-cache',
124
+ 'Expires': '0'
118
125
  },
119
126
  'body': JSON.stringify(crawlRequest),
120
127
  };
@@ -125,14 +132,49 @@ class WebcrawlerClient {
125
132
  let delayIntervalMs = initialPullDelayMs;
126
133
  for (let i = 0; i < MaxPullRetries; i++) {
127
134
  yield new Promise(resolve => setTimeout(resolve, delayIntervalMs));
128
- const job = yield this.getJob(jobIdResponse.id);
135
+ const timestamp = new Date().getTime();
136
+ const job = yield this.getJob(`${jobIdResponse.id}?t=${timestamp}`);
129
137
  if (job.status !== 'in_progress' && job.status !== 'new') {
138
+ // Transform each job item to include getContent method
139
+ job.job_items = job.job_items.map(item => (Object.assign(Object.assign({}, item), { getContent: function () {
140
+ return __awaiter(this, void 0, void 0, function* () {
141
+ if (this.status !== 'done') {
142
+ return null;
143
+ }
144
+ let contentUrl;
145
+ switch (job.scrape_type) {
146
+ case 'html':
147
+ contentUrl = this.raw_content_url;
148
+ break;
149
+ case 'cleaned':
150
+ contentUrl = this.cleaned_content_url;
151
+ break;
152
+ case 'markdown':
153
+ contentUrl = this.markdown_content_url;
154
+ break;
155
+ }
156
+ if (!contentUrl) {
157
+ return null;
158
+ }
159
+ const response = yield fetch(contentUrl, {
160
+ headers: {
161
+ 'Accept-Encoding': 'gzip, deflate, br',
162
+ 'Accept': '*/*'
163
+ }
164
+ });
165
+ if (!response.ok) {
166
+ throw new Error(`Failed to fetch content: ${response.statusText}`);
167
+ }
168
+ return yield response.text();
169
+ });
170
+ } })));
130
171
  return job;
131
172
  }
132
173
  if (job.recommended_pull_delay_ms > 0) {
133
174
  delayIntervalMs = job.recommended_pull_delay_ms;
134
175
  }
135
176
  }
177
+ throw new Error("Crawling took too long, please retry or increase the number of polling retries");
136
178
  });
137
179
  }
138
180
  crawlAsync(crawlRequest) {
@@ -157,7 +199,10 @@ class WebcrawlerClient {
157
199
  'headers': {
158
200
  'Content-Type': 'application/json',
159
201
  'Authorization': `Bearer ${this.apiKey}`,
160
- "User-Agent": "WebcrawlerAPI-NodeJS-Client"
202
+ "User-Agent": "WebcrawlerAPI-NodeJS-Client",
203
+ 'Cache-Control': 'no-cache, no-store, must-revalidate',
204
+ 'Pragma': 'no-cache',
205
+ 'Expires': '0'
161
206
  }
162
207
  };
163
208
  const response = yield fetch(url, requestOptions);
package/dist/model.d.ts CHANGED
@@ -56,4 +56,7 @@ export interface JobItem {
56
56
  updated_at: string;
57
57
  cost: number;
58
58
  referred_url: string;
59
+ raw_content_url?: string;
60
+ cleaned_content_url?: string;
61
+ getContent(): Promise<string | null>;
59
62
  }
package/model.ts CHANGED
@@ -61,4 +61,7 @@ export interface JobItem {
61
61
  updated_at: string;
62
62
  cost: number;
63
63
  referred_url: string;
64
+ raw_content_url?: string;
65
+ cleaned_content_url?: string;
66
+ getContent(): Promise<string | null>;
64
67
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "webcrawlerapi-js",
3
- "version": "1.0.7",
3
+ "version": "1.0.8",
4
4
  "description": "JS client for WecrawlerAPI",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",