scrapfly-sdk 0.6.2 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/esm/client.d.ts +17 -14
- package/esm/client.d.ts.map +1 -1
- package/esm/client.js +63 -24
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts.map +1 -1
- package/esm/polyfill.d.ts +5 -0
- package/esm/polyfill.d.ts.map +1 -0
- package/esm/polyfill.js +20 -0
- package/esm/result.d.ts +0 -1
- package/esm/result.d.ts.map +1 -1
- package/esm/result.js +9 -5
- package/esm/utils.d.ts +9 -2
- package/esm/utils.d.ts.map +1 -1
- package/esm/utils.js +20 -2
- package/package.json +1 -1
- package/script/client.d.ts +17 -14
- package/script/client.d.ts.map +1 -1
- package/script/client.js +63 -24
- package/script/deps/jsr.io/@std/path/1.0.1/_common/assert_path.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/basename.js +3 -4
- package/script/deps/jsr.io/@std/path/1.0.1/_common/common.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/format.js +2 -3
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/glob_to_reg_exp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize_string.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/strip_trailing_separators.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/basename.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/common.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/extname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/format.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/glob_to_regexp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/is_absolute.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/is_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/join.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/join_globs.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/normalize_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/parse.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/_util.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/basename.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/extname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/format.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/glob_to_regexp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/is_absolute.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join_globs.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/parse.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/resolve.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_namespaced_path.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/resolve.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/to_namespaced_path.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/_util.js +3 -4
- package/script/deps/jsr.io/@std/path/1.0.1/windows/basename.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/extname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/format.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/glob_to_regexp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/is_absolute.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join_globs.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/parse.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/resolve.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_namespaced_path.js +1 -2
- package/script/polyfill.d.ts +5 -0
- package/script/polyfill.d.ts.map +1 -0
- package/script/polyfill.js +48 -0
- package/script/result.d.ts +0 -1
- package/script/result.d.ts.map +1 -1
- package/script/result.js +9 -5
- package/script/utils.d.ts +9 -2
- package/script/utils.d.ts.map +1 -1
- package/script/utils.js +23 -5
package/README.md
CHANGED
|
@@ -24,8 +24,10 @@ The SDK is distributed through:
|
|
|
24
24
|
3. Start scraping: 🚀
|
|
25
25
|
|
|
26
26
|
```javascript
|
|
27
|
-
// node
|
|
27
|
+
// node
|
|
28
28
|
import { ScrapflyClient, ScrapeConfig } from 'scrapfly-sdk';
|
|
29
|
+
// bun
|
|
30
|
+
import { ScrapflyClient, ScrapeConfig} from '@scrapfly/scrapfly-sdk';
|
|
29
31
|
// deno:
|
|
30
32
|
import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk';
|
|
31
33
|
|
|
@@ -97,7 +99,7 @@ $ deno lint
|
|
|
97
99
|
# publish JSR:
|
|
98
100
|
$ deno publish
|
|
99
101
|
# build NPM package:
|
|
100
|
-
$ deno build-npm
|
|
102
|
+
$ deno task build-npm
|
|
101
103
|
# publish NPM:
|
|
102
104
|
$ cd npm && npm publish
|
|
103
105
|
```
|
package/esm/client.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
import { fetchRetry } from './utils.js';
|
|
3
2
|
import * as errors from './errors.js';
|
|
4
3
|
import type { ScrapeConfig } from './scrapeconfig.js';
|
|
@@ -17,6 +16,10 @@ export declare class ScrapflyClient {
|
|
|
17
16
|
* Raise appropriate error for given response and scrape result
|
|
18
17
|
*/
|
|
19
18
|
errResult(response: Response, result: ScrapeResult): errors.ScrapflyError;
|
|
19
|
+
/**
|
|
20
|
+
* Handle clob and blob large objects
|
|
21
|
+
*/
|
|
22
|
+
handleLargeObjects(result: any, format: 'clob' | 'blob'): Promise<ScrapeResult>;
|
|
20
23
|
/**
|
|
21
24
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
22
25
|
*/
|
|
@@ -30,21 +33,21 @@ export declare class ScrapflyClient {
|
|
|
30
33
|
*/
|
|
31
34
|
scrape(config: ScrapeConfig): Promise<ScrapeResult>;
|
|
32
35
|
/**
|
|
33
|
-
|
|
34
|
-
|
|
36
|
+
Concurrently scrape multiple configs
|
|
37
|
+
This is a async generator call it like this:
|
|
35
38
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
39
|
+
const results = [];
|
|
40
|
+
const errors = [];
|
|
41
|
+
for await (const resultOrError of client.concurrentScrape(configs)) {
|
|
42
|
+
if (resultOrError instanceof Error) {
|
|
43
|
+
errors.push(resultOrError);
|
|
44
|
+
} else {
|
|
45
|
+
results.push(resultOrError);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
45
48
|
|
|
46
|
-
|
|
47
|
-
|
|
49
|
+
@param concurrencyLimit: if not set it will be taken from your account info
|
|
50
|
+
*/
|
|
48
51
|
concurrentScrape(configs: ScrapeConfig[], concurrencyLimit?: number): AsyncGenerator<ScrapeResult | Error | undefined, void, undefined>;
|
|
49
52
|
/**
|
|
50
53
|
* Save screenshot response to a file
|
package/esm/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACG,kBAAkB,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAgCrF;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAkDzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
|
package/esm/client.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import * as dntShim from "./_dnt.shims.js";
|
|
2
1
|
import { path } from './deps.js';
|
|
2
|
+
import { mkdir, writeFile } from './polyfill.js';
|
|
3
3
|
import { fetchRetry } from './utils.js';
|
|
4
4
|
import * as errors from './errors.js';
|
|
5
5
|
import { ExtractionResult, ScrapeResult, ScreenshotResult } from './result.js';
|
|
@@ -105,6 +105,39 @@ export class ScrapflyClient {
|
|
|
105
105
|
}
|
|
106
106
|
return new errors.ScrapflyError(message, args);
|
|
107
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* Handle clob and blob large objects
|
|
110
|
+
*/
|
|
111
|
+
async handleLargeObjects(result, format) {
|
|
112
|
+
let response;
|
|
113
|
+
try {
|
|
114
|
+
const url = new URL(result.content);
|
|
115
|
+
const params = { key: this.key };
|
|
116
|
+
url.search = new URLSearchParams(params).toString();
|
|
117
|
+
response = await this.fetch({
|
|
118
|
+
url: url.toString(),
|
|
119
|
+
method: 'GET',
|
|
120
|
+
headers: {
|
|
121
|
+
'user-agent': this.ua,
|
|
122
|
+
'accept-encoding': 'gzip, deflate, br',
|
|
123
|
+
accept: 'application/json',
|
|
124
|
+
},
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
catch (e) {
|
|
128
|
+
log.error('error', e);
|
|
129
|
+
throw e;
|
|
130
|
+
}
|
|
131
|
+
const content = await response.text();
|
|
132
|
+
result.content = content;
|
|
133
|
+
if (format === 'clob') {
|
|
134
|
+
result.format = 'text';
|
|
135
|
+
}
|
|
136
|
+
if (format === 'blob') {
|
|
137
|
+
result.format = 'binary';
|
|
138
|
+
}
|
|
139
|
+
return result;
|
|
140
|
+
}
|
|
108
141
|
/**
|
|
109
142
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
110
143
|
*/
|
|
@@ -127,14 +160,15 @@ export class ScrapflyClient {
|
|
|
127
160
|
const url = new URL(this.HOST + '/account');
|
|
128
161
|
const params = { key: this.key };
|
|
129
162
|
url.search = new URLSearchParams(params).toString();
|
|
130
|
-
response = await this.fetch(
|
|
163
|
+
response = await this.fetch({
|
|
164
|
+
url: url.toString(),
|
|
131
165
|
method: 'GET',
|
|
132
166
|
headers: {
|
|
133
167
|
'user-agent': this.ua,
|
|
134
168
|
'accept-encoding': 'gzip, deflate, br',
|
|
135
169
|
accept: 'application/json',
|
|
136
170
|
},
|
|
137
|
-
})
|
|
171
|
+
});
|
|
138
172
|
}
|
|
139
173
|
catch (e) {
|
|
140
174
|
log.error('error', e);
|
|
@@ -159,7 +193,8 @@ export class ScrapflyClient {
|
|
|
159
193
|
const url = new URL(this.HOST + '/scrape');
|
|
160
194
|
const params = config.toApiParams({ key: this.key });
|
|
161
195
|
url.search = new URLSearchParams(params).toString();
|
|
162
|
-
response = await this.fetch(
|
|
196
|
+
response = await this.fetch({
|
|
197
|
+
url: url.toString(),
|
|
163
198
|
method: config.method,
|
|
164
199
|
headers: {
|
|
165
200
|
'user-agent': this.ua,
|
|
@@ -170,7 +205,7 @@ export class ScrapflyClient {
|
|
|
170
205
|
accept: 'application/json',
|
|
171
206
|
},
|
|
172
207
|
body: config.body,
|
|
173
|
-
})
|
|
208
|
+
});
|
|
174
209
|
}
|
|
175
210
|
catch (e) {
|
|
176
211
|
log.error('error', e);
|
|
@@ -184,6 +219,10 @@ export class ScrapflyClient {
|
|
|
184
219
|
}
|
|
185
220
|
throw new errors.ApiHttpClientError(JSON.stringify(data));
|
|
186
221
|
}
|
|
222
|
+
const content_format = data.result.format;
|
|
223
|
+
if (content_format === 'clob' || content_format === 'blob') {
|
|
224
|
+
data.result = await this.handleLargeObjects(data.result, content_format);
|
|
225
|
+
}
|
|
187
226
|
const result = this.handleResponse(response, new ScrapeResult({
|
|
188
227
|
config: data.config,
|
|
189
228
|
context: data.context,
|
|
@@ -193,21 +232,21 @@ export class ScrapflyClient {
|
|
|
193
232
|
return result;
|
|
194
233
|
}
|
|
195
234
|
/**
|
|
196
|
-
|
|
197
|
-
|
|
235
|
+
Concurrently scrape multiple configs
|
|
236
|
+
This is a async generator call it like this:
|
|
198
237
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
238
|
+
const results = [];
|
|
239
|
+
const errors = [];
|
|
240
|
+
for await (const resultOrError of client.concurrentScrape(configs)) {
|
|
241
|
+
if (resultOrError instanceof Error) {
|
|
242
|
+
errors.push(resultOrError);
|
|
243
|
+
} else {
|
|
244
|
+
results.push(resultOrError);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
208
247
|
|
|
209
|
-
|
|
210
|
-
|
|
248
|
+
@param concurrencyLimit: if not set it will be taken from your account info
|
|
249
|
+
*/
|
|
211
250
|
async *concurrentScrape(configs, concurrencyLimit) {
|
|
212
251
|
if (concurrencyLimit === undefined) {
|
|
213
252
|
const account = await this.account();
|
|
@@ -250,7 +289,7 @@ export class ScrapflyClient {
|
|
|
250
289
|
const extension_name = result.metadata.extension_name;
|
|
251
290
|
let file_path;
|
|
252
291
|
if (savePath) {
|
|
253
|
-
|
|
292
|
+
await mkdir(savePath, { recursive: true });
|
|
254
293
|
file_path = path.join(savePath, `${name}.${extension_name}`);
|
|
255
294
|
}
|
|
256
295
|
else {
|
|
@@ -258,7 +297,7 @@ export class ScrapflyClient {
|
|
|
258
297
|
}
|
|
259
298
|
const content = new Uint8Array(result.image);
|
|
260
299
|
// Use Deno's write file method
|
|
261
|
-
await
|
|
300
|
+
await writeFile(file_path, content);
|
|
262
301
|
}
|
|
263
302
|
/**
|
|
264
303
|
* Turn scrapfly screenshot API response to ScreenshotResult or raise one of ScrapflyError
|
|
@@ -289,7 +328,8 @@ export class ScrapflyClient {
|
|
|
289
328
|
const url = new URL(this.HOST + '/screenshot');
|
|
290
329
|
const params = config.toApiParams({ key: this.key });
|
|
291
330
|
url.search = new URLSearchParams(params).toString();
|
|
292
|
-
|
|
331
|
+
response = await this.fetch({
|
|
332
|
+
url: url.toString(),
|
|
293
333
|
method: 'GET',
|
|
294
334
|
headers: {
|
|
295
335
|
'user-agent': this.ua,
|
|
@@ -297,7 +337,6 @@ export class ScrapflyClient {
|
|
|
297
337
|
accept: 'application/json',
|
|
298
338
|
},
|
|
299
339
|
});
|
|
300
|
-
response = await this.fetch(req);
|
|
301
340
|
}
|
|
302
341
|
catch (e) {
|
|
303
342
|
log.error('error', e);
|
|
@@ -345,12 +384,12 @@ export class ScrapflyClient {
|
|
|
345
384
|
if (config.document_compression_format && config.document_compression_format) {
|
|
346
385
|
headers['content-encoding'] = config.document_compression_format;
|
|
347
386
|
}
|
|
348
|
-
|
|
387
|
+
response = await this.fetch({
|
|
388
|
+
url: url.toString(),
|
|
349
389
|
method: 'POST',
|
|
350
390
|
headers: headers,
|
|
351
391
|
body: config.body,
|
|
352
392
|
});
|
|
353
|
-
response = await this.fetch(req);
|
|
354
393
|
}
|
|
355
394
|
catch (e) {
|
|
356
395
|
log.error('error', e);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"AAGA,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,OAM1C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/from_file_url.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,GAAG,MAAM,CAErD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,MAAM,CAKrD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.ts"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAU3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/to_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/to_file_url.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAE3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,MAAM,CAYrD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.ts"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAgB3C"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import * as dntShim from "./_dnt.shims.js";
|
|
2
|
+
export declare const isDeno: boolean;
|
|
3
|
+
export declare function mkdir(path: string | URL, options: dntShim.Deno.MkdirOptions): Promise<void>;
|
|
4
|
+
export declare function writeFile(path: string | URL, data: Uint8Array, options?: dntShim.Deno.WriteFileOptions): Promise<void>;
|
|
5
|
+
//# sourceMappingURL=polyfill.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"polyfill.d.ts","sourceRoot":"","sources":["../src/polyfill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,iBAAiB,CAAC;AAC3C,eAAO,MAAM,MAAM,SAAsC,CAAC;AAE1D,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjG;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,GAAG,GAAG,EAClB,IAAI,EAAE,UAAU,EAChB,OAAO,GAAE,OAAO,CAAC,IAAI,CAAC,gBAAqB,GAC1C,OAAO,CAAC,IAAI,CAAC,CAOf"}
|
package/esm/polyfill.js
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import * as dntShim from "./_dnt.shims.js";
|
|
2
|
+
export const isDeno = typeof dntShim.Deno !== 'undefined';
|
|
3
|
+
export async function mkdir(path, options) {
|
|
4
|
+
if (isDeno) {
|
|
5
|
+
await dntShim.Deno.mkdir(path, options);
|
|
6
|
+
}
|
|
7
|
+
else {
|
|
8
|
+
// @ts-ignore: type for Bun
|
|
9
|
+
await Bun.mkdir(path.toString(), options);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
export async function writeFile(path, data, options = {}) {
|
|
13
|
+
if (isDeno) {
|
|
14
|
+
await dntShim.Deno.writeFile(path, data, options);
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
// @ts-ignore: type for Bun
|
|
18
|
+
await Bun.write(path.toString(), data, options);
|
|
19
|
+
}
|
|
20
|
+
}
|
package/esm/result.d.ts
CHANGED
package/esm/result.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;IACf,uBAAuB,EAAE,OAAO,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACrB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE;QACJ,OAAO,EAAE,KAAK,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACtE,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpB,EAAE,EAAE;QACF,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,GAAG,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,YAAY,EAAE;QACZ,4BAA4B,CAAC,EAAE,MAAM,CAAC;QACtC,WAAW,CAAC,EAAE;YACZ,QAAQ,EAAE,MAAM,CAAC;YACjB,QAAQ,EAAE,MAAM,CAAC;YACjB,QAAQ,EAAE,GAAG,CAAC;YACd,KAAK,EAAE,KAAK,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC;gBACf,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACpB,QAAQ,EAAE,MAAM,CAAC;gBACjB,QAAQ,EAAE,OAAO,CAAC;gBAClB,MAAM,CAAC,EAAE,MAAM,CAAC;gBAChB,OAAO,EAAE,OAAO,CAAC;aAClB,CAAC,CAAC;SACJ,CAAC;QACF,kBAAkB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,oBAAoB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnC,UAAU,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACxB,QAAQ,CAAC,EAAE,KAAK,CAAC;YACf,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YACrB,MAAM,EAAE,MAAM,CAAC;YACf,IAAI,EAAE,MAAM,CAAC;YACb,GAAG,EAAE,MAAM,CAAC;YACZ,QAAQ,EAAE;gBACR,IAAI,EAAE,MAAM,CAAC;gBACb,QAAQ,EAAE,MAAM,CAAC;gBACjB,MAAM,EAAE,MAAM,CAAC;gBACf,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM,EAAE,MAAM,CAAC;aAChB,CAAC;SACH,CAAC,CAAC;KACJ,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,OAAO,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;IACH,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE;YACH,WAAW,EAAE,MAAM,CAAC;YACpB,QAAQ,EAAE,MAAM,CAAC;YACjB,IAAI,EAAE,MAAM,CAAC;YACb,MAAM,EAAE,MAAM,CAAC;YACf,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;SACtB,CAAC;QACF,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,WAAW,EAAE,GAAG,CAAC;QACf,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE;QACJ,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;KACxB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,qBAAa,YAAY;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,WAAW,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,SAAS,CAAiC;gBAEtC,IAAI,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,OAAO,EAAE,WAAW,CAAC;QAAC,MAAM,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAQhG,IAAI,QAAQ,IAAI,OAAO,CAAC,UAAU,CAWjC;CACF;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC;AAEvE,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,OAAO,EAAE;QACP,iBAAiB,EAAE,OAAO,CAAC;QAC3B,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,YAAY,EAAE,GAAG,CAAC;QAClB,YAAY,EAAE,GAAG,CAAC;QAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,OAAO,CAAC;QACvB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KACrB,CAAC;IACF,YAAY,EAAE;QACZ,OAAO,EAAE;YACP,kCAAkC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACzE,kCAAkC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACzE,eAAe,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACtD,UAAU,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;SAClD,CAAC;QACF,oBAAoB,EAAE,OAAO,CAAC;QAC9B,eAAe,EAAE,MAAM,CAAC;QACxB,MAAM,EAAE;YACN,KAAK,EAAE,MAAM,CAAC;YACd,GAAG,EAAE,MAAM,CAAC;SACb,CAAC;QACF,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE;YACL,QAAQ,EAAE;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC;YAC7C,MAAM,EAAE;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC;YAC3C,MAAM,EAAE;gBACN,gBAAgB,EAAE,MAAM,CAAC;gBACzB,oBAAoB,EAAE,MAAM,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC;gBAChB,KAAK,EAAE,MAAM,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC;gBACd,SAAS,EAAE,MAAM,CAAC;aACnB,CAAC;SACH,CAAC;KACH,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,gBAAgB;IAC3B,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEV,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW;IAMjD,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,cAAc;CAOvB;AAED,qBAAa,gBAAgB;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;gBAEV,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;CAKpF"}
|
package/esm/result.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as errors from './errors.js';
|
|
2
2
|
import { cheerio } from './deps.js';
|
|
3
|
+
import { normalizeHeaders } from './utils.js';
|
|
3
4
|
export class ScrapeResult {
|
|
4
5
|
constructor(data) {
|
|
5
6
|
Object.defineProperty(this, "config", {
|
|
@@ -40,7 +41,8 @@ export class ScrapeResult {
|
|
|
40
41
|
}
|
|
41
42
|
get selector() {
|
|
42
43
|
if (!this._selector) {
|
|
43
|
-
|
|
44
|
+
const headers = normalizeHeaders(this.result.response_headers);
|
|
45
|
+
if (!headers['content-type'].includes('text/html')) {
|
|
44
46
|
throw new errors.ContentTypeError(`Cannot use selector on non-html content-type, received: ${this.result.response_headers['content-type']}`);
|
|
45
47
|
}
|
|
46
48
|
this._selector = cheerio.load(this.result.content);
|
|
@@ -73,19 +75,21 @@ export class ScreenshotResult {
|
|
|
73
75
|
this.result = this.decodeResponse(response, data); // raw result
|
|
74
76
|
}
|
|
75
77
|
defineMetadata(response) {
|
|
76
|
-
const
|
|
78
|
+
const headers = normalizeHeaders(response.headers);
|
|
79
|
+
const contentType = headers['content-type'];
|
|
77
80
|
let extension_name = '';
|
|
78
81
|
if (contentType) {
|
|
79
82
|
extension_name = contentType.split('/')[1].split(';')[0];
|
|
80
83
|
}
|
|
81
84
|
return {
|
|
82
85
|
extension_name: extension_name,
|
|
83
|
-
upstream_status_code: parseInt(
|
|
84
|
-
upstream_url:
|
|
86
|
+
upstream_status_code: parseInt(headers['x-scrapfly-upstream-http-code'] || '200', 10),
|
|
87
|
+
upstream_url: headers['x-scrapfly-upstream-url'] || '',
|
|
85
88
|
};
|
|
86
89
|
}
|
|
87
90
|
decodeResponse(response, data) {
|
|
88
|
-
|
|
91
|
+
const headers = normalizeHeaders(response.headers);
|
|
92
|
+
if (headers['content-type'] === 'json') {
|
|
89
93
|
return JSON.parse(new TextDecoder().decode(data));
|
|
90
94
|
}
|
|
91
95
|
return null;
|
package/esm/utils.d.ts
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
import type { Rec } from './types.js';
|
|
2
2
|
export declare function urlsafe_b64encode(data: string): string;
|
|
3
|
-
export
|
|
3
|
+
export type RequestOptions = {
|
|
4
|
+
url: string;
|
|
5
|
+
method?: string;
|
|
6
|
+
headers?: any;
|
|
7
|
+
body?: string | Uint8Array;
|
|
8
|
+
};
|
|
9
|
+
export declare function fetchRetry(config: RequestOptions, retries?: number, retryDelay?: number): Promise<Response>;
|
|
10
|
+
export declare function normalizeHeaders(headers: Rec<string> | Headers): Rec<string>;
|
|
4
11
|
//# sourceMappingURL=utils.d.ts.map
|
package/esm/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;CAC5B,CAAC;AAEF,wBAAsB,UAAU,CAC9B,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,MAAU,EACnB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,QAAQ,CAAC,CA+BnB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,CAc5E"}
|
package/esm/utils.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { log } from './logger.js';
|
|
1
2
|
export function urlsafe_b64encode(data) {
|
|
2
3
|
const encoder = new TextEncoder();
|
|
3
4
|
const encoded = encoder.encode(data);
|
|
@@ -11,12 +12,15 @@ export async function fetchRetry(config, retries = 3, retryDelay = 1000) {
|
|
|
11
12
|
let lastError = null;
|
|
12
13
|
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
13
14
|
try {
|
|
14
|
-
// XXX:
|
|
15
|
-
const
|
|
15
|
+
// XXX: note that cloudflare workers don't support init options
|
|
16
|
+
const { url, ...reqInit } = config;
|
|
17
|
+
const response = await fetch(new Request(url, reqInit));
|
|
16
18
|
// retry 5xx status codes
|
|
17
19
|
if (response.status >= 500 && response.status < 600) {
|
|
20
|
+
const _text = await response.text(); // consume response to prevent leak
|
|
18
21
|
lastError = new Error(`Fetch failed with status: ${response.status}`);
|
|
19
22
|
if (attempt < retries) {
|
|
23
|
+
log.debug(`request failed ${response.status} (${_text}): retry ${attempt}/${retries} after ${retryDelay}ms`);
|
|
20
24
|
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
21
25
|
}
|
|
22
26
|
}
|
|
@@ -34,3 +38,17 @@ export async function fetchRetry(config, retries = 3, retryDelay = 1000) {
|
|
|
34
38
|
}
|
|
35
39
|
throw lastError;
|
|
36
40
|
}
|
|
41
|
+
export function normalizeHeaders(headers) {
|
|
42
|
+
const normalizedHeaders = {};
|
|
43
|
+
if (headers instanceof Headers) {
|
|
44
|
+
headers.forEach((value, key) => {
|
|
45
|
+
normalizedHeaders[key.toLowerCase()] = value;
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
Object.keys(headers).forEach((key) => {
|
|
50
|
+
normalizedHeaders[key.toLowerCase()] = headers[key];
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
return normalizedHeaders;
|
|
54
|
+
}
|
package/package.json
CHANGED
package/script/client.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
import { fetchRetry } from './utils.js';
|
|
3
2
|
import * as errors from './errors.js';
|
|
4
3
|
import type { ScrapeConfig } from './scrapeconfig.js';
|
|
@@ -17,6 +16,10 @@ export declare class ScrapflyClient {
|
|
|
17
16
|
* Raise appropriate error for given response and scrape result
|
|
18
17
|
*/
|
|
19
18
|
errResult(response: Response, result: ScrapeResult): errors.ScrapflyError;
|
|
19
|
+
/**
|
|
20
|
+
* Handle clob and blob large objects
|
|
21
|
+
*/
|
|
22
|
+
handleLargeObjects(result: any, format: 'clob' | 'blob'): Promise<ScrapeResult>;
|
|
20
23
|
/**
|
|
21
24
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
22
25
|
*/
|
|
@@ -30,21 +33,21 @@ export declare class ScrapflyClient {
|
|
|
30
33
|
*/
|
|
31
34
|
scrape(config: ScrapeConfig): Promise<ScrapeResult>;
|
|
32
35
|
/**
|
|
33
|
-
|
|
34
|
-
|
|
36
|
+
Concurrently scrape multiple configs
|
|
37
|
+
This is a async generator call it like this:
|
|
35
38
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
39
|
+
const results = [];
|
|
40
|
+
const errors = [];
|
|
41
|
+
for await (const resultOrError of client.concurrentScrape(configs)) {
|
|
42
|
+
if (resultOrError instanceof Error) {
|
|
43
|
+
errors.push(resultOrError);
|
|
44
|
+
} else {
|
|
45
|
+
results.push(resultOrError);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
45
48
|
|
|
46
|
-
|
|
47
|
-
|
|
49
|
+
@param concurrencyLimit: if not set it will be taken from your account info
|
|
50
|
+
*/
|
|
48
51
|
concurrentScrape(configs: ScrapeConfig[], concurrencyLimit?: number): AsyncGenerator<ScrapeResult | Error | undefined, void, undefined>;
|
|
49
52
|
/**
|
|
50
53
|
* Save screenshot response to a file
|
package/script/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACG,kBAAkB,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAgCrF;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAkDzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
|