scrapfly-sdk 0.6.4 → 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/esm/client.d.ts +4 -1
- package/esm/client.d.ts.map +1 -1
- package/esm/client.js +39 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts.map +1 -1
- package/esm/polyfill.d.ts +0 -1
- package/esm/polyfill.d.ts.map +1 -1
- package/esm/result.d.ts +0 -1
- package/esm/result.d.ts.map +1 -1
- package/esm/result.js +9 -5
- package/esm/utils.d.ts +2 -1
- package/esm/utils.d.ts.map +1 -1
- package/esm/utils.js +14 -0
- package/package.json +1 -1
- package/script/client.d.ts +4 -1
- package/script/client.d.ts.map +1 -1
- package/script/client.js +39 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/assert_path.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/basename.js +3 -4
- package/script/deps/jsr.io/@std/path/1.0.1/_common/common.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/format.js +2 -3
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/glob_to_reg_exp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize_string.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/strip_trailing_separators.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/basename.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/common.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/extname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/format.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/glob_to_regexp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/is_absolute.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/is_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/join.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/join_globs.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/normalize_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/parse.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/_util.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/basename.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/extname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/format.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/glob_to_regexp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/is_absolute.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join_globs.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/parse.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/resolve.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_namespaced_path.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/resolve.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/to_namespaced_path.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/_util.js +3 -4
- package/script/deps/jsr.io/@std/path/1.0.1/windows/basename.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/extname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/format.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/glob_to_regexp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/is_absolute.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join_globs.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/parse.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/resolve.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_namespaced_path.js +1 -2
- package/script/polyfill.d.ts +0 -1
- package/script/polyfill.d.ts.map +1 -1
- package/script/polyfill.js +3 -3
- package/script/result.d.ts +0 -1
- package/script/result.d.ts.map +1 -1
- package/script/result.js +9 -5
- package/script/utils.d.ts +2 -1
- package/script/utils.d.ts.map +1 -1
- package/script/utils.js +17 -3
package/README.md
CHANGED
package/esm/client.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
import { fetchRetry } from './utils.js';
|
|
3
2
|
import * as errors from './errors.js';
|
|
4
3
|
import type { ScrapeConfig } from './scrapeconfig.js';
|
|
@@ -17,6 +16,10 @@ export declare class ScrapflyClient {
|
|
|
17
16
|
* Raise appropriate error for given response and scrape result
|
|
18
17
|
*/
|
|
19
18
|
errResult(response: Response, result: ScrapeResult): errors.ScrapflyError;
|
|
19
|
+
/**
|
|
20
|
+
* Handle clob and blob large objects
|
|
21
|
+
*/
|
|
22
|
+
handleLargeObjects(result: any, format: 'clob' | 'blob'): Promise<ScrapeResult>;
|
|
20
23
|
/**
|
|
21
24
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
22
25
|
*/
|
package/esm/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACG,kBAAkB,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAqCrF;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAkDzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAqB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
|
package/esm/client.js
CHANGED
|
@@ -105,6 +105,41 @@ export class ScrapflyClient {
|
|
|
105
105
|
}
|
|
106
106
|
return new errors.ScrapflyError(message, args);
|
|
107
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* Handle clob and blob large objects
|
|
110
|
+
*/
|
|
111
|
+
async handleLargeObjects(result, format) {
|
|
112
|
+
let response;
|
|
113
|
+
try {
|
|
114
|
+
const url = new URL(result.content);
|
|
115
|
+
const params = { key: this.key };
|
|
116
|
+
url.search = new URLSearchParams(params).toString();
|
|
117
|
+
response = await this.fetch({
|
|
118
|
+
url: url.toString(),
|
|
119
|
+
method: 'GET',
|
|
120
|
+
headers: {
|
|
121
|
+
'user-agent': this.ua,
|
|
122
|
+
'accept-encoding': 'gzip, deflate, br',
|
|
123
|
+
accept: 'application/json',
|
|
124
|
+
},
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
catch (e) {
|
|
128
|
+
log.error('error', e);
|
|
129
|
+
throw e;
|
|
130
|
+
}
|
|
131
|
+
let content;
|
|
132
|
+
if (format === 'clob') {
|
|
133
|
+
content = await response.text();
|
|
134
|
+
result.format = 'text';
|
|
135
|
+
}
|
|
136
|
+
if (format === 'blob') {
|
|
137
|
+
content = new Uint8Array(await response.arrayBuffer());
|
|
138
|
+
result.format = 'binary';
|
|
139
|
+
}
|
|
140
|
+
result.content = content;
|
|
141
|
+
return result;
|
|
142
|
+
}
|
|
108
143
|
/**
|
|
109
144
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
110
145
|
*/
|
|
@@ -186,6 +221,10 @@ export class ScrapflyClient {
|
|
|
186
221
|
}
|
|
187
222
|
throw new errors.ApiHttpClientError(JSON.stringify(data));
|
|
188
223
|
}
|
|
224
|
+
const content_format = data.result.format;
|
|
225
|
+
if (content_format === 'clob' || content_format === 'blob') {
|
|
226
|
+
data.result = await this.handleLargeObjects(data.result, content_format);
|
|
227
|
+
}
|
|
189
228
|
const result = this.handleResponse(response, new ScrapeResult({
|
|
190
229
|
config: data.config,
|
|
191
230
|
context: data.context,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"AAGA,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,OAM1C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/from_file_url.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,GAAG,MAAM,CAErD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,MAAM,CAKrD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.ts"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAU3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/to_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/to_file_url.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAE3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,MAAM,CAYrD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.ts"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAgB3C"}
|
package/esm/polyfill.d.ts
CHANGED
package/esm/polyfill.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"polyfill.d.ts","sourceRoot":"","sources":["../src/polyfill.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"polyfill.d.ts","sourceRoot":"","sources":["../src/polyfill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,iBAAiB,CAAC;AAC3C,eAAO,MAAM,MAAM,SAAsC,CAAC;AAE1D,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjG;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,GAAG,GAAG,EAClB,IAAI,EAAE,UAAU,EAChB,OAAO,GAAE,OAAO,CAAC,IAAI,CAAC,gBAAqB,GAC1C,OAAO,CAAC,IAAI,CAAC,CAOf"}
|
package/esm/result.d.ts
CHANGED
package/esm/result.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;IACf,uBAAuB,EAAE,OAAO,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACrB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE;QACJ,OAAO,EAAE,KAAK,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACtE,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpB,EAAE,EAAE;QACF,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,GAAG,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,YAAY,EAAE;QACZ,4BAA4B,CAAC,EAAE,MAAM,CAAC;QACtC,WAAW,CAAC,EAAE;YACZ,QAAQ,EAAE,MAAM,CAAC;YACjB,QAAQ,EAAE,MAAM,CAAC;YACjB,QAAQ,EAAE,GAAG,CAAC;YACd,KAAK,EAAE,KAAK,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC;gBACf,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACpB,QAAQ,EAAE,MAAM,CAAC;gBACjB,QAAQ,EAAE,OAAO,CAAC;gBAClB,MAAM,CAAC,EAAE,MAAM,CAAC;gBAChB,OAAO,EAAE,OAAO,CAAC;aAClB,CAAC,CAAC;SACJ,CAAC;QACF,kBAAkB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,oBAAoB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnC,UAAU,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACxB,QAAQ,CAAC,EAAE,KAAK,CAAC;YACf,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YACrB,MAAM,EAAE,MAAM,CAAC;YACf,IAAI,EAAE,MAAM,CAAC;YACb,GAAG,EAAE,MAAM,CAAC;YACZ,QAAQ,EAAE;gBACR,IAAI,EAAE,MAAM,CAAC;gBACb,QAAQ,EAAE,MAAM,CAAC;gBACjB,MAAM,EAAE,MAAM,CAAC;gBACf,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM,EAAE,MAAM,CAAC;aAChB,CAAC;SACH,CAAC,CAAC;KACJ,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,OAAO,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;IACH,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE;YACH,WAAW,EAAE,MAAM,CAAC;YACpB,QAAQ,EAAE,MAAM,CAAC;YACjB,IAAI,EAAE,MAAM,CAAC;YACb,MAAM,EAAE,MAAM,CAAC;YACf,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;SACtB,CAAC;QACF,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,WAAW,EAAE,GAAG,CAAC;QACf,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE;QACJ,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;KACxB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,qBAAa,YAAY;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,WAAW,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,SAAS,CAAiC;gBAEtC,IAAI,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,OAAO,EAAE,WAAW,CAAC;QAAC,MAAM,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAQhG,IAAI,QAAQ,IAAI,OAAO,CAAC,UAAU,CAWjC;CACF;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC;AAEvE,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,OAAO,EAAE;QACP,iBAAiB,EAAE,OAAO,CAAC;QAC3B,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,YAAY,EAAE,GAAG,CAAC;QAClB,YAAY,EAAE,GAAG,CAAC;QAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,OAAO,CAAC;QACvB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KACrB,CAAC;IACF,YAAY,EAAE;QACZ,OAAO,EAAE;YACP,kCAAkC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACzE,kCAAkC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACzE,eAAe,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACtD,UAAU,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;SAClD,CAAC;QACF,oBAAoB,EAAE,OAAO,CAAC;QAC9B,eAAe,EAAE,MAAM,CAAC;QACxB,MAAM,EAAE;YACN,KAAK,EAAE,MAAM,CAAC;YACd,GAAG,EAAE,MAAM,CAAC;SACb,CAAC;QACF,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE;YACL,QAAQ,EAAE;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC;YAC7C,MAAM,EAAE;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC;YAC3C,MAAM,EAAE;gBACN,gBAAgB,EAAE,MAAM,CAAC;gBACzB,oBAAoB,EAAE,MAAM,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC;gBAChB,KAAK,EAAE,MAAM,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC;gBACd,SAAS,EAAE,MAAM,CAAC;aACnB,CAAC;SACH,CAAC;KACH,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,gBAAgB;IAC3B,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEV,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW;IAMjD,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,cAAc;CAOvB;AAED,qBAAa,gBAAgB;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;gBAEV,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;CAKpF"}
|
package/esm/result.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as errors from './errors.js';
|
|
2
2
|
import { cheerio } from './deps.js';
|
|
3
|
+
import { normalizeHeaders } from './utils.js';
|
|
3
4
|
export class ScrapeResult {
|
|
4
5
|
constructor(data) {
|
|
5
6
|
Object.defineProperty(this, "config", {
|
|
@@ -40,7 +41,8 @@ export class ScrapeResult {
|
|
|
40
41
|
}
|
|
41
42
|
get selector() {
|
|
42
43
|
if (!this._selector) {
|
|
43
|
-
|
|
44
|
+
const headers = normalizeHeaders(this.result.response_headers);
|
|
45
|
+
if (!headers['content-type'].includes('text/html')) {
|
|
44
46
|
throw new errors.ContentTypeError(`Cannot use selector on non-html content-type, received: ${this.result.response_headers['content-type']}`);
|
|
45
47
|
}
|
|
46
48
|
this._selector = cheerio.load(this.result.content);
|
|
@@ -73,19 +75,21 @@ export class ScreenshotResult {
|
|
|
73
75
|
this.result = this.decodeResponse(response, data); // raw result
|
|
74
76
|
}
|
|
75
77
|
defineMetadata(response) {
|
|
76
|
-
const
|
|
78
|
+
const headers = normalizeHeaders(response.headers);
|
|
79
|
+
const contentType = headers['content-type'];
|
|
77
80
|
let extension_name = '';
|
|
78
81
|
if (contentType) {
|
|
79
82
|
extension_name = contentType.split('/')[1].split(';')[0];
|
|
80
83
|
}
|
|
81
84
|
return {
|
|
82
85
|
extension_name: extension_name,
|
|
83
|
-
upstream_status_code: parseInt(
|
|
84
|
-
upstream_url:
|
|
86
|
+
upstream_status_code: parseInt(headers['x-scrapfly-upstream-http-code'] || '200', 10),
|
|
87
|
+
upstream_url: headers['x-scrapfly-upstream-url'] || '',
|
|
85
88
|
};
|
|
86
89
|
}
|
|
87
90
|
decodeResponse(response, data) {
|
|
88
|
-
|
|
91
|
+
const headers = normalizeHeaders(response.headers);
|
|
92
|
+
if (headers['content-type'] === 'json') {
|
|
89
93
|
return JSON.parse(new TextDecoder().decode(data));
|
|
90
94
|
}
|
|
91
95
|
return null;
|
package/esm/utils.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
import type { Rec } from './types.js';
|
|
2
2
|
export declare function urlsafe_b64encode(data: string): string;
|
|
3
3
|
export type RequestOptions = {
|
|
4
4
|
url: string;
|
|
@@ -7,4 +7,5 @@ export type RequestOptions = {
|
|
|
7
7
|
body?: string | Uint8Array;
|
|
8
8
|
};
|
|
9
9
|
export declare function fetchRetry(config: RequestOptions, retries?: number, retryDelay?: number): Promise<Response>;
|
|
10
|
+
export declare function normalizeHeaders(headers: Rec<string> | Headers): Rec<string>;
|
|
10
11
|
//# sourceMappingURL=utils.d.ts.map
|
package/esm/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;CAC5B,CAAC;AAEF,wBAAsB,UAAU,CAC9B,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,MAAU,EACnB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,QAAQ,CAAC,CA+BnB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,CAc5E"}
|
package/esm/utils.js
CHANGED
|
@@ -38,3 +38,17 @@ export async function fetchRetry(config, retries = 3, retryDelay = 1000) {
|
|
|
38
38
|
}
|
|
39
39
|
throw lastError;
|
|
40
40
|
}
|
|
41
|
+
export function normalizeHeaders(headers) {
|
|
42
|
+
const normalizedHeaders = {};
|
|
43
|
+
if (headers instanceof Headers) {
|
|
44
|
+
headers.forEach((value, key) => {
|
|
45
|
+
normalizedHeaders[key.toLowerCase()] = value;
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
Object.keys(headers).forEach((key) => {
|
|
50
|
+
normalizedHeaders[key.toLowerCase()] = headers[key];
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
return normalizedHeaders;
|
|
54
|
+
}
|
package/package.json
CHANGED
package/script/client.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
import { fetchRetry } from './utils.js';
|
|
3
2
|
import * as errors from './errors.js';
|
|
4
3
|
import type { ScrapeConfig } from './scrapeconfig.js';
|
|
@@ -17,6 +16,10 @@ export declare class ScrapflyClient {
|
|
|
17
16
|
* Raise appropriate error for given response and scrape result
|
|
18
17
|
*/
|
|
19
18
|
errResult(response: Response, result: ScrapeResult): errors.ScrapflyError;
|
|
19
|
+
/**
|
|
20
|
+
* Handle clob and blob large objects
|
|
21
|
+
*/
|
|
22
|
+
handleLargeObjects(result: any, format: 'clob' | 'blob'): Promise<ScrapeResult>;
|
|
20
23
|
/**
|
|
21
24
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
22
25
|
*/
|
package/script/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACG,kBAAkB,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAqCrF;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAkDzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAqB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
|
package/script/client.js
CHANGED
|
@@ -131,6 +131,41 @@ class ScrapflyClient {
|
|
|
131
131
|
}
|
|
132
132
|
return new errors.ScrapflyError(message, args);
|
|
133
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* Handle clob and blob large objects
|
|
136
|
+
*/
|
|
137
|
+
async handleLargeObjects(result, format) {
|
|
138
|
+
let response;
|
|
139
|
+
try {
|
|
140
|
+
const url = new URL(result.content);
|
|
141
|
+
const params = { key: this.key };
|
|
142
|
+
url.search = new URLSearchParams(params).toString();
|
|
143
|
+
response = await this.fetch({
|
|
144
|
+
url: url.toString(),
|
|
145
|
+
method: 'GET',
|
|
146
|
+
headers: {
|
|
147
|
+
'user-agent': this.ua,
|
|
148
|
+
'accept-encoding': 'gzip, deflate, br',
|
|
149
|
+
accept: 'application/json',
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
catch (e) {
|
|
154
|
+
logger_js_1.log.error('error', e);
|
|
155
|
+
throw e;
|
|
156
|
+
}
|
|
157
|
+
let content;
|
|
158
|
+
if (format === 'clob') {
|
|
159
|
+
content = await response.text();
|
|
160
|
+
result.format = 'text';
|
|
161
|
+
}
|
|
162
|
+
if (format === 'blob') {
|
|
163
|
+
content = new Uint8Array(await response.arrayBuffer());
|
|
164
|
+
result.format = 'binary';
|
|
165
|
+
}
|
|
166
|
+
result.content = content;
|
|
167
|
+
return result;
|
|
168
|
+
}
|
|
134
169
|
/**
|
|
135
170
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
136
171
|
*/
|
|
@@ -212,6 +247,10 @@ class ScrapflyClient {
|
|
|
212
247
|
}
|
|
213
248
|
throw new errors.ApiHttpClientError(JSON.stringify(data));
|
|
214
249
|
}
|
|
250
|
+
const content_format = data.result.format;
|
|
251
|
+
if (content_format === 'clob' || content_format === 'blob') {
|
|
252
|
+
data.result = await this.handleLargeObjects(data.result, content_format);
|
|
253
|
+
}
|
|
215
254
|
const result = this.handleResponse(response, new result_js_1.ScrapeResult({
|
|
216
255
|
config: data.config,
|
|
217
256
|
context: data.context,
|
|
@@ -2,10 +2,9 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// Copyright the Browserify authors. MIT License.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.assertPath =
|
|
5
|
+
exports.assertPath = assertPath;
|
|
6
6
|
function assertPath(path) {
|
|
7
7
|
if (typeof path !== "string") {
|
|
8
8
|
throw new TypeError(`Path must be a string. Received ${JSON.stringify(path)}`);
|
|
9
9
|
}
|
|
10
10
|
}
|
|
11
|
-
exports.assertPath = assertPath;
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.
|
|
5
|
+
exports.stripSuffix = stripSuffix;
|
|
6
|
+
exports.lastPathSegment = lastPathSegment;
|
|
7
|
+
exports.assertArgs = assertArgs;
|
|
6
8
|
const assert_path_js_1 = require("./assert_path.js");
|
|
7
9
|
function stripSuffix(name, suffix) {
|
|
8
10
|
if (suffix.length >= name.length) {
|
|
@@ -16,7 +18,6 @@ function stripSuffix(name, suffix) {
|
|
|
16
18
|
}
|
|
17
19
|
return name.slice(0, -suffix.length);
|
|
18
20
|
}
|
|
19
|
-
exports.stripSuffix = stripSuffix;
|
|
20
21
|
function lastPathSegment(path, isSep, start = 0) {
|
|
21
22
|
let matchedNonSeparator = false;
|
|
22
23
|
let end = path.length;
|
|
@@ -34,7 +35,6 @@ function lastPathSegment(path, isSep, start = 0) {
|
|
|
34
35
|
}
|
|
35
36
|
return path.slice(start, end);
|
|
36
37
|
}
|
|
37
|
-
exports.lastPathSegment = lastPathSegment;
|
|
38
38
|
function assertArgs(path, suffix) {
|
|
39
39
|
(0, assert_path_js_1.assertPath)(path);
|
|
40
40
|
if (path.length === 0)
|
|
@@ -43,4 +43,3 @@ function assertArgs(path, suffix) {
|
|
|
43
43
|
throw new TypeError(`Suffix must be a string. Received ${JSON.stringify(suffix)}`);
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
|
-
exports.assertArgs = assertArgs;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.common =
|
|
5
|
+
exports.common = common;
|
|
6
6
|
function common(paths, sep) {
|
|
7
7
|
const [first = "", ...remaining] = paths;
|
|
8
8
|
const parts = first.split(sep);
|
|
@@ -24,4 +24,3 @@ function common(paths, sep) {
|
|
|
24
24
|
}
|
|
25
25
|
return parts.slice(0, endOfPrefix).join(sep) + append;
|
|
26
26
|
}
|
|
27
|
-
exports.common = common;
|
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.assertArg =
|
|
5
|
+
exports.assertArg = assertArg;
|
|
6
6
|
const assert_path_js_1 = require("./assert_path.js");
|
|
7
7
|
function assertArg(path) {
|
|
8
8
|
(0, assert_path_js_1.assertPath)(path);
|
|
9
9
|
if (path.length === 0)
|
|
10
10
|
return ".";
|
|
11
11
|
}
|
|
12
|
-
exports.assertArg = assertArg;
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.
|
|
5
|
+
exports._format = _format;
|
|
6
|
+
exports.assertArg = assertArg;
|
|
6
7
|
function _format(sep, pathObject) {
|
|
7
8
|
const dir = pathObject.dir || pathObject.root;
|
|
8
9
|
const base = pathObject.base ||
|
|
@@ -15,10 +16,8 @@ function _format(sep, pathObject) {
|
|
|
15
16
|
return dir + base;
|
|
16
17
|
return dir + sep + base;
|
|
17
18
|
}
|
|
18
|
-
exports._format = _format;
|
|
19
19
|
function assertArg(pathObject) {
|
|
20
20
|
if (pathObject === null || typeof pathObject !== "object") {
|
|
21
21
|
throw new TypeError(`The "pathObject" argument must be of type Object. Received type ${typeof pathObject}`);
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
|
-
exports.assertArg = assertArg;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"AAGA,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,OAM1C"}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.assertArg =
|
|
5
|
+
exports.assertArg = assertArg;
|
|
6
6
|
function assertArg(url) {
|
|
7
7
|
url = url instanceof URL ? url : new URL(url);
|
|
8
8
|
if (url.protocol !== "file:") {
|
|
@@ -10,4 +10,3 @@ function assertArg(url) {
|
|
|
10
10
|
}
|
|
11
11
|
return url;
|
|
12
12
|
}
|
|
13
|
-
exports.assertArg = assertArg;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports._globToRegExp =
|
|
5
|
+
exports._globToRegExp = _globToRegExp;
|
|
6
6
|
const REG_EXP_ESCAPE_CHARS = [
|
|
7
7
|
"!",
|
|
8
8
|
"$",
|
|
@@ -235,4 +235,3 @@ caseInsensitive = false, } = {}) {
|
|
|
235
235
|
regExpString = `^${regExpString}$`;
|
|
236
236
|
return new RegExp(regExpString, caseInsensitive ? "i" : "");
|
|
237
237
|
}
|
|
238
|
-
exports._globToRegExp = _globToRegExp;
|
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.assertArg =
|
|
5
|
+
exports.assertArg = assertArg;
|
|
6
6
|
const assert_path_js_1 = require("./assert_path.js");
|
|
7
7
|
function assertArg(path) {
|
|
8
8
|
(0, assert_path_js_1.assertPath)(path);
|
|
9
9
|
if (path.length === 0)
|
|
10
10
|
return ".";
|
|
11
11
|
}
|
|
12
|
-
exports.assertArg = assertArg;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// Ported from https://github.com/browserify/path-browserify/
|
|
5
5
|
// This module is browser compatible.
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.normalizeString =
|
|
7
|
+
exports.normalizeString = normalizeString;
|
|
8
8
|
const constants_js_1 = require("./constants.js");
|
|
9
9
|
// Resolves . and .. elements in a path with directory names
|
|
10
10
|
function normalizeString(path, allowAboveRoot, separator, isPathSeparator) {
|
|
@@ -78,4 +78,3 @@ function normalizeString(path, allowAboveRoot, separator, isPathSeparator) {
|
|
|
78
78
|
}
|
|
79
79
|
return res;
|
|
80
80
|
}
|
|
81
|
-
exports.normalizeString = normalizeString;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.assertArgs =
|
|
5
|
+
exports.assertArgs = assertArgs;
|
|
6
6
|
const assert_path_js_1 = require("./assert_path.js");
|
|
7
7
|
function assertArgs(from, to) {
|
|
8
8
|
(0, assert_path_js_1.assertPath)(from);
|
|
@@ -10,4 +10,3 @@ function assertArgs(from, to) {
|
|
|
10
10
|
if (from === to)
|
|
11
11
|
return "";
|
|
12
12
|
}
|
|
13
|
-
exports.assertArgs = assertArgs;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// Ported from https://github.com/browserify/path-browserify/
|
|
5
5
|
// This module is browser compatible.
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.stripTrailingSeparators =
|
|
7
|
+
exports.stripTrailingSeparators = stripTrailingSeparators;
|
|
8
8
|
function stripTrailingSeparators(segment, isSep) {
|
|
9
9
|
if (segment.length <= 1) {
|
|
10
10
|
return segment;
|
|
@@ -20,4 +20,3 @@ function stripTrailingSeparators(segment, isSep) {
|
|
|
20
20
|
}
|
|
21
21
|
return segment.slice(0, end);
|
|
22
22
|
}
|
|
23
|
-
exports.stripTrailingSeparators = stripTrailingSeparators;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.encodeWhitespace =
|
|
5
|
+
exports.encodeWhitespace = encodeWhitespace;
|
|
6
6
|
const WHITESPACE_ENCODINGS = {
|
|
7
7
|
"\u0009": "%09",
|
|
8
8
|
"\u000A": "%0A",
|
|
@@ -16,4 +16,3 @@ function encodeWhitespace(string) {
|
|
|
16
16
|
return WHITESPACE_ENCODINGS[c] ?? c;
|
|
17
17
|
});
|
|
18
18
|
}
|
|
19
|
-
exports.encodeWhitespace = encodeWhitespace;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.basename =
|
|
5
|
+
exports.basename = basename;
|
|
6
6
|
const _os_js_1 = require("./_os.js");
|
|
7
7
|
const basename_js_1 = require("./posix/basename.js");
|
|
8
8
|
const basename_js_2 = require("./windows/basename.js");
|
|
@@ -34,4 +34,3 @@ function basename(path, suffix = "") {
|
|
|
34
34
|
? (0, basename_js_2.basename)(path, suffix)
|
|
35
35
|
: (0, basename_js_1.basename)(path, suffix);
|
|
36
36
|
}
|
|
37
|
-
exports.basename = basename;
|