scrapfly-sdk 0.6.4 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/esm/client.d.ts +4 -1
- package/esm/client.d.ts.map +1 -1
- package/esm/client.js +37 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts.map +1 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts +0 -1
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts.map +1 -1
- package/esm/polyfill.d.ts +0 -1
- package/esm/polyfill.d.ts.map +1 -1
- package/esm/result.d.ts +0 -1
- package/esm/result.d.ts.map +1 -1
- package/esm/result.js +9 -5
- package/esm/utils.d.ts +2 -1
- package/esm/utils.d.ts.map +1 -1
- package/esm/utils.js +14 -0
- package/package.json +1 -1
- package/script/client.d.ts +4 -1
- package/script/client.d.ts.map +1 -1
- package/script/client.js +37 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/assert_path.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/basename.js +3 -4
- package/script/deps/jsr.io/@std/path/1.0.1/_common/common.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/format.js +2 -3
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/glob_to_reg_exp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize_string.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/strip_trailing_separators.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/_common/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/basename.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/common.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/extname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/format.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/glob_to_regexp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/is_absolute.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/is_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/join.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/join_globs.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/normalize_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/parse.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/_util.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/basename.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/extname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/format.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/glob_to_regexp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/is_absolute.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join_globs.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/parse.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/resolve.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_namespaced_path.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/resolve.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/to_namespaced_path.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/_util.js +3 -4
- package/script/deps/jsr.io/@std/path/1.0.1/windows/basename.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/dirname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/extname.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/format.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/glob_to_regexp.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/is_absolute.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join_globs.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize_glob.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/parse.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/relative.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/resolve.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts +0 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts.map +1 -1
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.js +1 -2
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_namespaced_path.js +1 -2
- package/script/polyfill.d.ts +0 -1
- package/script/polyfill.d.ts.map +1 -1
- package/script/polyfill.js +3 -3
- package/script/result.d.ts +0 -1
- package/script/result.d.ts.map +1 -1
- package/script/result.js +9 -5
- package/script/utils.d.ts +2 -1
- package/script/utils.d.ts.map +1 -1
- package/script/utils.js +17 -3
package/README.md
CHANGED
package/esm/client.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
import { fetchRetry } from './utils.js';
|
|
3
2
|
import * as errors from './errors.js';
|
|
4
3
|
import type { ScrapeConfig } from './scrapeconfig.js';
|
|
@@ -17,6 +16,10 @@ export declare class ScrapflyClient {
|
|
|
17
16
|
* Raise appropriate error for given response and scrape result
|
|
18
17
|
*/
|
|
19
18
|
errResult(response: Response, result: ScrapeResult): errors.ScrapflyError;
|
|
19
|
+
/**
|
|
20
|
+
* Handle clob and blob large objects
|
|
21
|
+
*/
|
|
22
|
+
handleLargeObjects(result: any, format: 'clob' | 'blob'): Promise<ScrapeResult>;
|
|
20
23
|
/**
|
|
21
24
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
22
25
|
*/
|
package/esm/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACG,kBAAkB,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAgCrF;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAkDzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
|
package/esm/client.js
CHANGED
|
@@ -105,6 +105,39 @@ export class ScrapflyClient {
|
|
|
105
105
|
}
|
|
106
106
|
return new errors.ScrapflyError(message, args);
|
|
107
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* Handle clob and blob large objects
|
|
110
|
+
*/
|
|
111
|
+
async handleLargeObjects(result, format) {
|
|
112
|
+
let response;
|
|
113
|
+
try {
|
|
114
|
+
const url = new URL(result.content);
|
|
115
|
+
const params = { key: this.key };
|
|
116
|
+
url.search = new URLSearchParams(params).toString();
|
|
117
|
+
response = await this.fetch({
|
|
118
|
+
url: url.toString(),
|
|
119
|
+
method: 'GET',
|
|
120
|
+
headers: {
|
|
121
|
+
'user-agent': this.ua,
|
|
122
|
+
'accept-encoding': 'gzip, deflate, br',
|
|
123
|
+
accept: 'application/json',
|
|
124
|
+
},
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
catch (e) {
|
|
128
|
+
log.error('error', e);
|
|
129
|
+
throw e;
|
|
130
|
+
}
|
|
131
|
+
const content = await response.text();
|
|
132
|
+
result.content = content;
|
|
133
|
+
if (format === 'clob') {
|
|
134
|
+
result.format = 'text';
|
|
135
|
+
}
|
|
136
|
+
if (format === 'blob') {
|
|
137
|
+
result.format = 'binary';
|
|
138
|
+
}
|
|
139
|
+
return result;
|
|
140
|
+
}
|
|
108
141
|
/**
|
|
109
142
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
110
143
|
*/
|
|
@@ -186,6 +219,10 @@ export class ScrapflyClient {
|
|
|
186
219
|
}
|
|
187
220
|
throw new errors.ApiHttpClientError(JSON.stringify(data));
|
|
188
221
|
}
|
|
222
|
+
const content_format = data.result.format;
|
|
223
|
+
if (content_format === 'clob' || content_format === 'blob') {
|
|
224
|
+
data.result = await this.handleLargeObjects(data.result, content_format);
|
|
225
|
+
}
|
|
189
226
|
const result = this.handleResponse(response, new ScrapeResult({
|
|
190
227
|
config: data.config,
|
|
191
228
|
context: data.context,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"AAGA,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,OAM1C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/from_file_url.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,GAAG,MAAM,CAErD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,MAAM,CAKrD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.ts"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAU3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/to_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/to_file_url.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAE3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,MAAM,CAYrD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.ts"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAgB3C"}
|
package/esm/polyfill.d.ts
CHANGED
package/esm/polyfill.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"polyfill.d.ts","sourceRoot":"","sources":["../src/polyfill.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"polyfill.d.ts","sourceRoot":"","sources":["../src/polyfill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,iBAAiB,CAAC;AAC3C,eAAO,MAAM,MAAM,SAAsC,CAAC;AAE1D,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjG;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,GAAG,GAAG,EAClB,IAAI,EAAE,UAAU,EAChB,OAAO,GAAE,OAAO,CAAC,IAAI,CAAC,gBAAqB,GAC1C,OAAO,CAAC,IAAI,CAAC,CAOf"}
|
package/esm/result.d.ts
CHANGED
package/esm/result.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;IACf,uBAAuB,EAAE,OAAO,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACrB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE;QACJ,OAAO,EAAE,KAAK,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACtE,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpB,EAAE,EAAE;QACF,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,GAAG,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,YAAY,EAAE;QACZ,4BAA4B,CAAC,EAAE,MAAM,CAAC;QACtC,WAAW,CAAC,EAAE;YACZ,QAAQ,EAAE,MAAM,CAAC;YACjB,QAAQ,EAAE,MAAM,CAAC;YACjB,QAAQ,EAAE,GAAG,CAAC;YACd,KAAK,EAAE,KAAK,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC;gBACf,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACpB,QAAQ,EAAE,MAAM,CAAC;gBACjB,QAAQ,EAAE,OAAO,CAAC;gBAClB,MAAM,CAAC,EAAE,MAAM,CAAC;gBAChB,OAAO,EAAE,OAAO,CAAC;aAClB,CAAC,CAAC;SACJ,CAAC;QACF,kBAAkB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,oBAAoB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnC,UAAU,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACxB,QAAQ,CAAC,EAAE,KAAK,CAAC;YACf,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YACrB,MAAM,EAAE,MAAM,CAAC;YACf,IAAI,EAAE,MAAM,CAAC;YACb,GAAG,EAAE,MAAM,CAAC;YACZ,QAAQ,EAAE;gBACR,IAAI,EAAE,MAAM,CAAC;gBACb,QAAQ,EAAE,MAAM,CAAC;gBACjB,MAAM,EAAE,MAAM,CAAC;gBACf,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM,EAAE,MAAM,CAAC;aAChB,CAAC;SACH,CAAC,CAAC;KACJ,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,OAAO,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;IACH,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE;YACH,WAAW,EAAE,MAAM,CAAC;YACpB,QAAQ,EAAE,MAAM,CAAC;YACjB,IAAI,EAAE,MAAM,CAAC;YACb,MAAM,EAAE,MAAM,CAAC;YACf,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;SACtB,CAAC;QACF,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,WAAW,EAAE,GAAG,CAAC;QACf,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE;QACJ,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;KACxB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,qBAAa,YAAY;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,WAAW,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,SAAS,CAAiC;gBAEtC,IAAI,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,OAAO,EAAE,WAAW,CAAC;QAAC,MAAM,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAQhG,IAAI,QAAQ,IAAI,OAAO,CAAC,UAAU,CAWjC;CACF;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC;AAEvE,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,OAAO,EAAE;QACP,iBAAiB,EAAE,OAAO,CAAC;QAC3B,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,YAAY,EAAE,GAAG,CAAC;QAClB,YAAY,EAAE,GAAG,CAAC;QAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,OAAO,CAAC;QACvB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KACrB,CAAC;IACF,YAAY,EAAE;QACZ,OAAO,EAAE;YACP,kCAAkC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACzE,kCAAkC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACzE,eAAe,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACtD,UAAU,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;SAClD,CAAC;QACF,oBAAoB,EAAE,OAAO,CAAC;QAC9B,eAAe,EAAE,MAAM,CAAC;QACxB,MAAM,EAAE;YACN,KAAK,EAAE,MAAM,CAAC;YACd,GAAG,EAAE,MAAM,CAAC;SACb,CAAC;QACF,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE;YACL,QAAQ,EAAE;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC;YAC7C,MAAM,EAAE;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC;YAC3C,MAAM,EAAE;gBACN,gBAAgB,EAAE,MAAM,CAAC;gBACzB,oBAAoB,EAAE,MAAM,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC;gBAChB,KAAK,EAAE,MAAM,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC;gBACd,SAAS,EAAE,MAAM,CAAC;aACnB,CAAC;SACH,CAAC;KACH,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,gBAAgB;IAC3B,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEV,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW;IAMjD,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,cAAc;CAOvB;AAED,qBAAa,gBAAgB;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;gBAEV,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;CAKpF"}
|
package/esm/result.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as errors from './errors.js';
|
|
2
2
|
import { cheerio } from './deps.js';
|
|
3
|
+
import { normalizeHeaders } from './utils.js';
|
|
3
4
|
export class ScrapeResult {
|
|
4
5
|
constructor(data) {
|
|
5
6
|
Object.defineProperty(this, "config", {
|
|
@@ -40,7 +41,8 @@ export class ScrapeResult {
|
|
|
40
41
|
}
|
|
41
42
|
get selector() {
|
|
42
43
|
if (!this._selector) {
|
|
43
|
-
|
|
44
|
+
const headers = normalizeHeaders(this.result.response_headers);
|
|
45
|
+
if (!headers['content-type'].includes('text/html')) {
|
|
44
46
|
throw new errors.ContentTypeError(`Cannot use selector on non-html content-type, received: ${this.result.response_headers['content-type']}`);
|
|
45
47
|
}
|
|
46
48
|
this._selector = cheerio.load(this.result.content);
|
|
@@ -73,19 +75,21 @@ export class ScreenshotResult {
|
|
|
73
75
|
this.result = this.decodeResponse(response, data); // raw result
|
|
74
76
|
}
|
|
75
77
|
defineMetadata(response) {
|
|
76
|
-
const
|
|
78
|
+
const headers = normalizeHeaders(response.headers);
|
|
79
|
+
const contentType = headers['content-type'];
|
|
77
80
|
let extension_name = '';
|
|
78
81
|
if (contentType) {
|
|
79
82
|
extension_name = contentType.split('/')[1].split(';')[0];
|
|
80
83
|
}
|
|
81
84
|
return {
|
|
82
85
|
extension_name: extension_name,
|
|
83
|
-
upstream_status_code: parseInt(
|
|
84
|
-
upstream_url:
|
|
86
|
+
upstream_status_code: parseInt(headers['x-scrapfly-upstream-http-code'] || '200', 10),
|
|
87
|
+
upstream_url: headers['x-scrapfly-upstream-url'] || '',
|
|
85
88
|
};
|
|
86
89
|
}
|
|
87
90
|
decodeResponse(response, data) {
|
|
88
|
-
|
|
91
|
+
const headers = normalizeHeaders(response.headers);
|
|
92
|
+
if (headers['content-type'] === 'json') {
|
|
89
93
|
return JSON.parse(new TextDecoder().decode(data));
|
|
90
94
|
}
|
|
91
95
|
return null;
|
package/esm/utils.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
import type { Rec } from './types.js';
|
|
2
2
|
export declare function urlsafe_b64encode(data: string): string;
|
|
3
3
|
export type RequestOptions = {
|
|
4
4
|
url: string;
|
|
@@ -7,4 +7,5 @@ export type RequestOptions = {
|
|
|
7
7
|
body?: string | Uint8Array;
|
|
8
8
|
};
|
|
9
9
|
export declare function fetchRetry(config: RequestOptions, retries?: number, retryDelay?: number): Promise<Response>;
|
|
10
|
+
export declare function normalizeHeaders(headers: Rec<string> | Headers): Rec<string>;
|
|
10
11
|
//# sourceMappingURL=utils.d.ts.map
|
package/esm/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;CAC5B,CAAC;AAEF,wBAAsB,UAAU,CAC9B,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,MAAU,EACnB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,QAAQ,CAAC,CA+BnB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,CAc5E"}
|
package/esm/utils.js
CHANGED
|
@@ -38,3 +38,17 @@ export async function fetchRetry(config, retries = 3, retryDelay = 1000) {
|
|
|
38
38
|
}
|
|
39
39
|
throw lastError;
|
|
40
40
|
}
|
|
41
|
+
export function normalizeHeaders(headers) {
|
|
42
|
+
const normalizedHeaders = {};
|
|
43
|
+
if (headers instanceof Headers) {
|
|
44
|
+
headers.forEach((value, key) => {
|
|
45
|
+
normalizedHeaders[key.toLowerCase()] = value;
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
Object.keys(headers).forEach((key) => {
|
|
50
|
+
normalizedHeaders[key.toLowerCase()] = headers[key];
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
return normalizedHeaders;
|
|
54
|
+
}
|
package/package.json
CHANGED
package/script/client.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
import { fetchRetry } from './utils.js';
|
|
3
2
|
import * as errors from './errors.js';
|
|
4
3
|
import type { ScrapeConfig } from './scrapeconfig.js';
|
|
@@ -17,6 +16,10 @@ export declare class ScrapflyClient {
|
|
|
17
16
|
* Raise appropriate error for given response and scrape result
|
|
18
17
|
*/
|
|
19
18
|
errResult(response: Response, result: ScrapeResult): errors.ScrapflyError;
|
|
19
|
+
/**
|
|
20
|
+
* Handle clob and blob large objects
|
|
21
|
+
*/
|
|
22
|
+
handleLargeObjects(result: any, format: 'clob' | 'blob'): Promise<ScrapeResult>;
|
|
20
23
|
/**
|
|
21
24
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
22
25
|
*/
|
package/script/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACG,kBAAkB,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAgCrF;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAkDzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
|
package/script/client.js
CHANGED
|
@@ -131,6 +131,39 @@ class ScrapflyClient {
|
|
|
131
131
|
}
|
|
132
132
|
return new errors.ScrapflyError(message, args);
|
|
133
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* Handle clob and blob large objects
|
|
136
|
+
*/
|
|
137
|
+
async handleLargeObjects(result, format) {
|
|
138
|
+
let response;
|
|
139
|
+
try {
|
|
140
|
+
const url = new URL(result.content);
|
|
141
|
+
const params = { key: this.key };
|
|
142
|
+
url.search = new URLSearchParams(params).toString();
|
|
143
|
+
response = await this.fetch({
|
|
144
|
+
url: url.toString(),
|
|
145
|
+
method: 'GET',
|
|
146
|
+
headers: {
|
|
147
|
+
'user-agent': this.ua,
|
|
148
|
+
'accept-encoding': 'gzip, deflate, br',
|
|
149
|
+
accept: 'application/json',
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
catch (e) {
|
|
154
|
+
logger_js_1.log.error('error', e);
|
|
155
|
+
throw e;
|
|
156
|
+
}
|
|
157
|
+
const content = await response.text();
|
|
158
|
+
result.content = content;
|
|
159
|
+
if (format === 'clob') {
|
|
160
|
+
result.format = 'text';
|
|
161
|
+
}
|
|
162
|
+
if (format === 'blob') {
|
|
163
|
+
result.format = 'binary';
|
|
164
|
+
}
|
|
165
|
+
return result;
|
|
166
|
+
}
|
|
134
167
|
/**
|
|
135
168
|
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
136
169
|
*/
|
|
@@ -212,6 +245,10 @@ class ScrapflyClient {
|
|
|
212
245
|
}
|
|
213
246
|
throw new errors.ApiHttpClientError(JSON.stringify(data));
|
|
214
247
|
}
|
|
248
|
+
const content_format = data.result.format;
|
|
249
|
+
if (content_format === 'clob' || content_format === 'blob') {
|
|
250
|
+
data.result = await this.handleLargeObjects(data.result, content_format);
|
|
251
|
+
}
|
|
215
252
|
const result = this.handleResponse(response, new result_js_1.ScrapeResult({
|
|
216
253
|
config: data.config,
|
|
217
254
|
context: data.context,
|
|
@@ -2,10 +2,9 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// Copyright the Browserify authors. MIT License.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.assertPath =
|
|
5
|
+
exports.assertPath = assertPath;
|
|
6
6
|
function assertPath(path) {
|
|
7
7
|
if (typeof path !== "string") {
|
|
8
8
|
throw new TypeError(`Path must be a string. Received ${JSON.stringify(path)}`);
|
|
9
9
|
}
|
|
10
10
|
}
|
|
11
|
-
exports.assertPath = assertPath;
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.
|
|
5
|
+
exports.stripSuffix = stripSuffix;
|
|
6
|
+
exports.lastPathSegment = lastPathSegment;
|
|
7
|
+
exports.assertArgs = assertArgs;
|
|
6
8
|
const assert_path_js_1 = require("./assert_path.js");
|
|
7
9
|
function stripSuffix(name, suffix) {
|
|
8
10
|
if (suffix.length >= name.length) {
|
|
@@ -16,7 +18,6 @@ function stripSuffix(name, suffix) {
|
|
|
16
18
|
}
|
|
17
19
|
return name.slice(0, -suffix.length);
|
|
18
20
|
}
|
|
19
|
-
exports.stripSuffix = stripSuffix;
|
|
20
21
|
function lastPathSegment(path, isSep, start = 0) {
|
|
21
22
|
let matchedNonSeparator = false;
|
|
22
23
|
let end = path.length;
|
|
@@ -34,7 +35,6 @@ function lastPathSegment(path, isSep, start = 0) {
|
|
|
34
35
|
}
|
|
35
36
|
return path.slice(start, end);
|
|
36
37
|
}
|
|
37
|
-
exports.lastPathSegment = lastPathSegment;
|
|
38
38
|
function assertArgs(path, suffix) {
|
|
39
39
|
(0, assert_path_js_1.assertPath)(path);
|
|
40
40
|
if (path.length === 0)
|
|
@@ -43,4 +43,3 @@ function assertArgs(path, suffix) {
|
|
|
43
43
|
throw new TypeError(`Suffix must be a string. Received ${JSON.stringify(suffix)}`);
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
|
-
exports.assertArgs = assertArgs;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.common =
|
|
5
|
+
exports.common = common;
|
|
6
6
|
function common(paths, sep) {
|
|
7
7
|
const [first = "", ...remaining] = paths;
|
|
8
8
|
const parts = first.split(sep);
|
|
@@ -24,4 +24,3 @@ function common(paths, sep) {
|
|
|
24
24
|
}
|
|
25
25
|
return parts.slice(0, endOfPrefix).join(sep) + append;
|
|
26
26
|
}
|
|
27
|
-
exports.common = common;
|
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.assertArg =
|
|
5
|
+
exports.assertArg = assertArg;
|
|
6
6
|
const assert_path_js_1 = require("./assert_path.js");
|
|
7
7
|
function assertArg(path) {
|
|
8
8
|
(0, assert_path_js_1.assertPath)(path);
|
|
9
9
|
if (path.length === 0)
|
|
10
10
|
return ".";
|
|
11
11
|
}
|
|
12
|
-
exports.assertArg = assertArg;
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.
|
|
5
|
+
exports._format = _format;
|
|
6
|
+
exports.assertArg = assertArg;
|
|
6
7
|
function _format(sep, pathObject) {
|
|
7
8
|
const dir = pathObject.dir || pathObject.root;
|
|
8
9
|
const base = pathObject.base ||
|
|
@@ -15,10 +16,8 @@ function _format(sep, pathObject) {
|
|
|
15
16
|
return dir + base;
|
|
16
17
|
return dir + sep + base;
|
|
17
18
|
}
|
|
18
|
-
exports._format = _format;
|
|
19
19
|
function assertArg(pathObject) {
|
|
20
20
|
if (pathObject === null || typeof pathObject !== "object") {
|
|
21
21
|
throw new TypeError(`The "pathObject" argument must be of type Object. Received type ${typeof pathObject}`);
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
|
-
exports.assertArg = assertArg;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"AAGA,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,OAM1C"}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.assertArg =
|
|
5
|
+
exports.assertArg = assertArg;
|
|
6
6
|
function assertArg(url) {
|
|
7
7
|
url = url instanceof URL ? url : new URL(url);
|
|
8
8
|
if (url.protocol !== "file:") {
|
|
@@ -10,4 +10,3 @@ function assertArg(url) {
|
|
|
10
10
|
}
|
|
11
11
|
return url;
|
|
12
12
|
}
|
|
13
|
-
exports.assertArg = assertArg;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports._globToRegExp =
|
|
5
|
+
exports._globToRegExp = _globToRegExp;
|
|
6
6
|
const REG_EXP_ESCAPE_CHARS = [
|
|
7
7
|
"!",
|
|
8
8
|
"$",
|
|
@@ -235,4 +235,3 @@ caseInsensitive = false, } = {}) {
|
|
|
235
235
|
regExpString = `^${regExpString}$`;
|
|
236
236
|
return new RegExp(regExpString, caseInsensitive ? "i" : "");
|
|
237
237
|
}
|
|
238
|
-
exports._globToRegExp = _globToRegExp;
|
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.assertArg =
|
|
5
|
+
exports.assertArg = assertArg;
|
|
6
6
|
const assert_path_js_1 = require("./assert_path.js");
|
|
7
7
|
function assertArg(path) {
|
|
8
8
|
(0, assert_path_js_1.assertPath)(path);
|
|
9
9
|
if (path.length === 0)
|
|
10
10
|
return ".";
|
|
11
11
|
}
|
|
12
|
-
exports.assertArg = assertArg;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// Ported from https://github.com/browserify/path-browserify/
|
|
5
5
|
// This module is browser compatible.
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.normalizeString =
|
|
7
|
+
exports.normalizeString = normalizeString;
|
|
8
8
|
const constants_js_1 = require("./constants.js");
|
|
9
9
|
// Resolves . and .. elements in a path with directory names
|
|
10
10
|
function normalizeString(path, allowAboveRoot, separator, isPathSeparator) {
|
|
@@ -78,4 +78,3 @@ function normalizeString(path, allowAboveRoot, separator, isPathSeparator) {
|
|
|
78
78
|
}
|
|
79
79
|
return res;
|
|
80
80
|
}
|
|
81
|
-
exports.normalizeString = normalizeString;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.assertArgs =
|
|
5
|
+
exports.assertArgs = assertArgs;
|
|
6
6
|
const assert_path_js_1 = require("./assert_path.js");
|
|
7
7
|
function assertArgs(from, to) {
|
|
8
8
|
(0, assert_path_js_1.assertPath)(from);
|
|
@@ -10,4 +10,3 @@ function assertArgs(from, to) {
|
|
|
10
10
|
if (from === to)
|
|
11
11
|
return "";
|
|
12
12
|
}
|
|
13
|
-
exports.assertArgs = assertArgs;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// Ported from https://github.com/browserify/path-browserify/
|
|
5
5
|
// This module is browser compatible.
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.stripTrailingSeparators =
|
|
7
|
+
exports.stripTrailingSeparators = stripTrailingSeparators;
|
|
8
8
|
function stripTrailingSeparators(segment, isSep) {
|
|
9
9
|
if (segment.length <= 1) {
|
|
10
10
|
return segment;
|
|
@@ -20,4 +20,3 @@ function stripTrailingSeparators(segment, isSep) {
|
|
|
20
20
|
}
|
|
21
21
|
return segment.slice(0, end);
|
|
22
22
|
}
|
|
23
|
-
exports.stripTrailingSeparators = stripTrailingSeparators;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.encodeWhitespace =
|
|
5
|
+
exports.encodeWhitespace = encodeWhitespace;
|
|
6
6
|
const WHITESPACE_ENCODINGS = {
|
|
7
7
|
"\u0009": "%09",
|
|
8
8
|
"\u000A": "%0A",
|
|
@@ -16,4 +16,3 @@ function encodeWhitespace(string) {
|
|
|
16
16
|
return WHITESPACE_ENCODINGS[c] ?? c;
|
|
17
17
|
});
|
|
18
18
|
}
|
|
19
|
-
exports.encodeWhitespace = encodeWhitespace;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
3
3
|
// This module is browser compatible.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.basename =
|
|
5
|
+
exports.basename = basename;
|
|
6
6
|
const _os_js_1 = require("./_os.js");
|
|
7
7
|
const basename_js_1 = require("./posix/basename.js");
|
|
8
8
|
const basename_js_2 = require("./windows/basename.js");
|
|
@@ -34,4 +34,3 @@ function basename(path, suffix = "") {
|
|
|
34
34
|
? (0, basename_js_2.basename)(path, suffix)
|
|
35
35
|
: (0, basename_js_1.basename)(path, suffix);
|
|
36
36
|
}
|
|
37
|
-
exports.basename = basename;
|