scrapfly-sdk 0.6.2 → 0.6.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/README.md +4 -2
  2. package/esm/client.d.ts +17 -14
  3. package/esm/client.d.ts.map +1 -1
  4. package/esm/client.js +63 -24
  5. package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts +0 -1
  6. package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts.map +1 -1
  7. package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts +0 -1
  8. package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts.map +1 -1
  9. package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts +0 -1
  10. package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts.map +1 -1
  11. package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts +0 -1
  12. package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts.map +1 -1
  13. package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts +0 -1
  14. package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts.map +1 -1
  15. package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts +0 -1
  16. package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts.map +1 -1
  17. package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts +0 -1
  18. package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts.map +1 -1
  19. package/esm/polyfill.d.ts +5 -0
  20. package/esm/polyfill.d.ts.map +1 -0
  21. package/esm/polyfill.js +20 -0
  22. package/esm/result.d.ts +0 -1
  23. package/esm/result.d.ts.map +1 -1
  24. package/esm/result.js +9 -5
  25. package/esm/utils.d.ts +9 -2
  26. package/esm/utils.d.ts.map +1 -1
  27. package/esm/utils.js +20 -2
  28. package/package.json +1 -1
  29. package/script/client.d.ts +17 -14
  30. package/script/client.d.ts.map +1 -1
  31. package/script/client.js +63 -24
  32. package/script/deps/jsr.io/@std/path/1.0.1/_common/assert_path.js +1 -2
  33. package/script/deps/jsr.io/@std/path/1.0.1/_common/basename.js +3 -4
  34. package/script/deps/jsr.io/@std/path/1.0.1/_common/common.js +1 -2
  35. package/script/deps/jsr.io/@std/path/1.0.1/_common/dirname.js +1 -2
  36. package/script/deps/jsr.io/@std/path/1.0.1/_common/format.js +2 -3
  37. package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts +0 -1
  38. package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts.map +1 -1
  39. package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.js +1 -2
  40. package/script/deps/jsr.io/@std/path/1.0.1/_common/glob_to_reg_exp.js +1 -2
  41. package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize.js +1 -2
  42. package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize_string.js +1 -2
  43. package/script/deps/jsr.io/@std/path/1.0.1/_common/relative.js +1 -2
  44. package/script/deps/jsr.io/@std/path/1.0.1/_common/strip_trailing_separators.js +1 -2
  45. package/script/deps/jsr.io/@std/path/1.0.1/_common/to_file_url.js +1 -2
  46. package/script/deps/jsr.io/@std/path/1.0.1/basename.js +1 -2
  47. package/script/deps/jsr.io/@std/path/1.0.1/common.js +1 -2
  48. package/script/deps/jsr.io/@std/path/1.0.1/dirname.js +1 -2
  49. package/script/deps/jsr.io/@std/path/1.0.1/extname.js +1 -2
  50. package/script/deps/jsr.io/@std/path/1.0.1/format.js +1 -2
  51. package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts +0 -1
  52. package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts.map +1 -1
  53. package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.js +1 -2
  54. package/script/deps/jsr.io/@std/path/1.0.1/glob_to_regexp.js +1 -2
  55. package/script/deps/jsr.io/@std/path/1.0.1/is_absolute.js +1 -2
  56. package/script/deps/jsr.io/@std/path/1.0.1/is_glob.js +1 -2
  57. package/script/deps/jsr.io/@std/path/1.0.1/join.js +1 -2
  58. package/script/deps/jsr.io/@std/path/1.0.1/join_globs.js +1 -2
  59. package/script/deps/jsr.io/@std/path/1.0.1/normalize.js +1 -2
  60. package/script/deps/jsr.io/@std/path/1.0.1/normalize_glob.js +1 -2
  61. package/script/deps/jsr.io/@std/path/1.0.1/parse.js +1 -2
  62. package/script/deps/jsr.io/@std/path/1.0.1/posix/_util.js +1 -2
  63. package/script/deps/jsr.io/@std/path/1.0.1/posix/basename.js +1 -2
  64. package/script/deps/jsr.io/@std/path/1.0.1/posix/dirname.js +1 -2
  65. package/script/deps/jsr.io/@std/path/1.0.1/posix/extname.js +1 -2
  66. package/script/deps/jsr.io/@std/path/1.0.1/posix/format.js +1 -2
  67. package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts +0 -1
  68. package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts.map +1 -1
  69. package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.js +1 -2
  70. package/script/deps/jsr.io/@std/path/1.0.1/posix/glob_to_regexp.js +1 -2
  71. package/script/deps/jsr.io/@std/path/1.0.1/posix/is_absolute.js +1 -2
  72. package/script/deps/jsr.io/@std/path/1.0.1/posix/join.js +1 -2
  73. package/script/deps/jsr.io/@std/path/1.0.1/posix/join_globs.js +1 -2
  74. package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize.js +1 -2
  75. package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize_glob.js +1 -2
  76. package/script/deps/jsr.io/@std/path/1.0.1/posix/parse.js +1 -2
  77. package/script/deps/jsr.io/@std/path/1.0.1/posix/relative.js +1 -2
  78. package/script/deps/jsr.io/@std/path/1.0.1/posix/resolve.js +1 -2
  79. package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts +0 -1
  80. package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts.map +1 -1
  81. package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.js +1 -2
  82. package/script/deps/jsr.io/@std/path/1.0.1/posix/to_namespaced_path.js +1 -2
  83. package/script/deps/jsr.io/@std/path/1.0.1/relative.js +1 -2
  84. package/script/deps/jsr.io/@std/path/1.0.1/resolve.js +1 -2
  85. package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts +0 -1
  86. package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts.map +1 -1
  87. package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.js +1 -2
  88. package/script/deps/jsr.io/@std/path/1.0.1/to_namespaced_path.js +1 -2
  89. package/script/deps/jsr.io/@std/path/1.0.1/windows/_util.js +3 -4
  90. package/script/deps/jsr.io/@std/path/1.0.1/windows/basename.js +1 -2
  91. package/script/deps/jsr.io/@std/path/1.0.1/windows/dirname.js +1 -2
  92. package/script/deps/jsr.io/@std/path/1.0.1/windows/extname.js +1 -2
  93. package/script/deps/jsr.io/@std/path/1.0.1/windows/format.js +1 -2
  94. package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts +0 -1
  95. package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts.map +1 -1
  96. package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.js +1 -2
  97. package/script/deps/jsr.io/@std/path/1.0.1/windows/glob_to_regexp.js +1 -2
  98. package/script/deps/jsr.io/@std/path/1.0.1/windows/is_absolute.js +1 -2
  99. package/script/deps/jsr.io/@std/path/1.0.1/windows/join.js +1 -2
  100. package/script/deps/jsr.io/@std/path/1.0.1/windows/join_globs.js +1 -2
  101. package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize.js +1 -2
  102. package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize_glob.js +1 -2
  103. package/script/deps/jsr.io/@std/path/1.0.1/windows/parse.js +1 -2
  104. package/script/deps/jsr.io/@std/path/1.0.1/windows/relative.js +1 -2
  105. package/script/deps/jsr.io/@std/path/1.0.1/windows/resolve.js +1 -2
  106. package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts +0 -1
  107. package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts.map +1 -1
  108. package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.js +1 -2
  109. package/script/deps/jsr.io/@std/path/1.0.1/windows/to_namespaced_path.js +1 -2
  110. package/script/polyfill.d.ts +5 -0
  111. package/script/polyfill.d.ts.map +1 -0
  112. package/script/polyfill.js +48 -0
  113. package/script/result.d.ts +0 -1
  114. package/script/result.d.ts.map +1 -1
  115. package/script/result.js +9 -5
  116. package/script/utils.d.ts +9 -2
  117. package/script/utils.d.ts.map +1 -1
  118. package/script/utils.js +23 -5
package/README.md CHANGED
@@ -24,8 +24,10 @@ The SDK is distributed through:
24
24
  3. Start scraping: 🚀
25
25
 
26
26
  ```javascript
27
- // node or bun:
27
+ // node
28
28
  import { ScrapflyClient, ScrapeConfig } from 'scrapfly-sdk';
29
+ // bun
30
+ import { ScrapflyClient, ScrapeConfig} from '@scrapfly/scrapfly-sdk';
29
31
  // deno:
30
32
  import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk';
31
33
 
@@ -97,7 +99,7 @@ $ deno lint
97
99
  # publish JSR:
98
100
  $ deno publish
99
101
  # build NPM package:
100
- $ deno build-npm
102
+ $ deno task build-npm
101
103
  # publish NPM:
102
104
  $ cd npm && npm publish
103
105
  ```
package/esm/client.d.ts CHANGED
@@ -1,4 +1,3 @@
1
- /// <reference types="node" />
2
1
  import { fetchRetry } from './utils.js';
3
2
  import * as errors from './errors.js';
4
3
  import type { ScrapeConfig } from './scrapeconfig.js';
@@ -17,6 +16,10 @@ export declare class ScrapflyClient {
17
16
  * Raise appropriate error for given response and scrape result
18
17
  */
19
18
  errResult(response: Response, result: ScrapeResult): errors.ScrapflyError;
19
+ /**
20
+ * Handle clob and blob large objects
21
+ */
22
+ handleLargeObjects(result: any, format: 'clob' | 'blob'): Promise<ScrapeResult>;
20
23
  /**
21
24
  * Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
22
25
  */
@@ -30,21 +33,21 @@ export declare class ScrapflyClient {
30
33
  */
31
34
  scrape(config: ScrapeConfig): Promise<ScrapeResult>;
32
35
  /**
33
- Concurrently scrape multiple configs
34
- This is a async generator call it like this:
36
+ Concurrently scrape multiple configs
37
+ This is a async generator call it like this:
35
38
 
36
- const results = [];
37
- const errors = [];
38
- for await (const resultOrError of client.concurrentScrape(configs)) {
39
- if (resultOrError instanceof Error) {
40
- errors.push(resultOrError);
41
- } else {
42
- results.push(resultOrError);
43
- }
44
- }
39
+ const results = [];
40
+ const errors = [];
41
+ for await (const resultOrError of client.concurrentScrape(configs)) {
42
+ if (resultOrError instanceof Error) {
43
+ errors.push(resultOrError);
44
+ } else {
45
+ results.push(resultOrError);
46
+ }
47
+ }
45
48
 
46
- @param concurrencyLimit: if not set it will be taken from your account info
47
- */
49
+ @param concurrencyLimit: if not set it will be taken from your account info
50
+ */
48
51
  concurrentScrape(configs: ScrapeConfig[], concurrencyLimit?: number): AsyncGenerator<ScrapeResult | Error | undefined, void, undefined>;
49
52
  /**
50
53
  * Save screenshot response to a file
@@ -1 +1 @@
1
- {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA+BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA6CzD;;;;;;;;;;;;;;;QAeI;IACG,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
1
+ {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACG,kBAAkB,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAgCrF;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAkDzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
package/esm/client.js CHANGED
@@ -1,5 +1,5 @@
1
- import * as dntShim from "./_dnt.shims.js";
2
1
  import { path } from './deps.js';
2
+ import { mkdir, writeFile } from './polyfill.js';
3
3
  import { fetchRetry } from './utils.js';
4
4
  import * as errors from './errors.js';
5
5
  import { ExtractionResult, ScrapeResult, ScreenshotResult } from './result.js';
@@ -105,6 +105,39 @@ export class ScrapflyClient {
105
105
  }
106
106
  return new errors.ScrapflyError(message, args);
107
107
  }
108
+ /**
109
+ * Handle clob and blob large objects
110
+ */
111
+ async handleLargeObjects(result, format) {
112
+ let response;
113
+ try {
114
+ const url = new URL(result.content);
115
+ const params = { key: this.key };
116
+ url.search = new URLSearchParams(params).toString();
117
+ response = await this.fetch({
118
+ url: url.toString(),
119
+ method: 'GET',
120
+ headers: {
121
+ 'user-agent': this.ua,
122
+ 'accept-encoding': 'gzip, deflate, br',
123
+ accept: 'application/json',
124
+ },
125
+ });
126
+ }
127
+ catch (e) {
128
+ log.error('error', e);
129
+ throw e;
130
+ }
131
+ const content = await response.text();
132
+ result.content = content;
133
+ if (format === 'clob') {
134
+ result.format = 'text';
135
+ }
136
+ if (format === 'blob') {
137
+ result.format = 'binary';
138
+ }
139
+ return result;
140
+ }
108
141
  /**
109
142
  * Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
110
143
  */
@@ -127,14 +160,15 @@ export class ScrapflyClient {
127
160
  const url = new URL(this.HOST + '/account');
128
161
  const params = { key: this.key };
129
162
  url.search = new URLSearchParams(params).toString();
130
- response = await this.fetch(new Request(url.toString(), {
163
+ response = await this.fetch({
164
+ url: url.toString(),
131
165
  method: 'GET',
132
166
  headers: {
133
167
  'user-agent': this.ua,
134
168
  'accept-encoding': 'gzip, deflate, br',
135
169
  accept: 'application/json',
136
170
  },
137
- }));
171
+ });
138
172
  }
139
173
  catch (e) {
140
174
  log.error('error', e);
@@ -159,7 +193,8 @@ export class ScrapflyClient {
159
193
  const url = new URL(this.HOST + '/scrape');
160
194
  const params = config.toApiParams({ key: this.key });
161
195
  url.search = new URLSearchParams(params).toString();
162
- response = await this.fetch(new Request(url.toString(), {
196
+ response = await this.fetch({
197
+ url: url.toString(),
163
198
  method: config.method,
164
199
  headers: {
165
200
  'user-agent': this.ua,
@@ -170,7 +205,7 @@ export class ScrapflyClient {
170
205
  accept: 'application/json',
171
206
  },
172
207
  body: config.body,
173
- }));
208
+ });
174
209
  }
175
210
  catch (e) {
176
211
  log.error('error', e);
@@ -184,6 +219,10 @@ export class ScrapflyClient {
184
219
  }
185
220
  throw new errors.ApiHttpClientError(JSON.stringify(data));
186
221
  }
222
+ const content_format = data.result.format;
223
+ if (content_format === 'clob' || content_format === 'blob') {
224
+ data.result = await this.handleLargeObjects(data.result, content_format);
225
+ }
187
226
  const result = this.handleResponse(response, new ScrapeResult({
188
227
  config: data.config,
189
228
  context: data.context,
@@ -193,21 +232,21 @@ export class ScrapflyClient {
193
232
  return result;
194
233
  }
195
234
  /**
196
- Concurrently scrape multiple configs
197
- This is a async generator call it like this:
235
+ Concurrently scrape multiple configs
236
+ This is a async generator call it like this:
198
237
 
199
- const results = [];
200
- const errors = [];
201
- for await (const resultOrError of client.concurrentScrape(configs)) {
202
- if (resultOrError instanceof Error) {
203
- errors.push(resultOrError);
204
- } else {
205
- results.push(resultOrError);
206
- }
207
- }
238
+ const results = [];
239
+ const errors = [];
240
+ for await (const resultOrError of client.concurrentScrape(configs)) {
241
+ if (resultOrError instanceof Error) {
242
+ errors.push(resultOrError);
243
+ } else {
244
+ results.push(resultOrError);
245
+ }
246
+ }
208
247
 
209
- @param concurrencyLimit: if not set it will be taken from your account info
210
- */
248
+ @param concurrencyLimit: if not set it will be taken from your account info
249
+ */
211
250
  async *concurrentScrape(configs, concurrencyLimit) {
212
251
  if (concurrencyLimit === undefined) {
213
252
  const account = await this.account();
@@ -250,7 +289,7 @@ export class ScrapflyClient {
250
289
  const extension_name = result.metadata.extension_name;
251
290
  let file_path;
252
291
  if (savePath) {
253
- dntShim.Deno.mkdir(savePath, { recursive: true });
292
+ await mkdir(savePath, { recursive: true });
254
293
  file_path = path.join(savePath, `${name}.${extension_name}`);
255
294
  }
256
295
  else {
@@ -258,7 +297,7 @@ export class ScrapflyClient {
258
297
  }
259
298
  const content = new Uint8Array(result.image);
260
299
  // Use Deno's write file method
261
- await dntShim.Deno.writeFile(file_path, content);
300
+ await writeFile(file_path, content);
262
301
  }
263
302
  /**
264
303
  * Turn scrapfly screenshot API response to ScreenshotResult or raise one of ScrapflyError
@@ -289,7 +328,8 @@ export class ScrapflyClient {
289
328
  const url = new URL(this.HOST + '/screenshot');
290
329
  const params = config.toApiParams({ key: this.key });
291
330
  url.search = new URLSearchParams(params).toString();
292
- const req = new Request(url.toString(), {
331
+ response = await this.fetch({
332
+ url: url.toString(),
293
333
  method: 'GET',
294
334
  headers: {
295
335
  'user-agent': this.ua,
@@ -297,7 +337,6 @@ export class ScrapflyClient {
297
337
  accept: 'application/json',
298
338
  },
299
339
  });
300
- response = await this.fetch(req);
301
340
  }
302
341
  catch (e) {
303
342
  log.error('error', e);
@@ -345,12 +384,12 @@ export class ScrapflyClient {
345
384
  if (config.document_compression_format && config.document_compression_format) {
346
385
  headers['content-encoding'] = config.document_compression_format;
347
386
  }
348
- const req = new Request(url.toString(), {
387
+ response = await this.fetch({
388
+ url: url.toString(),
349
389
  method: 'POST',
350
390
  headers: headers,
351
391
  body: config.body,
352
392
  });
353
- response = await this.fetch(req);
354
393
  }
355
394
  catch (e) {
356
395
  log.error('error', e);
@@ -1,3 +1,2 @@
1
- /// <reference types="node" />
2
1
  export declare function assertArg(url: URL | string): URL;
3
2
  //# sourceMappingURL=from_file_url.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":";AAGA,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,OAM1C"}
1
+ {"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.ts"],"names":[],"mappings":"AAGA,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,OAM1C"}
@@ -1,4 +1,3 @@
1
- /// <reference types="node" />
2
1
  /**
3
2
  * Converts a file URL to a path string.
4
3
  *
@@ -1 +1 @@
1
- {"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/from_file_url.ts"],"names":[],"mappings":";AAOA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,GAAG,MAAM,CAErD"}
1
+ {"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/from_file_url.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,GAAG,MAAM,CAErD"}
@@ -1,4 +1,3 @@
1
- /// <reference types="node" />
2
1
  /**
3
2
  * Converts a file URL to a path string.
4
3
  *
@@ -1 +1 @@
1
- {"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.ts"],"names":[],"mappings":";AAKA;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,MAAM,CAKrD"}
1
+ {"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,MAAM,CAKrD"}
@@ -1,4 +1,3 @@
1
- /// <reference types="node" />
2
1
  /**
3
2
  * Converts a path string to a file URL.
4
3
  *
@@ -1 +1 @@
1
- {"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.ts"],"names":[],"mappings":";AAMA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAU3C"}
1
+ {"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.ts"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAU3C"}
@@ -1,4 +1,3 @@
1
- /// <reference types="node" />
2
1
  /**
3
2
  * Converts a path string to a file URL.
4
3
  *
@@ -1 +1 @@
1
- {"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/to_file_url.ts"],"names":[],"mappings":";AAOA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAE3C"}
1
+ {"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../src/deps/jsr.io/@std/path/1.0.1/to_file_url.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAE3C"}
@@ -1,4 +1,3 @@
1
- /// <reference types="node" />
2
1
  /**
3
2
  * Converts a file URL to a path string.
4
3
  *
@@ -1 +1 @@
1
- {"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.ts"],"names":[],"mappings":";AAKA;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,MAAM,CAYrD"}
1
+ {"version":3,"file":"from_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,MAAM,CAYrD"}
@@ -1,4 +1,3 @@
1
- /// <reference types="node" />
2
1
  /**
3
2
  * Converts a path string to a file URL.
4
3
  *
@@ -1 +1 @@
1
- {"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.ts"],"names":[],"mappings":";AAMA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAgB3C"}
1
+ {"version":3,"file":"to_file_url.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.ts"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAgB3C"}
@@ -0,0 +1,5 @@
1
+ import * as dntShim from "./_dnt.shims.js";
2
+ export declare const isDeno: boolean;
3
+ export declare function mkdir(path: string | URL, options: dntShim.Deno.MkdirOptions): Promise<void>;
4
+ export declare function writeFile(path: string | URL, data: Uint8Array, options?: dntShim.Deno.WriteFileOptions): Promise<void>;
5
+ //# sourceMappingURL=polyfill.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"polyfill.d.ts","sourceRoot":"","sources":["../src/polyfill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,iBAAiB,CAAC;AAC3C,eAAO,MAAM,MAAM,SAAsC,CAAC;AAE1D,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjG;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,GAAG,GAAG,EAClB,IAAI,EAAE,UAAU,EAChB,OAAO,GAAE,OAAO,CAAC,IAAI,CAAC,gBAAqB,GAC1C,OAAO,CAAC,IAAI,CAAC,CAOf"}
@@ -0,0 +1,20 @@
1
+ import * as dntShim from "./_dnt.shims.js";
2
+ export const isDeno = typeof dntShim.Deno !== 'undefined';
3
+ export async function mkdir(path, options) {
4
+ if (isDeno) {
5
+ await dntShim.Deno.mkdir(path, options);
6
+ }
7
+ else {
8
+ // @ts-ignore: type for Bun
9
+ await Bun.mkdir(path.toString(), options);
10
+ }
11
+ }
12
+ export async function writeFile(path, data, options = {}) {
13
+ if (isDeno) {
14
+ await dntShim.Deno.writeFile(path, data, options);
15
+ }
16
+ else {
17
+ // @ts-ignore: type for Bun
18
+ await Bun.write(path.toString(), data, options);
19
+ }
20
+ }
package/esm/result.d.ts CHANGED
@@ -1,4 +1,3 @@
1
- /// <reference types="node" />
2
1
  import type { HttpMethod, Rec } from './types.js';
3
2
  import { cheerio } from './deps.js';
4
3
  export type ConfigData = {
@@ -1 +1 @@
1
- {"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;IACf,uBAAuB,EAAE,OAAO,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACrB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE;QACJ,OAAO,EAAE,KAAK,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACtE,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpB,EAAE,EAAE;QACF,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,GAAG,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,YAAY,EAAE;QACZ,4BAA4B,CAAC,EAAE,MAAM,CAAC;QACtC,WAAW,CAAC,EAAE;YACZ,QAAQ,EAAE,MAAM,CAAC;YACjB,QAAQ,EAAE,MAAM,CAAC;YACjB,QAAQ,EAAE,GAAG,CAAC;YACd,KAAK,EAAE,KAAK,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC;gBACf,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACpB,QAAQ,EAAE,MAAM,CAAC;gBACjB,QAAQ,EAAE,OAAO,CAAC;gBAClB,MAAM,CAAC,EAAE,MAAM,CAAC;gBAChB,OAAO,EAAE,OAAO,CAAC;aAClB,CAAC,CAAC;SACJ,CAAC;QACF,kBAAkB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,oBAAoB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnC,UAAU,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACxB,QAAQ,CAAC,EAAE,KAAK,CAAC;YACf,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YACrB,MAAM,EAAE,MAAM,CAAC;YACf,IAAI,EAAE,MAAM,CAAC;YACb,GAAG,EAAE,MAAM,CAAC;YACZ,QAAQ,EAAE;gBACR,IAAI,EAAE,MAAM,CAAC;gBACb,QAAQ,EAAE,MAAM,CAAC;gBACjB,MAAM,EAAE,MAAM,CAAC;gBACf,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM,EAAE,MAAM,CAAC;aAChB,CAAC;SACH,CAAC,CAAC;KACJ,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,OAAO,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;IACH,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE;YACH,WAAW,EAAE,MAAM,CAAC;YACpB,QAAQ,EAAE,MAAM,CAAC;YACjB,IAAI,EAAE,MAAM,CAAC;YACb,MAAM,EAAE,MAAM,CAAC;YACf,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;SACtB,CAAC;QACF,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,WAAW,EAAE,GAAG,CAAC;QACf,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE;QACJ,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;KACxB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,qBAAa,YAAY;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,WAAW,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,SAAS,CAAiC;gBAEtC,IAAI,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,OAAO,EAAE,WAAW,CAAC;QAAC,MAAM,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAQhG,IAAI,QAAQ,IAAI,OAAO,CAAC,UAAU,CAUjC;CACF;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC;AAEvE,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,OAAO,EAAE;QACP,iBAAiB,EAAE,OAAO,CAAC;QAC3B,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,YAAY,EAAE,GAAG,CAAC;QAClB,YAAY,EAAE,GAAG,CAAC;QAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,OAAO,CAAC;QACvB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KACrB,CAAC;IACF,YAAY,EAAE;QACZ,OAAO,EAAE;YACP,kCAAkC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACzE,kCAAkC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACzE,eAAe,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACtD,UAAU,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;SAClD,CAAC;QACF,oBAAoB,EAAE,OAAO,CAAC;QAC9B,eAAe,EAAE,MAAM,CAAC;QACxB,MAAM,EAAE;YACN,KAAK,EAAE,MAAM,CAAC;YACd,GAAG,EAAE,MAAM,CAAC;SACb,CAAC;QACF,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE;YACL,QAAQ,EAAE;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC;YAC7C,MAAM,EAAE;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC;YAC3C,MAAM,EAAE;gBACN,gBAAgB,EAAE,MAAM,CAAC;gBACzB,oBAAoB,EAAE,MAAM,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC;gBAChB,KAAK,EAAE,MAAM,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC;gBACd,SAAS,EAAE,MAAM,CAAC;aACnB,CAAC;SACH,CAAC;KACH,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,gBAAgB;IAC3B,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEV,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW;IAMjD,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,cAAc;CAMvB;AAED,qBAAa,gBAAgB;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;gBAEV,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;CAKpF"}
1
+ {"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;IACf,uBAAuB,EAAE,OAAO,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACrB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,WAAW,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE;QACJ,OAAO,EAAE,KAAK,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACtE,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpB,EAAE,EAAE;QACF,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,GAAG,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,YAAY,EAAE;QACZ,4BAA4B,CAAC,EAAE,MAAM,CAAC;QACtC,WAAW,CAAC,EAAE;YACZ,QAAQ,EAAE,MAAM,CAAC;YACjB,QAAQ,EAAE,MAAM,CAAC;YACjB,QAAQ,EAAE,GAAG,CAAC;YACd,KAAK,EAAE,KAAK,CAAC;gBACX,MAAM,EAAE,MAAM,CAAC;gBACf,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACpB,QAAQ,EAAE,MAAM,CAAC;gBACjB,QAAQ,EAAE,OAAO,CAAC;gBAClB,MAAM,CAAC,EAAE,MAAM,CAAC;gBAChB,OAAO,EAAE,OAAO,CAAC;aAClB,CAAC,CAAC;SACJ,CAAC;QACF,kBAAkB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,oBAAoB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnC,UAAU,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACxB,QAAQ,CAAC,EAAE,KAAK,CAAC;YACf,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YACrB,MAAM,EAAE,MAAM,CAAC;YACf,IAAI,EAAE,MAAM,CAAC;YACb,GAAG,EAAE,MAAM,CAAC;YACZ,QAAQ,EAAE;gBACR,IAAI,EAAE,MAAM,CAAC;gBACb,QAAQ,EAAE,MAAM,CAAC;gBACjB,MAAM,EAAE,MAAM,CAAC;gBACf,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM,EAAE,MAAM,CAAC;aAChB,CAAC;SACH,CAAC,CAAC;KACJ,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,OAAO,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;IACH,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE;YACH,WAAW,EAAE,MAAM,CAAC;YACpB,QAAQ,EAAE,MAAM,CAAC;YACjB,IAAI,EAAE,MAAM,CAAC;YACb,MAAM,EAAE,MAAM,CAAC;YACf,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;SACtB,CAAC;QACF,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,WAAW,EAAE,GAAG,CAAC;QACf,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE;QACJ,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;KACxB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,qBAAa,YAAY;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,WAAW,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,SAAS,CAAiC;gBAEtC,IAAI,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,OAAO,EAAE,WAAW,CAAC;QAAC,MAAM,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAQhG,IAAI,QAAQ,IAAI,OAAO,CAAC,UAAU,CAWjC;CACF;AAED,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC;AAEvE,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,OAAO,EAAE;QACP,iBAAiB,EAAE,OAAO,CAAC;QAC3B,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,YAAY,EAAE,GAAG,CAAC;QAClB,YAAY,EAAE,GAAG,CAAC;QAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,OAAO,CAAC;QACvB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KACrB,CAAC;IACF,YAAY,EAAE;QACZ,OAAO,EAAE;YACP,kCAAkC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACzE,kCAAkC,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACzE,eAAe,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACtD,UAAU,EAAE;gBAAE,QAAQ,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;SAClD,CAAC;QACF,oBAAoB,EAAE,OAAO,CAAC;QAC9B,eAAe,EAAE,MAAM,CAAC;QACxB,MAAM,EAAE;YACN,KAAK,EAAE,MAAM,CAAC;YACd,GAAG,EAAE,MAAM,CAAC;SACb,CAAC;QACF,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE;YACL,QAAQ,EAAE;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC;YAC7C,MAAM,EAAE;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC;YAC3C,MAAM,EAAE;gBACN,gBAAgB,EAAE,MAAM,CAAC;gBACzB,oBAAoB,EAAE,MAAM,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC;gBAChB,KAAK,EAAE,MAAM,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC;gBACd,SAAS,EAAE,MAAM,CAAC;aACnB,CAAC;SACH,CAAC;KACH,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,gBAAgB;IAC3B,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEV,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW;IAMjD,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,cAAc;CAOvB;AAED,qBAAa,gBAAgB;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;gBAEV,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;CAKpF"}
package/esm/result.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import * as errors from './errors.js';
2
2
  import { cheerio } from './deps.js';
3
+ import { normalizeHeaders } from './utils.js';
3
4
  export class ScrapeResult {
4
5
  constructor(data) {
5
6
  Object.defineProperty(this, "config", {
@@ -40,7 +41,8 @@ export class ScrapeResult {
40
41
  }
41
42
  get selector() {
42
43
  if (!this._selector) {
43
- if (!this.result.response_headers['content-type'].includes('text/html')) {
44
+ const headers = normalizeHeaders(this.result.response_headers);
45
+ if (!headers['content-type'].includes('text/html')) {
44
46
  throw new errors.ContentTypeError(`Cannot use selector on non-html content-type, received: ${this.result.response_headers['content-type']}`);
45
47
  }
46
48
  this._selector = cheerio.load(this.result.content);
@@ -73,19 +75,21 @@ export class ScreenshotResult {
73
75
  this.result = this.decodeResponse(response, data); // raw result
74
76
  }
75
77
  defineMetadata(response) {
76
- const contentType = response.headers.get('content-type');
78
+ const headers = normalizeHeaders(response.headers);
79
+ const contentType = headers['content-type'];
77
80
  let extension_name = '';
78
81
  if (contentType) {
79
82
  extension_name = contentType.split('/')[1].split(';')[0];
80
83
  }
81
84
  return {
82
85
  extension_name: extension_name,
83
- upstream_status_code: parseInt(response.headers.get('X-Scrapfly-Upstream-Http-Code') || '200', 10),
84
- upstream_url: response.headers.get('X-Scrapfly-Upstream-Url') || '',
86
+ upstream_status_code: parseInt(headers['x-scrapfly-upstream-http-code'] || '200', 10),
87
+ upstream_url: headers['x-scrapfly-upstream-url'] || '',
85
88
  };
86
89
  }
87
90
  decodeResponse(response, data) {
88
- if (response.headers.get('content-type') === 'json') {
91
+ const headers = normalizeHeaders(response.headers);
92
+ if (headers['content-type'] === 'json') {
89
93
  return JSON.parse(new TextDecoder().decode(data));
90
94
  }
91
95
  return null;
package/esm/utils.d.ts CHANGED
@@ -1,4 +1,11 @@
1
- /// <reference types="node" />
1
+ import type { Rec } from './types.js';
2
2
  export declare function urlsafe_b64encode(data: string): string;
3
- export declare function fetchRetry(config: Request, retries?: number, retryDelay?: number): Promise<Response>;
3
+ export type RequestOptions = {
4
+ url: string;
5
+ method?: string;
6
+ headers?: any;
7
+ body?: string | Uint8Array;
8
+ };
9
+ export declare function fetchRetry(config: RequestOptions, retries?: number, retryDelay?: number): Promise<Response>;
10
+ export declare function normalizeHeaders(headers: Rec<string> | Headers): Rec<string>;
4
11
  //# sourceMappingURL=utils.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAAA,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,wBAAsB,UAAU,CAC9B,MAAM,EAAE,OAAO,EACf,OAAO,GAAE,MAAU,EACnB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,QAAQ,CAAC,CA4BnB"}
1
+ {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;CAC5B,CAAC;AAEF,wBAAsB,UAAU,CAC9B,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,MAAU,EACnB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,QAAQ,CAAC,CA+BnB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,CAc5E"}
package/esm/utils.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { log } from './logger.js';
1
2
  export function urlsafe_b64encode(data) {
2
3
  const encoder = new TextEncoder();
3
4
  const encoded = encoder.encode(data);
@@ -11,12 +12,15 @@ export async function fetchRetry(config, retries = 3, retryDelay = 1000) {
11
12
  let lastError = null;
12
13
  for (let attempt = 1; attempt <= retries; attempt++) {
13
14
  try {
14
- // XXX: this breaks cloudflare workers as they don't support init options
15
- const response = await fetch(config);
15
+ // XXX: note that cloudflare workers don't support init options
16
+ const { url, ...reqInit } = config;
17
+ const response = await fetch(new Request(url, reqInit));
16
18
  // retry 5xx status codes
17
19
  if (response.status >= 500 && response.status < 600) {
20
+ const _text = await response.text(); // consume response to prevent leak
18
21
  lastError = new Error(`Fetch failed with status: ${response.status}`);
19
22
  if (attempt < retries) {
23
+ log.debug(`request failed ${response.status} (${_text}): retry ${attempt}/${retries} after ${retryDelay}ms`);
20
24
  await new Promise((resolve) => setTimeout(resolve, retryDelay));
21
25
  }
22
26
  }
@@ -34,3 +38,17 @@ export async function fetchRetry(config, retries = 3, retryDelay = 1000) {
34
38
  }
35
39
  throw lastError;
36
40
  }
41
+ export function normalizeHeaders(headers) {
42
+ const normalizedHeaders = {};
43
+ if (headers instanceof Headers) {
44
+ headers.forEach((value, key) => {
45
+ normalizedHeaders[key.toLowerCase()] = value;
46
+ });
47
+ }
48
+ else {
49
+ Object.keys(headers).forEach((key) => {
50
+ normalizedHeaders[key.toLowerCase()] = headers[key];
51
+ });
52
+ }
53
+ return normalizedHeaders;
54
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "scrapfly-sdk",
3
- "version": "0.6.2",
3
+ "version": "0.6.5",
4
4
  "description": "SDK for Scrapfly.io API for web scraping, screenshotting and data extraction",
5
5
  "keywords": [
6
6
  "web scraping",
@@ -1,4 +1,3 @@
1
- /// <reference types="node" />
2
1
  import { fetchRetry } from './utils.js';
3
2
  import * as errors from './errors.js';
4
3
  import type { ScrapeConfig } from './scrapeconfig.js';
@@ -17,6 +16,10 @@ export declare class ScrapflyClient {
17
16
  * Raise appropriate error for given response and scrape result
18
17
  */
19
18
  errResult(response: Response, result: ScrapeResult): errors.ScrapflyError;
19
+ /**
20
+ * Handle clob and blob large objects
21
+ */
22
+ handleLargeObjects(result: any, format: 'clob' | 'blob'): Promise<ScrapeResult>;
20
23
  /**
21
24
  * Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
22
25
  */
@@ -30,21 +33,21 @@ export declare class ScrapflyClient {
30
33
  */
31
34
  scrape(config: ScrapeConfig): Promise<ScrapeResult>;
32
35
  /**
33
- Concurrently scrape multiple configs
34
- This is a async generator call it like this:
36
+ Concurrently scrape multiple configs
37
+ This is a async generator call it like this:
35
38
 
36
- const results = [];
37
- const errors = [];
38
- for await (const resultOrError of client.concurrentScrape(configs)) {
39
- if (resultOrError instanceof Error) {
40
- errors.push(resultOrError);
41
- } else {
42
- results.push(resultOrError);
43
- }
44
- }
39
+ const results = [];
40
+ const errors = [];
41
+ for await (const resultOrError of client.concurrentScrape(configs)) {
42
+ if (resultOrError instanceof Error) {
43
+ errors.push(resultOrError);
44
+ } else {
45
+ results.push(resultOrError);
46
+ }
47
+ }
45
48
 
46
- @param concurrencyLimit: if not set it will be taken from your account info
47
- */
49
+ @param concurrencyLimit: if not set it will be taken from your account info
50
+ */
48
51
  concurrentScrape(configs: ScrapeConfig[], concurrencyLimit?: number): AsyncGenerator<ScrapeResult | Error | undefined, void, undefined>;
49
52
  /**
50
53
  * Save screenshot response to a file
@@ -1 +1 @@
1
- {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA+BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA6CzD;;;;;;;;;;;;;;;QAeI;IACG,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
1
+ {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACG,kBAAkB,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAgCrF;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAkDzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}