scrapfly-sdk 0.5.1 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -14
- package/esm/_dnt.shims.d.ts +6 -0
- package/esm/_dnt.shims.d.ts.map +1 -0
- package/esm/_dnt.shims.js +61 -0
- package/esm/client.d.ts +70 -0
- package/esm/client.d.ts.map +1 -0
- package/esm/client.js +364 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/assert_path.d.ts +2 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/assert_path.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/assert_path.js +7 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/basename.d.ts +4 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/basename.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/basename.js +40 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/common.d.ts +2 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/common.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/common.js +23 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/constants.d.ts +40 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/constants.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/constants.js +46 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/dirname.d.ts +2 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/dirname.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/dirname.js +8 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/format.d.ts +4 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/format.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/format.js +19 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts +3 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.js +9 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/glob_to_reg_exp.d.ts +35 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/glob_to_reg_exp.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/glob_to_reg_exp.js +234 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/normalize.d.ts +2 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/normalize.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/normalize.js +8 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/normalize_string.d.ts +2 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/normalize_string.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/normalize_string.js +77 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/relative.d.ts +2 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/relative.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/relative.js +9 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/strip_trailing_separators.d.ts +2 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/strip_trailing_separators.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/strip_trailing_separators.js +19 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/to_file_url.d.ts +2 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/to_file_url.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_common/to_file_url.js +15 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_os.d.ts +2 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_os.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/_os.js +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/basename.d.ts +25 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/basename.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/basename.js +33 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/common.d.ts +28 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/common.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/common.js +33 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/constants.d.ts +15 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/constants.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/constants.js +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/dirname.d.ts +20 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/dirname.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/dirname.js +26 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/extname.d.ts +20 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/extname.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/extname.js +26 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/format.d.ts +22 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/format.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/format.js +27 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts +23 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/from_file_url.js +28 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/glob_to_regexp.d.ts +77 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/glob_to_regexp.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/glob_to_regexp.js +83 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/is_absolute.d.ts +22 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/is_absolute.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/is_absolute.js +28 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/is_glob.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/is_glob.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/is_glob.js +42 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/join.d.ts +20 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/join.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/join.js +26 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/join_globs.d.ts +28 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/join_globs.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/join_globs.js +34 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/mod.d.ts +51 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/mod.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/mod.js +54 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/normalize.d.ts +24 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/normalize.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/normalize.js +30 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/normalize_glob.d.ts +29 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/normalize_glob.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/normalize_glob.js +35 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/parse.d.ts +35 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/parse.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/parse.js +39 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/_util.d.ts +2 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/_util.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/_util.js +8 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/basename.d.ts +20 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/basename.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/basename.js +29 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/constants.d.ts +13 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/constants.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/constants.js +14 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/dirname.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/dirname.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/dirname.js +47 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/extname.d.ts +18 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/extname.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/extname.js +70 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/format.d.ts +24 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/format.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/format.js +28 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.js +21 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/glob_to_regexp.d.ts +72 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/glob_to_regexp.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/glob_to_regexp.js +82 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/is_absolute.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/is_absolute.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/is_absolute.js +23 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/join.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/join.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/join.js +26 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/join_globs.d.ts +20 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/join_globs.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/join_globs.js +39 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/normalize.d.ts +19 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/normalize.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/normalize.js +36 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/normalize_glob.d.ts +20 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/normalize_glob.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/normalize_glob.js +31 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/parse.d.ts +25 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/parse.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/parse.js +115 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/relative.d.ts +20 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/relative.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/relative.js +106 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/resolve.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/resolve.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/resolve.js +59 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts +18 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.js +27 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_namespaced_path.d.ts +16 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_namespaced_path.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/posix/to_namespaced_path.js +20 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/relative.d.ts +24 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/relative.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/relative.js +30 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/resolve.d.ts +22 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/resolve.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/resolve.js +30 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts +23 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/to_file_url.js +28 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/to_namespaced_path.d.ts +21 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/to_namespaced_path.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/to_namespaced_path.js +29 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/types.d.ts +38 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/types.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/types.js +3 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/_util.d.ts +4 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/_util.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/_util.js +15 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/basename.d.ts +20 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/basename.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/basename.js +41 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/constants.d.ts +13 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/constants.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/constants.js +14 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/dirname.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/dirname.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/dirname.js +109 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/extname.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/extname.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/extname.js +78 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/format.d.ts +24 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/format.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/format.js +28 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts +19 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.js +30 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/glob_to_regexp.d.ts +71 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/glob_to_regexp.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/glob_to_regexp.js +82 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/is_absolute.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/is_absolute.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/is_absolute.js +38 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/join.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/join.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/join.js +71 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/join_globs.d.ts +21 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/join_globs.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/join_globs.js +40 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/normalize.d.ts +19 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/normalize.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/normalize.js +131 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/normalize_glob.d.ts +20 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/normalize_glob.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/normalize_glob.js +31 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/parse.d.ts +25 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/parse.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/parse.js +176 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/relative.d.ts +23 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/relative.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/relative.js +131 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/resolve.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/resolve.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/resolve.js +154 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts +18 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.js +34 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_namespaced_path.d.ts +17 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_namespaced_path.d.ts.map +1 -0
- package/esm/deps/jsr.io/@std/path/1.0.1/windows/to_namespaced_path.js +49 -0
- package/esm/deps.d.ts +4 -0
- package/esm/deps.d.ts.map +1 -0
- package/esm/deps.js +3 -0
- package/{build/src → esm}/errors.d.ts +9 -0
- package/esm/errors.d.ts.map +1 -0
- package/{build/src → esm}/errors.js +19 -3
- package/esm/extractionconfig.d.ts +47 -0
- package/esm/extractionconfig.d.ts.map +1 -0
- package/esm/extractionconfig.js +156 -0
- package/{build/src → esm}/logger.d.ts +1 -0
- package/esm/logger.d.ts.map +1 -0
- package/{build/src → esm}/logger.js +12 -3
- package/esm/main.d.ts +9 -0
- package/esm/main.d.ts.map +1 -0
- package/esm/main.js +7 -0
- package/esm/package.json +3 -0
- package/esm/polyfill.d.ts +6 -0
- package/esm/polyfill.d.ts.map +1 -0
- package/esm/polyfill.js +20 -0
- package/{build/src → esm}/result.d.ts +30 -4
- package/esm/result.d.ts.map +1 -0
- package/esm/result.js +118 -0
- package/esm/scrapeconfig.d.ts +120 -0
- package/esm/scrapeconfig.d.ts.map +1 -0
- package/esm/scrapeconfig.js +505 -0
- package/esm/screenshotconfig.d.ts +69 -0
- package/esm/screenshotconfig.d.ts.map +1 -0
- package/esm/screenshotconfig.js +220 -0
- package/{build/src → esm}/types.d.ts +1 -0
- package/esm/types.d.ts.map +1 -0
- package/esm/types.js +1 -0
- package/esm/utils.d.ts +10 -0
- package/esm/utils.d.ts.map +1 -0
- package/esm/utils.js +40 -0
- package/package.json +36 -61
- package/script/_dnt.shims.d.ts +6 -0
- package/script/_dnt.shims.d.ts.map +1 -0
- package/script/_dnt.shims.js +65 -0
- package/script/client.d.ts +70 -0
- package/script/client.d.ts.map +1 -0
- package/script/client.js +391 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/assert_path.d.ts +2 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/assert_path.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/assert_path.js +11 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/basename.d.ts +4 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/basename.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/basename.js +46 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/common.d.ts +2 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/common.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/common.js +27 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/constants.d.ts +40 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/constants.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/constants.js +49 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/dirname.d.ts +2 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/dirname.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/dirname.js +12 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/format.d.ts +4 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/format.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/format.js +24 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts +3 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/from_file_url.js +13 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/glob_to_reg_exp.d.ts +35 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/glob_to_reg_exp.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/glob_to_reg_exp.js +238 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize.d.ts +2 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize.js +12 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize_string.d.ts +2 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize_string.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/normalize_string.js +81 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/relative.d.ts +2 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/relative.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/relative.js +13 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/strip_trailing_separators.d.ts +2 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/strip_trailing_separators.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/strip_trailing_separators.js +23 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/to_file_url.d.ts +2 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/to_file_url.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_common/to_file_url.js +19 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_os.d.ts +2 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_os.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/_os.js +43 -0
- package/script/deps/jsr.io/@std/path/1.0.1/basename.d.ts +25 -0
- package/script/deps/jsr.io/@std/path/1.0.1/basename.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/basename.js +37 -0
- package/script/deps/jsr.io/@std/path/1.0.1/common.d.ts +28 -0
- package/script/deps/jsr.io/@std/path/1.0.1/common.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/common.js +37 -0
- package/script/deps/jsr.io/@std/path/1.0.1/constants.d.ts +15 -0
- package/script/deps/jsr.io/@std/path/1.0.1/constants.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/constants.js +20 -0
- package/script/deps/jsr.io/@std/path/1.0.1/dirname.d.ts +20 -0
- package/script/deps/jsr.io/@std/path/1.0.1/dirname.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/dirname.js +30 -0
- package/script/deps/jsr.io/@std/path/1.0.1/extname.d.ts +20 -0
- package/script/deps/jsr.io/@std/path/1.0.1/extname.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/extname.js +30 -0
- package/script/deps/jsr.io/@std/path/1.0.1/format.d.ts +22 -0
- package/script/deps/jsr.io/@std/path/1.0.1/format.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/format.js +31 -0
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts +23 -0
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/from_file_url.js +32 -0
- package/script/deps/jsr.io/@std/path/1.0.1/glob_to_regexp.d.ts +77 -0
- package/script/deps/jsr.io/@std/path/1.0.1/glob_to_regexp.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/glob_to_regexp.js +87 -0
- package/script/deps/jsr.io/@std/path/1.0.1/is_absolute.d.ts +22 -0
- package/script/deps/jsr.io/@std/path/1.0.1/is_absolute.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/is_absolute.js +32 -0
- package/script/deps/jsr.io/@std/path/1.0.1/is_glob.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/is_glob.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/is_glob.js +46 -0
- package/script/deps/jsr.io/@std/path/1.0.1/join.d.ts +20 -0
- package/script/deps/jsr.io/@std/path/1.0.1/join.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/join.js +30 -0
- package/script/deps/jsr.io/@std/path/1.0.1/join_globs.d.ts +28 -0
- package/script/deps/jsr.io/@std/path/1.0.1/join_globs.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/join_globs.js +38 -0
- package/script/deps/jsr.io/@std/path/1.0.1/mod.d.ts +51 -0
- package/script/deps/jsr.io/@std/path/1.0.1/mod.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/mod.js +70 -0
- package/script/deps/jsr.io/@std/path/1.0.1/normalize.d.ts +24 -0
- package/script/deps/jsr.io/@std/path/1.0.1/normalize.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/normalize.js +34 -0
- package/script/deps/jsr.io/@std/path/1.0.1/normalize_glob.d.ts +29 -0
- package/script/deps/jsr.io/@std/path/1.0.1/normalize_glob.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/normalize_glob.js +39 -0
- package/script/deps/jsr.io/@std/path/1.0.1/parse.d.ts +35 -0
- package/script/deps/jsr.io/@std/path/1.0.1/parse.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/parse.js +43 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/_util.d.ts +2 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/_util.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/_util.js +12 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/basename.d.ts +20 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/basename.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/basename.js +33 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/constants.d.ts +13 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/constants.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/constants.js +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/dirname.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/dirname.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/dirname.js +51 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/extname.d.ts +18 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/extname.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/extname.js +74 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/format.d.ts +24 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/format.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/format.js +32 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/from_file_url.js +25 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/glob_to_regexp.d.ts +72 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/glob_to_regexp.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/glob_to_regexp.js +86 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/is_absolute.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/is_absolute.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/is_absolute.js +27 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join.js +30 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join_globs.d.ts +20 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join_globs.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/join_globs.js +43 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize.d.ts +19 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize.js +40 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize_glob.d.ts +20 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize_glob.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/normalize_glob.js +35 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/parse.d.ts +25 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/parse.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/parse.js +119 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/relative.d.ts +20 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/relative.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/relative.js +110 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/resolve.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/resolve.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/resolve.js +86 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts +18 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_file_url.js +31 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_namespaced_path.d.ts +16 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_namespaced_path.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/posix/to_namespaced_path.js +24 -0
- package/script/deps/jsr.io/@std/path/1.0.1/relative.d.ts +24 -0
- package/script/deps/jsr.io/@std/path/1.0.1/relative.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/relative.js +34 -0
- package/script/deps/jsr.io/@std/path/1.0.1/resolve.d.ts +22 -0
- package/script/deps/jsr.io/@std/path/1.0.1/resolve.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/resolve.js +34 -0
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts +23 -0
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/to_file_url.js +32 -0
- package/script/deps/jsr.io/@std/path/1.0.1/to_namespaced_path.d.ts +21 -0
- package/script/deps/jsr.io/@std/path/1.0.1/to_namespaced_path.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/to_namespaced_path.js +33 -0
- package/script/deps/jsr.io/@std/path/1.0.1/types.d.ts +38 -0
- package/script/deps/jsr.io/@std/path/1.0.1/types.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/types.js +4 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/_util.d.ts +4 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/_util.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/_util.js +21 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/basename.d.ts +20 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/basename.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/basename.js +45 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/constants.d.ts +13 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/constants.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/constants.js +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/dirname.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/dirname.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/dirname.js +113 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/extname.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/extname.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/extname.js +82 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/format.d.ts +24 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/format.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/format.js +32 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts +19 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/from_file_url.js +34 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/glob_to_regexp.d.ts +71 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/glob_to_regexp.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/glob_to_regexp.js +86 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/is_absolute.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/is_absolute.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/is_absolute.js +42 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join.js +75 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join_globs.d.ts +21 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join_globs.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/join_globs.js +44 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize.d.ts +19 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize.js +135 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize_glob.d.ts +20 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize_glob.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/normalize_glob.js +35 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/parse.d.ts +25 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/parse.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/parse.js +180 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/relative.d.ts +23 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/relative.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/relative.js +135 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/resolve.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/resolve.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/resolve.js +181 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts +18 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_file_url.js +38 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_namespaced_path.d.ts +17 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_namespaced_path.d.ts.map +1 -0
- package/script/deps/jsr.io/@std/path/1.0.1/windows/to_namespaced_path.js +53 -0
- package/script/deps.d.ts +4 -0
- package/script/deps.d.ts.map +1 -0
- package/script/deps.js +30 -0
- package/script/errors.d.ts +53 -0
- package/script/errors.d.ts.map +1 -0
- package/script/errors.js +102 -0
- package/script/extractionconfig.d.ts +47 -0
- package/script/extractionconfig.d.ts.map +1 -0
- package/script/extractionconfig.js +183 -0
- package/script/logger.d.ts +19 -0
- package/script/logger.d.ts.map +1 -0
- package/script/logger.js +58 -0
- package/script/main.d.ts +9 -0
- package/script/main.d.ts.map +1 -0
- package/script/main.js +46 -0
- package/script/package.json +3 -0
- package/script/polyfill.d.ts +6 -0
- package/script/polyfill.d.ts.map +1 -0
- package/script/polyfill.js +48 -0
- package/script/result.d.ts +297 -0
- package/script/result.d.ts.map +1 -0
- package/script/result.js +147 -0
- package/script/scrapeconfig.d.ts +120 -0
- package/script/scrapeconfig.d.ts.map +1 -0
- package/script/scrapeconfig.js +509 -0
- package/script/screenshotconfig.d.ts +69 -0
- package/script/screenshotconfig.d.ts.map +1 -0
- package/script/screenshotconfig.js +224 -0
- package/script/types.d.ts +3 -0
- package/script/types.d.ts.map +1 -0
- package/script/types.js +2 -0
- package/script/utils.d.ts +10 -0
- package/script/utils.d.ts.map +1 -0
- package/script/utils.js +45 -0
- package/build/src/client.d.ts +0 -45
- package/build/src/client.js +0 -199
- package/build/src/client.js.map +0 -1
- package/build/src/errors.js.map +0 -1
- package/build/src/logger.js.map +0 -1
- package/build/src/main.d.ts +0 -5
- package/build/src/main.js +0 -6
- package/build/src/main.js.map +0 -1
- package/build/src/result.js +0 -30
- package/build/src/result.js.map +0 -1
- package/build/src/scrapeconfig.d.ts +0 -85
- package/build/src/scrapeconfig.js +0 -257
- package/build/src/scrapeconfig.js.map +0 -1
- package/build/src/types.js +0 -2
- package/build/src/types.js.map +0 -1
- package/build/src/utils.d.ts +0 -1
- package/build/src/utils.js +0 -7
- package/build/src/utils.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
# Scrapfly SDK
|
|
2
2
|
|
|
3
|
-
`npm install scrapfly-sdk`
|
|
3
|
+
`npm install scrapfly-sdk`
|
|
4
|
+
`deno add jsr:@scrapfly/scrapfly-sdk`
|
|
5
|
+
`bun jsr add @scrapfly/scrapfly-sdk`
|
|
4
6
|
|
|
5
|
-
Typescript/
|
|
7
|
+
Typescript/Javascript SDK for [Scrapfly.io](https://scrapfly.io/) web scraping API which allows to:
|
|
6
8
|
|
|
7
9
|
- Scrape the web without being blocked.
|
|
8
10
|
- Use headless browsers to access Javascript-powered page data.
|
|
@@ -11,6 +13,10 @@ Typescript/NodeJS SDK for [Scrapfly.io](https://scrapfly.io/) web scraping API w
|
|
|
11
13
|
|
|
12
14
|
For web scraping guides see [our blog](https://scrapfly.io/blog/) and [#scrapeguide](https://scrapfly.io/blog/tag/scrapeguide/) tag for how to scrape specific targets.
|
|
13
15
|
|
|
16
|
+
The SDK is distributed through:
|
|
17
|
+
- [npmjs.com/package/scrapfly-sdk](https://www.npmjs.com/package/scrapfly-sdk)
|
|
18
|
+
- [jsr.io/@scrapfly/scrapfly-sdk](https://jsr.io/@scrapfly/scrapfly-sdk)
|
|
19
|
+
|
|
14
20
|
## Quick Intro
|
|
15
21
|
|
|
16
22
|
1. Register a [Scrapfly account for free](https://scrapfly.io/register)
|
|
@@ -18,7 +24,12 @@ For web scraping guides see [our blog](https://scrapfly.io/blog/) and [#scrapegu
|
|
|
18
24
|
3. Start scraping: 🚀
|
|
19
25
|
|
|
20
26
|
```javascript
|
|
27
|
+
// node
|
|
21
28
|
import { ScrapflyClient, ScrapeConfig } from 'scrapfly-sdk';
|
|
29
|
+
// bun
|
|
30
|
+
import { ScrapflyClient, ScrapeConfig} from '@scrapfly/scrapfly-sdk';
|
|
31
|
+
// deno:
|
|
32
|
+
import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk';
|
|
22
33
|
|
|
23
34
|
const key = 'YOUR SCRAPFLY KEY';
|
|
24
35
|
const client = new ScrapflyClient({ key });
|
|
@@ -70,15 +81,25 @@ new ScrapeConfig({
|
|
|
70
81
|
|
|
71
82
|
## Development
|
|
72
83
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
```
|
|
82
|
-
|
|
83
|
-
$
|
|
84
|
-
|
|
84
|
+
This is a Deno Typescript project that builds to NPM through [DNT](https://github.com/denoland/dnt).
|
|
85
|
+
|
|
86
|
+
- `/src` directory contains all of the source code with `main.ts` being the entry point.
|
|
87
|
+
- `__tests__` directory contains tests for the source code.
|
|
88
|
+
- `deno.json` contains meta information
|
|
89
|
+
- `build.ts` is the build script that builds the project to nodejs ESM package.
|
|
90
|
+
- `/npm` directory will be produced when `built.ts` is executed for building node package.
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
# make modifications and run tests
|
|
94
|
+
$ deno task test
|
|
95
|
+
# format
|
|
96
|
+
$ deno fmt
|
|
97
|
+
# lint
|
|
98
|
+
$ deno lint
|
|
99
|
+
# publish JSR:
|
|
100
|
+
$ deno publish
|
|
101
|
+
# build NPM package:
|
|
102
|
+
$ deno build-npm
|
|
103
|
+
# publish NPM:
|
|
104
|
+
$ cd npm && npm publish
|
|
105
|
+
```
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"_dnt.shims.d.ts","sourceRoot":"","sources":["../src/_dnt.shims.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAKvC,eAAO,MAAM,aAAa;;CAA2C,CAAC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { Deno } from "@deno/shim-deno";
|
|
2
|
+
export { Deno } from "@deno/shim-deno";
|
|
3
|
+
const dntGlobals = {
|
|
4
|
+
Deno,
|
|
5
|
+
};
|
|
6
|
+
export const dntGlobalThis = createMergeProxy(globalThis, dntGlobals);
|
|
7
|
+
function createMergeProxy(baseObj, extObj) {
|
|
8
|
+
return new Proxy(baseObj, {
|
|
9
|
+
get(_target, prop, _receiver) {
|
|
10
|
+
if (prop in extObj) {
|
|
11
|
+
return extObj[prop];
|
|
12
|
+
}
|
|
13
|
+
else {
|
|
14
|
+
return baseObj[prop];
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
set(_target, prop, value) {
|
|
18
|
+
if (prop in extObj) {
|
|
19
|
+
delete extObj[prop];
|
|
20
|
+
}
|
|
21
|
+
baseObj[prop] = value;
|
|
22
|
+
return true;
|
|
23
|
+
},
|
|
24
|
+
deleteProperty(_target, prop) {
|
|
25
|
+
let success = false;
|
|
26
|
+
if (prop in extObj) {
|
|
27
|
+
delete extObj[prop];
|
|
28
|
+
success = true;
|
|
29
|
+
}
|
|
30
|
+
if (prop in baseObj) {
|
|
31
|
+
delete baseObj[prop];
|
|
32
|
+
success = true;
|
|
33
|
+
}
|
|
34
|
+
return success;
|
|
35
|
+
},
|
|
36
|
+
ownKeys(_target) {
|
|
37
|
+
const baseKeys = Reflect.ownKeys(baseObj);
|
|
38
|
+
const extKeys = Reflect.ownKeys(extObj);
|
|
39
|
+
const extKeysSet = new Set(extKeys);
|
|
40
|
+
return [...baseKeys.filter((k) => !extKeysSet.has(k)), ...extKeys];
|
|
41
|
+
},
|
|
42
|
+
defineProperty(_target, prop, desc) {
|
|
43
|
+
if (prop in extObj) {
|
|
44
|
+
delete extObj[prop];
|
|
45
|
+
}
|
|
46
|
+
Reflect.defineProperty(baseObj, prop, desc);
|
|
47
|
+
return true;
|
|
48
|
+
},
|
|
49
|
+
getOwnPropertyDescriptor(_target, prop) {
|
|
50
|
+
if (prop in extObj) {
|
|
51
|
+
return Reflect.getOwnPropertyDescriptor(extObj, prop);
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
return Reflect.getOwnPropertyDescriptor(baseObj, prop);
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
has(_target, prop) {
|
|
58
|
+
return prop in extObj || prop in baseObj;
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
}
|
package/esm/client.d.ts
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { fetchRetry } from './utils.js';
|
|
3
|
+
import * as errors from './errors.js';
|
|
4
|
+
import type { ScrapeConfig } from './scrapeconfig.js';
|
|
5
|
+
import type { ScreenshotConfig } from './screenshotconfig.js';
|
|
6
|
+
import type { ExtractionConfig } from './extractionconfig.js';
|
|
7
|
+
import { type AccountData, ExtractionResult, ScrapeResult, ScreenshotResult } from './result.js';
|
|
8
|
+
export declare class ScrapflyClient {
|
|
9
|
+
HOST: string;
|
|
10
|
+
private key;
|
|
11
|
+
private ua;
|
|
12
|
+
fetch: typeof fetchRetry;
|
|
13
|
+
constructor(options: {
|
|
14
|
+
key: string;
|
|
15
|
+
});
|
|
16
|
+
/**
|
|
17
|
+
* Raise appropriate error for given response and scrape result
|
|
18
|
+
*/
|
|
19
|
+
errResult(response: Response, result: ScrapeResult): errors.ScrapflyError;
|
|
20
|
+
/**
|
|
21
|
+
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
22
|
+
*/
|
|
23
|
+
handleResponse(response: Response, result: ScrapeResult): ScrapeResult;
|
|
24
|
+
/**
|
|
25
|
+
* Retrieve Scrapfly account details
|
|
26
|
+
*/
|
|
27
|
+
account(): Promise<AccountData>;
|
|
28
|
+
/**
|
|
29
|
+
* Issue a single scrape command from a given scrape configuration
|
|
30
|
+
*/
|
|
31
|
+
scrape(config: ScrapeConfig): Promise<ScrapeResult>;
|
|
32
|
+
/**
|
|
33
|
+
Concurrently scrape multiple configs
|
|
34
|
+
This is a async generator call it like this:
|
|
35
|
+
|
|
36
|
+
const results = [];
|
|
37
|
+
const errors = [];
|
|
38
|
+
for await (const resultOrError of client.concurrentScrape(configs)) {
|
|
39
|
+
if (resultOrError instanceof Error) {
|
|
40
|
+
errors.push(resultOrError);
|
|
41
|
+
} else {
|
|
42
|
+
results.push(resultOrError);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
@param concurrencyLimit: if not set it will be taken from your account info
|
|
47
|
+
*/
|
|
48
|
+
concurrentScrape(configs: ScrapeConfig[], concurrencyLimit?: number): AsyncGenerator<ScrapeResult | Error | undefined, void, undefined>;
|
|
49
|
+
/**
|
|
50
|
+
* Save screenshot response to a file
|
|
51
|
+
*/
|
|
52
|
+
saveScreenshot(result: ScreenshotResult, name: string, savePath?: string): Promise<any>;
|
|
53
|
+
/**
|
|
54
|
+
* Turn scrapfly screenshot API response to ScreenshotResult or raise one of ScrapflyError
|
|
55
|
+
*/
|
|
56
|
+
handleScreenshotResponse(response: Response): Promise<ScreenshotResult>;
|
|
57
|
+
/**
|
|
58
|
+
* Take a screenshot
|
|
59
|
+
*/
|
|
60
|
+
screenshot(config: ScreenshotConfig): Promise<ScreenshotResult>;
|
|
61
|
+
/**
|
|
62
|
+
* Turn scrapfly Extraction API response to ExtractionResult or raise one of ScrapflyError
|
|
63
|
+
*/
|
|
64
|
+
handleExtractionResponse(response: Response): Promise<ExtractionResult>;
|
|
65
|
+
/**
|
|
66
|
+
* Extract structured data from a web page
|
|
67
|
+
*/
|
|
68
|
+
extract(config: ExtractionConfig): Promise<ExtractionResult>;
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIjG,qBAAa,cAAc;IAClB,IAAI,SAA6B;IACxC,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAAS;IACnB,KAAK,oBAAc;gBAEP,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE;IAQpC;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,aAAa;IAkEzE;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY;IAUtE;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IA8BrC;;OAEG;IACG,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA4CzD;;;;;;;;;;;;;;;MAeE;IACK,gBAAgB,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,gBAAgB,CAAC,EAAE,MAAM,GACxB,cAAc,CAAC,YAAY,GAAG,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC;IAqCpE;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAoB7F;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuBrE;;OAEG;IACG,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB7E;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA6BnE"}
|
package/esm/client.js
ADDED
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
import { path } from './deps.js';
|
|
2
|
+
import { mkdir, writeFile } from './polyfill.js';
|
|
3
|
+
import { fetchRetry } from './utils.js';
|
|
4
|
+
import * as errors from './errors.js';
|
|
5
|
+
import { ExtractionResult, ScrapeResult, ScreenshotResult } from './result.js';
|
|
6
|
+
import { log } from './logger.js';
|
|
7
|
+
export class ScrapflyClient {
|
|
8
|
+
constructor(options) {
|
|
9
|
+
Object.defineProperty(this, "HOST", {
|
|
10
|
+
enumerable: true,
|
|
11
|
+
configurable: true,
|
|
12
|
+
writable: true,
|
|
13
|
+
value: 'https://api.scrapfly.io'
|
|
14
|
+
});
|
|
15
|
+
Object.defineProperty(this, "key", {
|
|
16
|
+
enumerable: true,
|
|
17
|
+
configurable: true,
|
|
18
|
+
writable: true,
|
|
19
|
+
value: void 0
|
|
20
|
+
});
|
|
21
|
+
Object.defineProperty(this, "ua", {
|
|
22
|
+
enumerable: true,
|
|
23
|
+
configurable: true,
|
|
24
|
+
writable: true,
|
|
25
|
+
value: void 0
|
|
26
|
+
});
|
|
27
|
+
Object.defineProperty(this, "fetch", {
|
|
28
|
+
enumerable: true,
|
|
29
|
+
configurable: true,
|
|
30
|
+
writable: true,
|
|
31
|
+
value: fetchRetry
|
|
32
|
+
});
|
|
33
|
+
if (typeof options.key !== 'string' || options.key.trim() === '') {
|
|
34
|
+
throw new errors.BadApiKeyError('Invalid key. Key must be a non-empty string');
|
|
35
|
+
}
|
|
36
|
+
this.key = options.key;
|
|
37
|
+
this.ua = 'Typescript Scrapfly SDK';
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Raise appropriate error for given response and scrape result
|
|
41
|
+
*/
|
|
42
|
+
errResult(response, result) {
|
|
43
|
+
const error = result.result.error;
|
|
44
|
+
const message = error?.message ?? '';
|
|
45
|
+
const args = {
|
|
46
|
+
code: result.result.status,
|
|
47
|
+
http_status_code: result.result.status_code,
|
|
48
|
+
is_retryable: error?.retryable ?? false,
|
|
49
|
+
api_response: result,
|
|
50
|
+
resource: result.result.status ? result.result.status.split('::')[1] : null,
|
|
51
|
+
retry_delay: error?.retryable ? 5 : response.headers.get('X-Retry') ?? 5,
|
|
52
|
+
retry_times: 3,
|
|
53
|
+
documentation_url: error?.doc_url ?? 'https://scrapfly.io/docs/scrape-api/errors#api',
|
|
54
|
+
};
|
|
55
|
+
if (result.result.success === true) {
|
|
56
|
+
switch (args.http_status_code) {
|
|
57
|
+
case 500:
|
|
58
|
+
return new errors.ApiHttpServerError(message, args);
|
|
59
|
+
case 401:
|
|
60
|
+
return new errors.BadApiKeyError(message, args);
|
|
61
|
+
case 429:
|
|
62
|
+
return new errors.TooManyRequests(message, args);
|
|
63
|
+
}
|
|
64
|
+
switch (args.resource) {
|
|
65
|
+
case 'SCRAPE':
|
|
66
|
+
return new errors.ScrapflyScrapeError(message, args);
|
|
67
|
+
case 'WEBHOOK':
|
|
68
|
+
return new errors.ScrapflyWebhookError(message, args);
|
|
69
|
+
case 'PROXY':
|
|
70
|
+
return new errors.ScrapflyProxyError(message, args);
|
|
71
|
+
case 'SCHEDULE':
|
|
72
|
+
return new errors.ScrapflyScheduleError(message, args);
|
|
73
|
+
case 'ASP':
|
|
74
|
+
return new errors.ScrapflyAspError(message, args);
|
|
75
|
+
case 'SESSION':
|
|
76
|
+
return new errors.ScrapflySessionError(message, args);
|
|
77
|
+
}
|
|
78
|
+
if (args.resource) {
|
|
79
|
+
return new errors.ApiHttpClientError(message, args);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
if (args.code === 'ERR::SCRAPE::BAD_UPSTREAM_RESPONSE') {
|
|
84
|
+
if (args.http_status_code >= 400 && args.http_status_code < 500) {
|
|
85
|
+
return new errors.UpstreamHttpClientError(message, args);
|
|
86
|
+
}
|
|
87
|
+
if (args.http_status_code >= 500 && args.http_status_code < 600) {
|
|
88
|
+
return new errors.UpstreamHttpServerError(message, args);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
switch (args.resource) {
|
|
92
|
+
case 'SCRAPE':
|
|
93
|
+
return new errors.ScrapflyScrapeError(message, args);
|
|
94
|
+
case 'WEBHOOK':
|
|
95
|
+
return new errors.ScrapflyWebhookError(message, args);
|
|
96
|
+
case 'PROXY':
|
|
97
|
+
return new errors.ScrapflyProxyError(message, args);
|
|
98
|
+
case 'SCHEDULE':
|
|
99
|
+
return new errors.ScrapflyScheduleError(message, args);
|
|
100
|
+
case 'ASP':
|
|
101
|
+
return new errors.ScrapflyAspError(message, args);
|
|
102
|
+
case 'SESSION':
|
|
103
|
+
return new errors.ScrapflySessionError(message, args);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return new errors.ScrapflyError(message, args);
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
|
|
110
|
+
*/
|
|
111
|
+
handleResponse(response, result) {
|
|
112
|
+
// success
|
|
113
|
+
log.debug('scrape log url: ', result.result.log_url);
|
|
114
|
+
if (result.result.status === 'DONE' && result.result.success === true) {
|
|
115
|
+
return result;
|
|
116
|
+
}
|
|
117
|
+
// something went wrong
|
|
118
|
+
throw this.errResult(response, result);
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Retrieve Scrapfly account details
|
|
122
|
+
*/
|
|
123
|
+
async account() {
|
|
124
|
+
log.debug('retrieving account info');
|
|
125
|
+
let response;
|
|
126
|
+
try {
|
|
127
|
+
const url = new URL(this.HOST + '/account');
|
|
128
|
+
const params = { key: this.key };
|
|
129
|
+
url.search = new URLSearchParams(params).toString();
|
|
130
|
+
response = await this.fetch({
|
|
131
|
+
url: url.toString(),
|
|
132
|
+
method: 'GET',
|
|
133
|
+
headers: {
|
|
134
|
+
'user-agent': this.ua,
|
|
135
|
+
'accept-encoding': 'gzip, deflate, br',
|
|
136
|
+
accept: 'application/json',
|
|
137
|
+
},
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
catch (e) {
|
|
141
|
+
log.error('error', e);
|
|
142
|
+
throw e;
|
|
143
|
+
}
|
|
144
|
+
const data = await response.json();
|
|
145
|
+
if ('error_id' in data || Object.keys(data).length === 0) {
|
|
146
|
+
if (data.http_code == 401 || response.status == 401) {
|
|
147
|
+
throw new errors.BadApiKeyError(JSON.stringify(data));
|
|
148
|
+
}
|
|
149
|
+
throw new errors.ApiHttpClientError(JSON.stringify(data));
|
|
150
|
+
}
|
|
151
|
+
return data;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Issue a single scrape command from a given scrape configuration
|
|
155
|
+
*/
|
|
156
|
+
async scrape(config) {
|
|
157
|
+
log.debug('scraping', { method: config.method, url: config.url });
|
|
158
|
+
let response;
|
|
159
|
+
try {
|
|
160
|
+
const url = new URL(this.HOST + '/scrape');
|
|
161
|
+
const params = config.toApiParams({ key: this.key });
|
|
162
|
+
url.search = new URLSearchParams(params).toString();
|
|
163
|
+
response = await this.fetch({
|
|
164
|
+
url: url.toString(),
|
|
165
|
+
method: config.method,
|
|
166
|
+
headers: {
|
|
167
|
+
'user-agent': this.ua,
|
|
168
|
+
'content-type': config.method === 'POST'
|
|
169
|
+
? config.headers?.['content-type'] ?? 'application/json'
|
|
170
|
+
: 'application/json',
|
|
171
|
+
'accept-encoding': 'gzip, deflate, br',
|
|
172
|
+
accept: 'application/json',
|
|
173
|
+
},
|
|
174
|
+
body: config.body,
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
catch (e) {
|
|
178
|
+
log.error('error', e);
|
|
179
|
+
e.scrapeConfig = config;
|
|
180
|
+
throw e;
|
|
181
|
+
}
|
|
182
|
+
const data = await response.json();
|
|
183
|
+
if ('error_id' in data || Object.keys(data).length === 0) {
|
|
184
|
+
if (data.http_code == 401 || response.status == 401) {
|
|
185
|
+
throw new errors.BadApiKeyError(JSON.stringify(data));
|
|
186
|
+
}
|
|
187
|
+
throw new errors.ApiHttpClientError(JSON.stringify(data));
|
|
188
|
+
}
|
|
189
|
+
const result = this.handleResponse(response, new ScrapeResult({
|
|
190
|
+
config: data.config,
|
|
191
|
+
context: data.context,
|
|
192
|
+
result: data.result,
|
|
193
|
+
uuid: data.uuid,
|
|
194
|
+
}));
|
|
195
|
+
return result;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
Concurrently scrape multiple configs
|
|
199
|
+
This is a async generator call it like this:
|
|
200
|
+
|
|
201
|
+
const results = [];
|
|
202
|
+
const errors = [];
|
|
203
|
+
for await (const resultOrError of client.concurrentScrape(configs)) {
|
|
204
|
+
if (resultOrError instanceof Error) {
|
|
205
|
+
errors.push(resultOrError);
|
|
206
|
+
} else {
|
|
207
|
+
results.push(resultOrError);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
@param concurrencyLimit: if not set it will be taken from your account info
|
|
212
|
+
*/
|
|
213
|
+
async *concurrentScrape(configs, concurrencyLimit) {
|
|
214
|
+
if (concurrencyLimit === undefined) {
|
|
215
|
+
const account = await this.account();
|
|
216
|
+
concurrencyLimit = account.subscription.usage.scrape.concurrent_limit;
|
|
217
|
+
log.info(`concurrency not provided - setting it to ${concurrencyLimit} from account info`);
|
|
218
|
+
}
|
|
219
|
+
const activePromises = new Set();
|
|
220
|
+
const configsIterator = configs[Symbol.iterator]();
|
|
221
|
+
// Helper function to start a new scrape and add it to activePromises
|
|
222
|
+
const startNewScrape = () => {
|
|
223
|
+
const { value: config, done } = configsIterator.next();
|
|
224
|
+
if (done)
|
|
225
|
+
return; // No more configs
|
|
226
|
+
const promise = this.scrape(config).catch((error) => error); // Catch errors and return them
|
|
227
|
+
activePromises.add(promise);
|
|
228
|
+
promise.finally(() => {
|
|
229
|
+
activePromises.delete(promise);
|
|
230
|
+
// After each scrape, start a new one if there are remaining configs
|
|
231
|
+
startNewScrape();
|
|
232
|
+
});
|
|
233
|
+
};
|
|
234
|
+
// Initially start as many scrapes as the concurrency limit
|
|
235
|
+
for (let i = 0; i < concurrencyLimit; i++) {
|
|
236
|
+
startNewScrape();
|
|
237
|
+
}
|
|
238
|
+
// As each scrape finishes, yield the result or error and start a new one if there are remaining configs
|
|
239
|
+
while (activePromises.size > 0) {
|
|
240
|
+
log.debug(`concurrently scraping ${activePromises.size}/${configs.length}}`);
|
|
241
|
+
const resultOrError = await Promise.race(activePromises);
|
|
242
|
+
yield resultOrError;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Save screenshot response to a file
|
|
247
|
+
*/
|
|
248
|
+
async saveScreenshot(result, name, savePath) {
|
|
249
|
+
if (!result.image) {
|
|
250
|
+
throw new Error('Screenshot binary does not exist');
|
|
251
|
+
}
|
|
252
|
+
const extension_name = result.metadata.extension_name;
|
|
253
|
+
let file_path;
|
|
254
|
+
if (savePath) {
|
|
255
|
+
await mkdir(savePath, { recursive: true });
|
|
256
|
+
file_path = path.join(savePath, `${name}.${extension_name}`);
|
|
257
|
+
}
|
|
258
|
+
else {
|
|
259
|
+
file_path = `${name}.${extension_name}`;
|
|
260
|
+
}
|
|
261
|
+
const content = new Uint8Array(result.image);
|
|
262
|
+
// Use Deno's write file method
|
|
263
|
+
await writeFile(file_path, content);
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Turn scrapfly screenshot API response to ScreenshotResult or raise one of ScrapflyError
|
|
267
|
+
*/
|
|
268
|
+
async handleScreenshotResponse(response) {
|
|
269
|
+
if (response.headers.get('content-type') === 'application/json') {
|
|
270
|
+
const data = (await response.json());
|
|
271
|
+
if (data.http_code == 401 || response.status == 401) {
|
|
272
|
+
throw new errors.BadApiKeyError(JSON.stringify(data));
|
|
273
|
+
}
|
|
274
|
+
if ('error_id' in data) {
|
|
275
|
+
throw new errors.ScreenshotApiError(JSON.stringify(data));
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
if (!response.ok) {
|
|
279
|
+
throw new errors.ScreenshotApiError(JSON.stringify(await response.json()));
|
|
280
|
+
}
|
|
281
|
+
const data = await response.arrayBuffer();
|
|
282
|
+
const result = new ScreenshotResult(response, data);
|
|
283
|
+
return result;
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Take a screenshot
|
|
287
|
+
*/
|
|
288
|
+
async screenshot(config) {
|
|
289
|
+
let response;
|
|
290
|
+
try {
|
|
291
|
+
const url = new URL(this.HOST + '/screenshot');
|
|
292
|
+
const params = config.toApiParams({ key: this.key });
|
|
293
|
+
url.search = new URLSearchParams(params).toString();
|
|
294
|
+
response = await this.fetch({
|
|
295
|
+
url: url.toString(),
|
|
296
|
+
method: 'GET',
|
|
297
|
+
headers: {
|
|
298
|
+
'user-agent': this.ua,
|
|
299
|
+
'accept-encoding': 'gzip, deflate, br',
|
|
300
|
+
accept: 'application/json',
|
|
301
|
+
},
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
catch (e) {
|
|
305
|
+
log.error('error', e);
|
|
306
|
+
throw e;
|
|
307
|
+
}
|
|
308
|
+
const result = await this.handleScreenshotResponse(response);
|
|
309
|
+
return result;
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Turn scrapfly Extraction API response to ExtractionResult or raise one of ScrapflyError
|
|
313
|
+
*/
|
|
314
|
+
async handleExtractionResponse(response) {
|
|
315
|
+
const data = (await response.json());
|
|
316
|
+
if ('error_id' in data) {
|
|
317
|
+
if (data.http_code == 401 || response.status == 401) {
|
|
318
|
+
throw new errors.BadApiKeyError(JSON.stringify(data));
|
|
319
|
+
}
|
|
320
|
+
throw new errors.ExtractionApiError(JSON.stringify(data));
|
|
321
|
+
}
|
|
322
|
+
if (!response.ok) {
|
|
323
|
+
throw new errors.ApiHttpClientError(JSON.stringify(await response.json()));
|
|
324
|
+
}
|
|
325
|
+
const result = new ExtractionResult({
|
|
326
|
+
data: data.data,
|
|
327
|
+
content_type: data.content_type,
|
|
328
|
+
});
|
|
329
|
+
return result;
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* Extract structured data from a web page
|
|
333
|
+
*/
|
|
334
|
+
async extract(config) {
|
|
335
|
+
log.debug('extacting data from', { content_type: config.content_type });
|
|
336
|
+
let response;
|
|
337
|
+
try {
|
|
338
|
+
const url = new URL(this.HOST + '/extraction');
|
|
339
|
+
const params = await config.toApiParams({ key: this.key });
|
|
340
|
+
url.search = new URLSearchParams(params).toString();
|
|
341
|
+
const headers = {
|
|
342
|
+
'user-agent': this.ua,
|
|
343
|
+
'accept-encoding': 'gzip, deflate, br',
|
|
344
|
+
'content-type': config.content_type,
|
|
345
|
+
'accept': 'application/json',
|
|
346
|
+
};
|
|
347
|
+
if (config.document_compression_format && config.document_compression_format) {
|
|
348
|
+
headers['content-encoding'] = config.document_compression_format;
|
|
349
|
+
}
|
|
350
|
+
response = await this.fetch({
|
|
351
|
+
url: url.toString(),
|
|
352
|
+
method: 'POST',
|
|
353
|
+
headers: headers,
|
|
354
|
+
body: config.body,
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
catch (e) {
|
|
358
|
+
log.error('error', e);
|
|
359
|
+
throw e;
|
|
360
|
+
}
|
|
361
|
+
const result = await this.handleExtractionResponse(response);
|
|
362
|
+
return result;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assert_path.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/assert_path.ts"],"names":[],"mappings":"AAGA,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,QAMvC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
2
|
+
// Copyright the Browserify authors. MIT License.
|
|
3
|
+
export function assertPath(path) {
|
|
4
|
+
if (typeof path !== "string") {
|
|
5
|
+
throw new TypeError(`Path must be a string. Received ${JSON.stringify(path)}`);
|
|
6
|
+
}
|
|
7
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare function stripSuffix(name: string, suffix: string): string;
|
|
2
|
+
export declare function lastPathSegment(path: string, isSep: (char: number) => boolean, start?: number): string;
|
|
3
|
+
export declare function assertArgs(path: string, suffix: string): string | undefined;
|
|
4
|
+
//# sourceMappingURL=basename.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"basename.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/basename.ts"],"names":[],"mappings":"AAKA,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAchE;AAED,wBAAgB,eAAe,CAC7B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,EAChC,KAAK,SAAI,GACR,MAAM,CAiBR;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,sBAQtD"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
|
2
|
+
// This module is browser compatible.
|
|
3
|
+
import { assertPath } from "./assert_path.js";
|
|
4
|
+
export function stripSuffix(name, suffix) {
|
|
5
|
+
if (suffix.length >= name.length) {
|
|
6
|
+
return name;
|
|
7
|
+
}
|
|
8
|
+
const lenDiff = name.length - suffix.length;
|
|
9
|
+
for (let i = suffix.length - 1; i >= 0; --i) {
|
|
10
|
+
if (name.charCodeAt(lenDiff + i) !== suffix.charCodeAt(i)) {
|
|
11
|
+
return name;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
return name.slice(0, -suffix.length);
|
|
15
|
+
}
|
|
16
|
+
export function lastPathSegment(path, isSep, start = 0) {
|
|
17
|
+
let matchedNonSeparator = false;
|
|
18
|
+
let end = path.length;
|
|
19
|
+
for (let i = path.length - 1; i >= start; --i) {
|
|
20
|
+
if (isSep(path.charCodeAt(i))) {
|
|
21
|
+
if (matchedNonSeparator) {
|
|
22
|
+
start = i + 1;
|
|
23
|
+
break;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
else if (!matchedNonSeparator) {
|
|
27
|
+
matchedNonSeparator = true;
|
|
28
|
+
end = i + 1;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return path.slice(start, end);
|
|
32
|
+
}
|
|
33
|
+
export function assertArgs(path, suffix) {
|
|
34
|
+
assertPath(path);
|
|
35
|
+
if (path.length === 0)
|
|
36
|
+
return path;
|
|
37
|
+
if (typeof suffix !== "string") {
|
|
38
|
+
throw new TypeError(`Suffix must be a string. Received ${JSON.stringify(suffix)}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@std/path/1.0.1/_common/common.ts"],"names":[],"mappings":"AAGA,wBAAgB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAsB3D"}
|