scrapfly-sdk 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -2
- package/build/src/client.d.ts +0 -2
- package/build/src/client.js +1 -2
- package/build/src/client.js.map +1 -1
- package/build/src/errors.d.ts +2 -0
- package/build/src/errors.js +2 -0
- package/build/src/errors.js.map +1 -1
- package/build/src/logger.d.ts +18 -0
- package/build/src/logger.js +46 -0
- package/build/src/logger.js.map +1 -0
- package/build/src/main.d.ts +3 -0
- package/build/src/main.js +3 -0
- package/build/src/main.js.map +1 -1
- package/build/src/result.d.ts +9 -6
- package/build/src/result.js +12 -0
- package/build/src/result.js.map +1 -1
- package/build/src/scrapeconfig.js +2 -3
- package/build/src/scrapeconfig.js.map +1 -1
- package/package.json +6 -7
package/README.md
CHANGED
|
@@ -2,9 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
`npm install scrapfly-sdk`
|
|
4
4
|
|
|
5
|
+
Typescript/NodeJS SDK for [Scrapfly.io](https://scrapfly.io/) web scraping API which allows to:
|
|
6
|
+
|
|
7
|
+
- Scrape the web without being blocked.
|
|
8
|
+
- Use headless browsers to access Javascript-powered page data.
|
|
9
|
+
- Scale up web scraping.
|
|
10
|
+
- ... and [much more](https://scrapfly.io/docs/scrape-api/getting-started)!
|
|
11
|
+
|
|
12
|
+
For web scraping guides see [our blog](https://scrapfly.io/blog/) and [#scrapeguide](https://scrapfly.io/blog/tag/scrapeguide/) tag for how to scrape specific targets.
|
|
13
|
+
|
|
5
14
|
## Quick Intro
|
|
6
15
|
|
|
7
|
-
|
|
16
|
+
1. Register a [Scrapfly account for free](https://scrapfly.io/register)
|
|
17
|
+
2. Get your API Key on [scrapfly.io/dashboard](https://scrapfly.io/dashboard)
|
|
18
|
+
3. Start scraping: 🚀
|
|
8
19
|
|
|
9
20
|
```javascript
|
|
10
21
|
import { ScrapflyClient, ScrapeConfig } from 'scrapfly-sdk';
|
|
@@ -27,12 +38,36 @@ const apiResponse = await client.scrape(
|
|
|
27
38
|
}),
|
|
28
39
|
);
|
|
29
40
|
console.log(apiResponse.result.content); // html content
|
|
41
|
+
// Parse HTML directly with SDK (through cheerio)
|
|
42
|
+
console.log(apiResponse.result.selector('h3').text());
|
|
30
43
|
```
|
|
31
44
|
|
|
32
45
|
For more see [/examples](/examples/) directory.
|
|
33
|
-
For more on Scrapfly API see
|
|
46
|
+
For more on Scrapfly API see our [getting started documentation](https://scrapfly.io/docs/scrape-api/getting-started)
|
|
34
47
|
For Python see [Scrapfly Python SDK](https://github.com/scrapfly/python-scrapfly)
|
|
35
48
|
|
|
49
|
+
## Debugging
|
|
50
|
+
|
|
51
|
+
To enable debug logs set Scrapfly's log level to `"DEBUG"`:
|
|
52
|
+
|
|
53
|
+
```javascript
|
|
54
|
+
import { log } from 'scrapfly-sdk';
|
|
55
|
+
|
|
56
|
+
log.setLevel('DEBUG');
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Additionally, set `debug=true` in `ScrapeConfig` to access debug information in [Scrapfly web dashboard](https://scrapfly.io/dashboard):
|
|
60
|
+
|
|
61
|
+
```typescript
|
|
62
|
+
import { ScrapflyClient } from 'scrapfly-sdk';
|
|
63
|
+
|
|
64
|
+
new ScrapeConfig({
|
|
65
|
+
url: 'https://web-scraping.dev/product/1',
|
|
66
|
+
debug: true,
|
|
67
|
+
// ^ enable debug information - this will show extra details on web dashboard
|
|
68
|
+
});
|
|
69
|
+
```
|
|
70
|
+
|
|
36
71
|
## Development
|
|
37
72
|
|
|
38
73
|
Install and setup environment:
|
package/build/src/client.d.ts
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
import { ScrapeConfig } from './scrapeconfig.js';
|
|
2
2
|
import * as errors from './errors.js';
|
|
3
3
|
import { ScrapeResult, AccountData } from './result.js';
|
|
4
|
-
import { Logger, ILogObj } from 'tslog';
|
|
5
4
|
import { AxiosResponse } from 'axios';
|
|
6
|
-
export declare const log: Logger<ILogObj>;
|
|
7
5
|
export declare class ScrapflyClient {
|
|
8
6
|
HOST: string;
|
|
9
7
|
private key;
|
package/build/src/client.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import * as errors from './errors.js';
|
|
2
2
|
import { ScrapeResult } from './result.js';
|
|
3
|
-
import { Logger } from 'tslog';
|
|
4
3
|
import axios from 'axios';
|
|
5
|
-
|
|
4
|
+
import { log } from './logger.js';
|
|
6
5
|
export class ScrapflyClient {
|
|
7
6
|
HOST = 'https://api.scrapfly.io';
|
|
8
7
|
key;
|
package/build/src/client.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/client.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAuC,YAAY,EAAe,MAAM,aAAa,CAAC;AAC7F,OAAO,
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/client.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAuC,YAAY,EAAe,MAAM,aAAa,CAAC;AAC7F,OAAO,KAAwB,MAAM,OAAO,CAAC;AAC7C,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAElC,MAAM,OAAO,cAAc;IAChB,IAAI,GAAG,yBAAyB,CAAC;IAChC,GAAG,CAAS;IACZ,EAAE,CAAS;IAEnB,YAAY,OAAwB;QAChC,IAAI,OAAO,OAAO,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC9D,MAAM,IAAI,MAAM,CAAC,cAAc,CAAC,6CAA6C,CAAC,CAAC;SAClF;QACD,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QACvB,IAAI,CAAC,EAAE,GAAG,yBAAyB,CAAC;IACxC,CAAC;IACD;;OAEG;IACH,SAAS,CAAC,QAAuB,EAAE,MAAoB;QACnD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;QAClC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG;YACT,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC1B,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;YAC3C,YAAY,EAAE,KAAK,CAAC,SAAS,IAAI,KAAK;YACtC,YAAY,EAAE,MAAM;YACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;YAC3E,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;YAC3E,WAAW,EAAE,CAAC;YACd,iBAAiB,EAAE,KAAK,CAAC,OAAO,IAAI,gDAAgD;SACvF,CAAC;QACF,MAAM,cAAc,GAAG;YACnB,MAAM,EAAE,MAAM,CAAC,mBAAmB;YAClC,OAAO,EAAE,MAAM,CAAC,oBAAoB;YACpC,KAAK,EAAE,MAAM,CAAC,kBAAkB;YAChC,QAAQ,EAAE,MAAM,CAAC,qBAAqB;YACtC,GAAG,EAAE,MAAM,CAAC,gBAAgB;YAC5B,OAAO,EAAE,MAAM,CAAC,oBAAoB;SACvC,CAAC;QACF,MAAM,gBAAgB,GAAG;YACrB,GAAG,EAAE,MAAM,CAAC,cAAc;YAC1B,GAAG,EAAE,MAAM,CAAC,eAAe;SAC9B,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE;YAChC,IAAI,IAAI,CAAC,gBAAgB,IAAI,GAAG,EAAE;gBAC9B,OAAO,IAAI,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;aACvD;YACD,IAAI,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE;gBACzC,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;aACrE;YACD,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAC/B,OAAO,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;aAC3D;YACD,OAAO,IAAI,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;SACvD;aAAM;YACH,IAAI,IAAI,CAAC,IAAI,KAAK,oCAAoC,EAAE;gBACpD,IAAI,IAAI,CAAC,gBAAgB,IAAI,GAAG,EAAE;oBAC9B,OAAO,IAAI,MAAM,CAAC,uBAAuB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;iBAC5D;gBACD,IAAI,IAAI,CAAC,gBAAgB,IAAI,GAAG,EAAE;oBAC9B,OAAO,IAAI,MAAM,CAAC,uBAAuB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;iBAC5D;aACJ;YACD,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAC/B,OAAO,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;aAC3D;YACD,MAAM,IAAI,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;SACjD;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,QAAuB;QACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAKrB,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC;QACtC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrD,UAAU;QACV,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE;YACnE,OAAO,MAAM,CAAC;SACjB;QACD,uBAAuB;QACvB,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,GAAG,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACrC,IAAI;YACA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC;gBACjC,MAAM,EAAE,KAAK;gBACb,GAAG,EAAE,IAAI,CAAC,IAAI,GAAG,UAAU;gBAC3B,OAAO,EAAE;oBACL,YAAY,EAAE,IAAI,CAAC,EAAE;oBACrB,gBAAgB,EAAE,mBAAmB;oBACrC,MAAM,EAAE,kBAAkB;iBAC7B;gBACD,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;aAC5B,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,IAAI,CAAC;SACxB;QAAC,OAAO,CAAC,EAAE;YACR,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtB,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;gBACzC,MAAM,IAAI,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;aACpE;YACD,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,8CAA8C,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;SACpG;IACL,CAAC;IACD;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,MAAoB;QAC7B,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QAClE,IAAI,QAAuB,CAAC;QAC5B,IAAI;YACA,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC;gBAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,GAAG,EAAE,IAAI,CAAC,IAAI,GAAG,SAAS;gBAC1B,OAAO,EAAE;oBACL,YAAY,EAAE,IAAI,CAAC,EAAE;oBACrB,cAAc,EAAE,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,kBAAkB;oBAC9F,gBAAgB,EAAE,mBAAmB;oBACrC,MAAM,EAAE,kBAAkB;iBAC7B;gBACD,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7C,IAAI,EAAE,MAAM,CAAC,IAAI;aACpB,CAAC,CAAC;SACN;QAAC,OAAO,CAAC,EAAE;YACR,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtB,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;gBACzC,MAAM,IAAI,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;aACpE;YACD,MAAM,CAAC,CAAC;SACX;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QACnD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;;;;;;;;;;;SAeK;IACL,KAAK,CAAC,CAAC,gBAAgB,CACnB,OAAuB,EACvB,gBAAyB;QAEzB,IAAI,gBAAgB,KAAK,SAAS,EAAE;YAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC;YACtE,GAAG,CAAC,IAAI,CAAC,4CAA4C,gBAAgB,oBAAoB,CAAC,CAAC;SAC9F;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAiC,CAAC;QAChE,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAEnD,qEAAqE;QACrE,MAAM,cAAc,GAAG,GAAG,EAAE;YACxB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC;YACvD,IAAI,IAAI;gBAAE,OAAO,CAAC,kBAAkB;YAEpC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,+BAA+B;YAC5F,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE5B,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE;gBACjB,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC/B,oEAAoE;gBACpE,cAAc,EAAE,CAAC;YACrB,CAAC,CAAC,CAAC;QACP,CAAC,CAAC;QAEF,2DAA2D;QAC3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,EAAE,CAAC,EAAE,EAAE;YACvC,cAAc,EAAE,CAAC;SACpB;QAED,wGAAwG;QACxG,OAAO,cAAc,CAAC,IAAI,GAAG,CAAC,EAAE;YAC5B,GAAG,CAAC,KAAK,CAAC,yBAAyB,cAAc,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7E,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACzD,MAAM,aAAa,CAAC;SACvB;IACL,CAAC;CACJ"}
|
package/build/src/errors.d.ts
CHANGED
|
@@ -6,6 +6,8 @@ export declare class ScrapeConfigError extends ScrapflyError {
|
|
|
6
6
|
}
|
|
7
7
|
export declare class EncodeError extends ScrapflyError {
|
|
8
8
|
}
|
|
9
|
+
export declare class ContentTypeError extends ScrapflyError {
|
|
10
|
+
}
|
|
9
11
|
export declare class HttpError extends ScrapflyError {
|
|
10
12
|
}
|
|
11
13
|
export declare class UpstreamHttpError extends HttpError {
|
package/build/src/errors.js
CHANGED
|
@@ -11,6 +11,8 @@ export class ScrapeConfigError extends ScrapflyError {
|
|
|
11
11
|
// raised when scrape parameters cannot be encoded
|
|
12
12
|
export class EncodeError extends ScrapflyError {
|
|
13
13
|
}
|
|
14
|
+
export class ContentTypeError extends ScrapflyError {
|
|
15
|
+
}
|
|
14
16
|
// Base error for all http related operations
|
|
15
17
|
export class HttpError extends ScrapflyError {
|
|
16
18
|
}
|
package/build/src/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,aAAc,SAAQ,KAAK;IACpC,IAAI,CAAsB;IAE1B,YAAY,OAAe,EAAE,IAA0B;QACnD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;CACJ;AAED,uCAAuC;AACvC,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD,kDAAkD;AAClD,MAAM,OAAO,WAAY,SAAQ,aAAa;CAAG;
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,aAAc,SAAQ,KAAK;IACpC,IAAI,CAAsB;IAE1B,YAAY,OAAe,EAAE,IAA0B;QACnD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;CACJ;AAED,uCAAuC;AACvC,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD,kDAAkD;AAClD,MAAM,OAAO,WAAY,SAAQ,aAAa;CAAG;AACjD,MAAM,OAAO,gBAAiB,SAAQ,aAAa;CAAG;AAEtD,6CAA6C;AAC7C,MAAM,OAAO,SAAU,SAAQ,aAAa;CAAG;AAC/C,MAAM,OAAO,iBAAkB,SAAQ,SAAS;CAAG;AACnD,MAAM,OAAO,uBAAwB,SAAQ,iBAAiB;CAAG;AACjE,MAAM,OAAO,uBAAwB,SAAQ,uBAAuB;CAAG;AAEvE,MAAM,OAAO,kBAAmB,SAAQ,SAAS;CAAG;AACpD,sEAAsE;AACtE,MAAM,OAAO,cAAe,SAAQ,kBAAkB;CAAG;AAEzD,MAAM,OAAO,eAAgB,SAAQ,SAAS;CAAG;AACjD,MAAM,OAAO,kBAAmB,SAAQ,SAAS;CAAG;AACpD,MAAM,OAAO,mBAAoB,SAAQ,SAAS;CAAG;AACrD,sGAAsG;AACtG,MAAM,OAAO,kBAAmB,SAAQ,SAAS;CAAG;AACpD,MAAM,OAAO,qBAAsB,SAAQ,SAAS;CAAG;AACvD,gEAAgE;AAChE,MAAM,OAAO,gBAAiB,SAAQ,SAAS;CAAG;AAClD,MAAM,OAAO,qBAAsB,SAAQ,SAAS;CAAG;AACvD,0EAA0E;AAC1E,MAAM,OAAO,oBAAqB,SAAQ,SAAS;CAAG;AACtD,6CAA6C;AAC7C,MAAM,OAAO,oBAAqB,SAAQ,SAAS;CAAG;AACtD,wDAAwD;AACxD,MAAM,OAAO,yBAA0B,SAAQ,SAAS;CAAG;AAC3D,+CAA+C;AAC/C,MAAM,OAAO,iBAAkB,SAAQ,SAAS;CAAG"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
declare enum LogLevel {
|
|
2
|
+
DEBUG = "DEBUG",
|
|
3
|
+
INFO = "INFO",
|
|
4
|
+
WARN = "WARN",
|
|
5
|
+
ERROR = "ERROR"
|
|
6
|
+
}
|
|
7
|
+
declare class Logger {
|
|
8
|
+
private level;
|
|
9
|
+
name: string;
|
|
10
|
+
constructor(name: string, level?: LogLevel);
|
|
11
|
+
debug(...args: any[]): void;
|
|
12
|
+
info(...args: any[]): void;
|
|
13
|
+
warn(...args: any[]): void;
|
|
14
|
+
error(...args: any[]): void;
|
|
15
|
+
setLevel(level: string): void;
|
|
16
|
+
}
|
|
17
|
+
export default Logger;
|
|
18
|
+
export declare const log: Logger;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
var LogLevel;
|
|
2
|
+
(function (LogLevel) {
|
|
3
|
+
LogLevel["DEBUG"] = "DEBUG";
|
|
4
|
+
LogLevel["INFO"] = "INFO";
|
|
5
|
+
LogLevel["WARN"] = "WARN";
|
|
6
|
+
LogLevel["ERROR"] = "ERROR";
|
|
7
|
+
})(LogLevel || (LogLevel = {}));
|
|
8
|
+
class Logger {
|
|
9
|
+
level;
|
|
10
|
+
name;
|
|
11
|
+
constructor(name, level = LogLevel.INFO) {
|
|
12
|
+
this.name = name;
|
|
13
|
+
this.level = level;
|
|
14
|
+
}
|
|
15
|
+
debug(...args) {
|
|
16
|
+
if (this.level <= LogLevel.DEBUG) {
|
|
17
|
+
console.log(...args);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
info(...args) {
|
|
21
|
+
if (this.level <= LogLevel.INFO) {
|
|
22
|
+
console.info(...args);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
warn(...args) {
|
|
26
|
+
if (this.level <= LogLevel.WARN) {
|
|
27
|
+
console.warn(...args);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
error(...args) {
|
|
31
|
+
if (this.level <= LogLevel.ERROR) {
|
|
32
|
+
console.error(...args);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
setLevel(level) {
|
|
36
|
+
if (Object.values(LogLevel).includes(level)) {
|
|
37
|
+
this.level = level;
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
console.error(`Invalid log level: ${level}`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
export default Logger;
|
|
45
|
+
export const log = new Logger('scrapfly');
|
|
46
|
+
//# sourceMappingURL=logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/logger.ts"],"names":[],"mappings":"AAAA,IAAK,QAKJ;AALD,WAAK,QAAQ;IACT,2BAAe,CAAA;IACf,yBAAa,CAAA;IACb,yBAAa,CAAA;IACb,2BAAe,CAAA;AACnB,CAAC,EALI,QAAQ,KAAR,QAAQ,QAKZ;AAED,MAAM,MAAM;IACA,KAAK,CAAW;IACjB,IAAI,CAAS;IAEpB,YAAY,IAAY,EAAE,QAAkB,QAAQ,CAAC,IAAI;QACrD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,GAAG,IAAW;QAChB,IAAI,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAE;YAC9B,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;SACxB;IACL,CAAC;IAED,IAAI,CAAC,GAAG,IAAW;QACf,IAAI,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE;YAC7B,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;SACzB;IACL,CAAC;IAED,IAAI,CAAC,GAAG,IAAW;QACf,IAAI,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE;YAC7B,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;SACzB;IACL,CAAC;IAED,KAAK,CAAC,GAAG,IAAW;QAChB,IAAI,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAE;YAC9B,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;SAC1B;IACL,CAAC;IAED,QAAQ,CAAC,KAAa;QAClB,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,KAAiB,CAAC,EAAE;YACrD,IAAI,CAAC,KAAK,GAAG,KAAiB,CAAC;SAClC;aAAM;YACH,OAAO,CAAC,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;SAChD;IACL,CAAC;CACJ;AAED,eAAe,MAAM,CAAC;AACtB,MAAM,CAAC,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC"}
|
package/build/src/main.d.ts
CHANGED
package/build/src/main.js
CHANGED
package/build/src/main.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/main.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC"}
|
|
1
|
+
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/main.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,WAAW,EAA0B,YAAY,EAAe,MAAM,aAAa,CAAC;AAC7F,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC"}
|
package/build/src/result.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Rec, HttpMethod } from './types.js';
|
|
2
|
+
import * as cheerio from 'cheerio';
|
|
2
3
|
export type ConfigData = {
|
|
3
4
|
url: string;
|
|
4
5
|
retry: boolean;
|
|
@@ -95,8 +96,8 @@ export type ContextData = {
|
|
|
95
96
|
};
|
|
96
97
|
export type ResultData = {
|
|
97
98
|
browser_data: {
|
|
98
|
-
javascript_evaluation_result
|
|
99
|
-
js_scenario
|
|
99
|
+
javascript_evaluation_result?: string;
|
|
100
|
+
js_scenario?: {
|
|
100
101
|
duration: number;
|
|
101
102
|
executed: number;
|
|
102
103
|
response: any;
|
|
@@ -109,10 +110,10 @@ export type ResultData = {
|
|
|
109
110
|
success: boolean;
|
|
110
111
|
}>;
|
|
111
112
|
};
|
|
112
|
-
local_storage_data
|
|
113
|
-
session_storage_data
|
|
114
|
-
websockets
|
|
115
|
-
xhr_call
|
|
113
|
+
local_storage_data?: Rec<string>;
|
|
114
|
+
session_storage_data?: Rec<string>;
|
|
115
|
+
websockets?: Array<any>;
|
|
116
|
+
xhr_call?: Array<{
|
|
116
117
|
body?: string;
|
|
117
118
|
headers: Rec<string>;
|
|
118
119
|
method: string;
|
|
@@ -194,12 +195,14 @@ export declare class ScrapeResult {
|
|
|
194
195
|
context: ContextData;
|
|
195
196
|
result: ResultData;
|
|
196
197
|
uuid: string;
|
|
198
|
+
private _selector;
|
|
197
199
|
constructor(data: {
|
|
198
200
|
config: ConfigData;
|
|
199
201
|
context: ContextData;
|
|
200
202
|
result: ResultData;
|
|
201
203
|
uuid: string;
|
|
202
204
|
});
|
|
205
|
+
get selector(): cheerio.CheerioAPI;
|
|
203
206
|
}
|
|
204
207
|
export declare class AccountData {
|
|
205
208
|
acount: {
|
package/build/src/result.js
CHANGED
|
@@ -1,14 +1,26 @@
|
|
|
1
|
+
import * as errors from './errors.js';
|
|
2
|
+
import * as cheerio from 'cheerio';
|
|
1
3
|
export class ScrapeResult {
|
|
2
4
|
config;
|
|
3
5
|
context;
|
|
4
6
|
result;
|
|
5
7
|
uuid;
|
|
8
|
+
_selector;
|
|
6
9
|
constructor(data) {
|
|
7
10
|
this.config = data.config;
|
|
8
11
|
this.context = data.context;
|
|
9
12
|
this.result = data.result;
|
|
10
13
|
this.uuid = data.uuid;
|
|
11
14
|
}
|
|
15
|
+
get selector() {
|
|
16
|
+
if (!this._selector) {
|
|
17
|
+
if (!this.result.response_headers['content-type'].includes('text/html')) {
|
|
18
|
+
throw new errors.ContentTypeError(`Cannot use selector on non-html content-type, received: ${this.result.response_headers['content-type']}`);
|
|
19
|
+
}
|
|
20
|
+
this._selector = cheerio.load(this.result.content);
|
|
21
|
+
}
|
|
22
|
+
return this._selector;
|
|
23
|
+
}
|
|
12
24
|
}
|
|
13
25
|
export class AccountData {
|
|
14
26
|
acount;
|
package/build/src/result.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"result.js","sourceRoot":"","sources":["../../src/result.ts"],"names":[],"mappings":"AA+
|
|
1
|
+
{"version":3,"file":"result.js","sourceRoot":"","sources":["../../src/result.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AA+LnC,MAAM,OAAO,YAAY;IACrB,MAAM,CAAa;IACnB,OAAO,CAAc;IACrB,MAAM,CAAa;IACnB,IAAI,CAAS;IACL,SAAS,CAAqB;IAEtC,YAAY,IAAoF;QAC5F,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IAC1B,CAAC;IAED,IAAI,QAAQ;QACR,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;YACjB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;gBACrE,MAAM,IAAI,MAAM,CAAC,gBAAgB,CAC7B,2DAA2D,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAC5G,CAAC;aACL;YACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SACtD;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;CACJ;AAED,MAAM,OAAO,WAAW;IACpB,MAAM,CAIJ;IACF,OAAO,CAWL;IACF,YAAY,CA2BV;CACL"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { urlsafe_b64encode } from './utils.js';
|
|
2
|
-
import { log } from './
|
|
2
|
+
import { log } from './logger.js';
|
|
3
3
|
import { ScrapeConfigError } from './errors.js';
|
|
4
|
-
class ScrapeConfig {
|
|
4
|
+
export class ScrapeConfig {
|
|
5
5
|
static PUBLIC_DATACENTER_POOL = 'public_datacenter_pool';
|
|
6
6
|
static PUBLIC_RESIDENTIAL_POOL = 'public_residential_pool';
|
|
7
7
|
url;
|
|
@@ -237,5 +237,4 @@ class ScrapeConfig {
|
|
|
237
237
|
return params;
|
|
238
238
|
}
|
|
239
239
|
}
|
|
240
|
-
export { ScrapeConfig };
|
|
241
240
|
//# sourceMappingURL=scrapeconfig.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapeconfig.js","sourceRoot":"","sources":["../../src/scrapeconfig.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,
|
|
1
|
+
{"version":3,"file":"scrapeconfig.js","sourceRoot":"","sources":["../../src/scrapeconfig.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,MAAM,OAAO,YAAY;IACrB,MAAM,CAAC,sBAAsB,GAAG,wBAAwB,CAAC;IACzD,MAAM,CAAC,uBAAuB,GAAG,yBAAyB,CAAC;IAE3D,GAAG,CAAS;IACZ,KAAK,GAAG,IAAI,CAAC;IACb,MAAM,GAAe,KAAK,CAAC;IAC3B,OAAO,GAAY,IAAI,CAAC;IACxB,SAAS,GAAG,KAAK,CAAC;IAClB,KAAK,GAAG,KAAK,CAAC;IACd,WAAW,GAAG,KAAK,CAAC;IACpB,GAAG,GAAG,KAAK,CAAC;IACZ,GAAG,GAAG,KAAK,CAAC;IACZ,GAAG,GAAG,KAAK,CAAC;IACZ,KAAK,GAAG,KAAK,CAAC;IACd,uBAAuB,GAAG,IAAI,CAAC;IAC/B,SAAS,GAAY,IAAI,CAAC;IAC1B,UAAU,GAAY,IAAI,CAAC;IAC3B,OAAO,GAAY,IAAI,CAAC;IACxB,IAAI,GAAgB,IAAI,GAAG,EAAU,CAAC;IACtC,cAAc,GAAY,IAAI,CAAC;IAC/B,OAAO,GAAiB,IAAI,CAAC;IAC7B,IAAI,GAAY,IAAI,CAAC;IACrB,IAAI,GAAc,IAAI,CAAC;IACvB,OAAO,GAAiB,IAAI,CAAC;IAC7B,EAAE,GAAY,IAAI,CAAC;IACnB,cAAc,GAAY,IAAI,CAAC;IAC/B,iBAAiB,GAAY,IAAI,CAAC;IAClC,oBAAoB,GAAG,KAAK,CAAC;IAC7B,WAAW,GAAc,IAAI,CAAC;IAC9B,OAAO,GAAY,IAAI,CAAC;IACxB,OAAO,GAAY,IAAI,CAAC,CAAC,kBAAkB;IAC3C,WAAW,GAAc,IAAI,CAAC;IAC9B,OAAO,GAAc,IAAI,CAAC;IAC1B,IAAI,GAAc,IAAI,CAAC;IACvB,EAAE,GAAY,IAAI,CAAC;IACnB,WAAW,GAAa,IAAI,CAAC;IAE7B,YAAY,OAkCX;QACG,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACzC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;QAC5C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;QAC/C,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,IAAI,IAAI,CAAC,oBAAoB,CAAC;QACtF,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC;QAC3D,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO;YAC1B,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3F,CAAC,CAAC,EAAE,CAAC;QACT,IAAI,CAAC,uBAAuB,GAAG,OAAO,CAAC,uBAAuB,IAAI,IAAI,CAAC,uBAAuB,CAAC;QAC/F,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;QAC/C,IAAI,CAAC,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC;QAC/C,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC;QACpE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO;YAC1B,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3F,CAAC,CAAC,EAAE,CAAC;QACT,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC;QACtC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC;QACtC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC;QACpE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB,CAAC;QAC7E,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC;QAC3D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;QAC/C,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC;QAC3D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;QAC/C,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC;QAC3D,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACnC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACzC,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE;YACxB,MAAM,IAAI,iBAAiB,CAAC,+BAA+B,CAAC,CAAC;SAChE;QACD,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YAChD,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBACzB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;oBAC/B,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,mCAAmC,CAAC;oBACnE,IAAI,CAAC,IAAI,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;iBACzD;qBAAM;oBACH,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,kBAAkB,CAAC,EAAE;wBAC5D,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;qBACzC;yBAAM,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,mCAAmC,CAAC,EAAE;wBACpF,IAAI,CAAC,IAAI,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;qBACzD;yBAAM;wBACH,MAAM,IAAI,iBAAiB,CACvB,iBAAiB,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,6FAA6F,CAC7I,CAAC;qBACL;iBACJ;aACJ;iBAAM,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBAChC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,YAAY,CAAC;aAC/C;SACJ;IACL,CAAC;IAED,WAAW,CAAC,OAAwB;QAChC,MAAM,MAAM,GAAwB;YAChC,GAAG,EAAE,OAAO,CAAC,GAAG;SACnB,CAAC;QACF,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;SACjC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;gBAClD,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC;YACtC,CAAC,CAAC,CAAC;SACN;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;YACtC,MAAM,eAAe,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBACpD,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;iBACxC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEhB,IAAI,MAAM,CAAC,iBAAiB,CAAC,EAAE;gBAC3B,iEAAiE;gBACjE,IAAI,MAAM,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE;oBACzE,MAAM,CAAC,iBAAiB,CAAC,IAAI,GAAG,CAAC;iBACpC;gBACD,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,eAAe,EAAE,CAAC;aACtD;iBAAM;gBACH,MAAM,CAAC,iBAAiB,CAAC,GAAG,eAAe,CAAC;aAC/C;SACJ;QACD,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC;SACtC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;SACjC;QACD,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE;YACzB,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;YACxB,IAAI,IAAI,CAAC,iBAAiB,KAAK,IAAI,EAAE;gBACjC,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC;aACrD;YACD,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,EAAE;gBAClB,MAAM,CAAC,EAAE,GAAG,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aAC1C;YACD,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE;gBAC3B,MAAM,CAAC,WAAW,GAAG,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;aAC5E;YACD,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,EAAE;gBAC9B,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;aAC/C;YACD,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC1C,MAAM,CAAC,eAAe,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBAC1D,CAAC,CAAC,CAAC;aACN;YACD,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE;gBAC3B,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;aACzC;SACJ;aAAM;YACH,IAAI,IAAI,CAAC,iBAAiB,KAAK,IAAI,EAAE;gBACjC,GAAG,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;aACzF;YACD,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE;gBAC3B,GAAG,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;aACnF;YACD,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE;gBAC3B,GAAG,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;aACnF;YACD,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,EAAE;gBAClB,GAAG,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;aAC1E;YACD,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,EAAE;gBAC9B,GAAG,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;aACtF;SACJ;QAED,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,EAAE;YACnB,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC;SACrB;QACD,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK,EAAE;YACtB,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;SACxB;QACD,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE;YACrB,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;YACpB,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC;aAC7B;YACD,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;aACrC;SACJ;aAAM;YACH,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,GAAG,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;aAC/E;YACD,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,GAAG,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;aAC7E;SACJ;QACD,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,EAAE;YACnB,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC;SACrB;QACD,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,EAAE;YACnB,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC;SACrB;QACD,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE;YACpB,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACjD;QACD,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;SAC/C;QACD,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC9B,IAAI,IAAI,CAAC,oBAAoB,EAAE;gBAC3B,MAAM,CAAC,oBAAoB,GAAG,IAAI,CAAC;aACtC;SACJ;aAAM;YACH,IAAI,IAAI,CAAC,oBAAoB,EAAE;gBAC3B,GAAG,CAAC,IAAI,CAAC,4EAA4E,CAAC,CAAC;aAC1F;SACJ;QACD,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE;YACrB,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;SACvB;QACD,IAAI,IAAI,CAAC,UAAU,EAAE;YACjB,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;SACvC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE;YACpB,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACrC;QACD,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,EAAE;YAClB,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;SACvB;QAED,8BAA8B;QAC9B,OAAO,MAAM,CAAC;IAClB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scrapfly-sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "SDK for Scrapfly.io web scraping service",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"types": "build/src/main.d.ts",
|
|
@@ -21,10 +21,9 @@
|
|
|
21
21
|
"build/src"
|
|
22
22
|
],
|
|
23
23
|
"engines": {
|
|
24
|
-
"node": ">= 18
|
|
24
|
+
"node": ">= 18 <19"
|
|
25
25
|
},
|
|
26
26
|
"devDependencies": {
|
|
27
|
-
"@types/jest": "~29.5",
|
|
28
27
|
"@types/node": "~18",
|
|
29
28
|
"@typescript-eslint/eslint-plugin": "~5.59",
|
|
30
29
|
"@typescript-eslint/parser": "~5.59",
|
|
@@ -36,7 +35,7 @@
|
|
|
36
35
|
"rimraf": "~5.0",
|
|
37
36
|
"ts-api-utils": "~0.0.44",
|
|
38
37
|
"ts-jest": "~29.1",
|
|
39
|
-
"typescript": "
|
|
38
|
+
"typescript": "^5.1.6"
|
|
40
39
|
},
|
|
41
40
|
"scripts": {
|
|
42
41
|
"clean": "rimraf coverage build tmp",
|
|
@@ -52,10 +51,10 @@
|
|
|
52
51
|
"author": "Bernardas Alisauskas <bernardas@scrapfly.io>",
|
|
53
52
|
"license": "BSD",
|
|
54
53
|
"dependencies": {
|
|
54
|
+
"@types/cheerio": "^0.22.31",
|
|
55
55
|
"axios": "^1.4.0",
|
|
56
|
-
"
|
|
57
|
-
"tslib": "~2.5"
|
|
58
|
-
"tslog": "^4.8.2"
|
|
56
|
+
"cheerio": "^1.0.0-rc.12",
|
|
57
|
+
"tslib": "~2.5"
|
|
59
58
|
},
|
|
60
59
|
"volta": {
|
|
61
60
|
"node": "18.17.0"
|