rezo 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -25
- package/dist/adapters/entries/curl.cjs +4 -4
- package/dist/adapters/entries/fetch.cjs +2 -2
- package/dist/adapters/entries/http.cjs +2 -2
- package/dist/adapters/entries/http2.cjs +4 -4
- package/dist/adapters/entries/react-native.cjs +2 -2
- package/dist/adapters/entries/xhr.cjs +2 -2
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +13 -13
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +16 -16
- package/dist/plugin/addon/decodo/types.cjs +1 -0
- package/dist/plugin/addon/decodo/types.js +1 -0
- package/dist/plugin/addon/oxylabs/types.cjs +1 -0
- package/dist/plugin/addon/oxylabs/types.js +1 -0
- package/dist/plugin/index.cjs +36 -36
- package/dist/plugin/scraper.cjs +1 -0
- package/dist/plugin/scraper.js +1 -0
- package/dist/utils/index.cjs +0 -1
- package/dist/utils/index.js +0 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<img src="assets/logo.svg" alt="Rezo HTTP Client" width="400">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/yuniqsolutions/rezo/main/assets/logo.svg" alt="Rezo HTTP Client" width="400">
|
|
3
3
|
</p>
|
|
4
4
|
|
|
5
5
|
<h1 align="center">Rezo HTTP Client</h1>
|
|
@@ -122,14 +122,17 @@ bun add rezo
|
|
|
122
122
|
```typescript
|
|
123
123
|
import rezo from 'rezo';
|
|
124
124
|
|
|
125
|
-
// Simple GET request
|
|
126
|
-
const
|
|
127
|
-
console.log(
|
|
125
|
+
// Simple GET request with destructured response
|
|
126
|
+
const { data, status, headers } = await rezo.get('https://api.example.com/users');
|
|
127
|
+
console.log(data); // Response body (auto-parsed JSON)
|
|
128
|
+
console.log(status); // 200
|
|
129
|
+
console.log(headers); // Response headers
|
|
128
130
|
|
|
129
131
|
// With query parameters
|
|
130
|
-
const users = await rezo.get('https://api.example.com/users', {
|
|
132
|
+
const { data: users } = await rezo.get('https://api.example.com/users', {
|
|
131
133
|
params: { page: 1, limit: 10 }
|
|
132
134
|
});
|
|
135
|
+
console.log(users);
|
|
133
136
|
```
|
|
134
137
|
|
|
135
138
|
### POST Request with JSON
|
|
@@ -137,13 +140,14 @@ const users = await rezo.get('https://api.example.com/users', {
|
|
|
137
140
|
```typescript
|
|
138
141
|
import rezo from 'rezo';
|
|
139
142
|
|
|
140
|
-
|
|
143
|
+
// Destructure the response for cleaner code
|
|
144
|
+
const { data, status, headers, config } = await rezo.post('https://api.example.com/users', {
|
|
141
145
|
name: 'John Doe',
|
|
142
146
|
email: 'john@example.com'
|
|
143
147
|
});
|
|
144
148
|
|
|
145
|
-
console.log(
|
|
146
|
-
console.log(
|
|
149
|
+
console.log(data); // Created user object
|
|
150
|
+
console.log(status); // 201
|
|
147
151
|
```
|
|
148
152
|
|
|
149
153
|
### Creating an Instance
|
|
@@ -1335,32 +1339,66 @@ await crawler.crawl('https://example.com');
|
|
|
1335
1339
|
|
|
1336
1340
|
## DOM Module
|
|
1337
1341
|
|
|
1338
|
-
Rezo
|
|
1342
|
+
Rezo provides a lightweight DOM parsing module for server-side HTML manipulation. Built on [linkedom](https://github.com/WebReflection/linkedom), it enables fast, standards-compliant DOM operations in Node, Bun, Deno or Edge environments - perfect for web scraping, HTML transformation, and testing.
|
|
1339
1343
|
|
|
1340
|
-
|
|
1341
|
-
import { parseHTML, DOMParser } from 'rezo/dom';
|
|
1344
|
+
### Parsing HTML
|
|
1342
1345
|
|
|
1343
|
-
|
|
1344
|
-
|
|
1346
|
+
```typescript
|
|
1347
|
+
import { parseHTML } from 'rezo/dom';
|
|
1348
|
+
|
|
1349
|
+
// Parse HTML and work with the document
|
|
1350
|
+
const { document, window } = parseHTML(`
|
|
1351
|
+
<html>
|
|
1352
|
+
<body>
|
|
1353
|
+
<h1>Hello World</h1>
|
|
1354
|
+
<ul id="items">
|
|
1355
|
+
<li>Item 1</li>
|
|
1356
|
+
<li>Item 2</li>
|
|
1357
|
+
</ul>
|
|
1358
|
+
</body>
|
|
1359
|
+
</html>
|
|
1360
|
+
`);
|
|
1361
|
+
|
|
1362
|
+
// Use familiar DOM APIs
|
|
1345
1363
|
console.log(document.querySelector('h1')?.textContent); // 'Hello World'
|
|
1364
|
+
console.log(document.querySelectorAll('li').length); // 2
|
|
1365
|
+
```
|
|
1366
|
+
|
|
1367
|
+
### Using with Rezo HTTP Client
|
|
1368
|
+
|
|
1369
|
+
```typescript
|
|
1370
|
+
import rezo from 'rezo';
|
|
1371
|
+
import { parseHTML } from 'rezo/dom';
|
|
1372
|
+
|
|
1373
|
+
// Fetch and parse a webpage
|
|
1374
|
+
const { data: html } = await rezo.get('https://example.com');
|
|
1375
|
+
const { document } = parseHTML(html);
|
|
1376
|
+
|
|
1377
|
+
// Extract data from the page
|
|
1378
|
+
const title = document.querySelector('title')?.textContent;
|
|
1379
|
+
const links = [...document.querySelectorAll('a')].map(a => a.getAttribute('href'));
|
|
1380
|
+
```
|
|
1381
|
+
|
|
1382
|
+
### DOMParser Interface
|
|
1383
|
+
|
|
1384
|
+
```typescript
|
|
1385
|
+
import { DOMParser } from 'rezo/dom';
|
|
1346
1386
|
|
|
1347
|
-
// Using DOMParser
|
|
1348
1387
|
const parser = new DOMParser();
|
|
1349
1388
|
const doc = parser.parseFromString('<div class="content">Text</div>', 'text/html');
|
|
1350
1389
|
console.log(doc.querySelector('.content')?.textContent); // 'Text'
|
|
1351
1390
|
```
|
|
1352
1391
|
|
|
1353
|
-
### Available
|
|
1354
|
-
|
|
1355
|
-
All linkedom exports are available from `rezo/dom`:
|
|
1392
|
+
### Available APIs
|
|
1356
1393
|
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1394
|
+
| Export | Description |
|
|
1395
|
+
|--------|-------------|
|
|
1396
|
+
| `parseHTML` | Parse HTML string into window/document objects |
|
|
1397
|
+
| `DOMParser` | Standard W3C DOMParser interface |
|
|
1398
|
+
| `Document` | Document constructor for creating new documents |
|
|
1399
|
+
| `Event` / `CustomEvent` | DOM Event interfaces |
|
|
1400
|
+
| `EventTarget` | Event handling interface |
|
|
1401
|
+
| `NodeList` | Standard NodeList interface |
|
|
1364
1402
|
|
|
1365
1403
|
---
|
|
1366
1404
|
|
|
@@ -1498,7 +1536,7 @@ See the [LICENSE](LICENSE) file for the complete license text.
|
|
|
1498
1536
|
---
|
|
1499
1537
|
|
|
1500
1538
|
<p align="center">
|
|
1501
|
-
<img src="assets/icon.svg" alt="Rezo Icon" width="60">
|
|
1539
|
+
<img src="https://raw.githubusercontent.com/yuniqsolutions/rezo/main/assets/icon.svg" alt="Rezo Icon" width="60">
|
|
1502
1540
|
</p>
|
|
1503
1541
|
|
|
1504
1542
|
<p align="center">
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeCurlRequest =
|
|
3
|
-
const
|
|
4
|
-
exports.executeHttpRequest =
|
|
1
|
+
const _mod_w0i5cb = require('../curl.cjs');
|
|
2
|
+
exports.executeCurlRequest = _mod_w0i5cb.executeRequest;;
|
|
3
|
+
const _mod_34vmei = require('../http.cjs');
|
|
4
|
+
exports.executeHttpRequest = _mod_34vmei.executeRequest;;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeRequest =
|
|
1
|
+
const _mod_secawm = require('../fetch.cjs');
|
|
2
|
+
exports.executeRequest = _mod_secawm.executeRequest;;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeRequest =
|
|
1
|
+
const _mod_a9efbu = require('../http.cjs');
|
|
2
|
+
exports.executeRequest = _mod_a9efbu.executeRequest;;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeHttp2Request =
|
|
3
|
-
const
|
|
4
|
-
exports.executeHttpRequest =
|
|
1
|
+
const _mod_qsy7lz = require('../http2.cjs');
|
|
2
|
+
exports.executeHttp2Request = _mod_qsy7lz.executeRequest;;
|
|
3
|
+
const _mod_02wdv6 = require('../http.cjs');
|
|
4
|
+
exports.executeHttpRequest = _mod_02wdv6.executeRequest;;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeRequest =
|
|
1
|
+
const _mod_djh9r7 = require('../react-native.cjs');
|
|
2
|
+
exports.executeRequest = _mod_djh9r7.executeRequest;;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeRequest =
|
|
1
|
+
const _mod_xprmqe = require('../xhr.cjs');
|
|
2
|
+
exports.executeRequest = _mod_xprmqe.executeRequest;;
|
package/dist/adapters/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.detectRuntime =
|
|
3
|
-
exports.getAdapterCapabilities =
|
|
4
|
-
exports.buildAdapterContext =
|
|
5
|
-
exports.getAvailableAdapters =
|
|
6
|
-
exports.selectAdapter =
|
|
1
|
+
const _mod_rxwmdm = require('./picker.cjs');
|
|
2
|
+
exports.detectRuntime = _mod_rxwmdm.detectRuntime;
|
|
3
|
+
exports.getAdapterCapabilities = _mod_rxwmdm.getAdapterCapabilities;
|
|
4
|
+
exports.buildAdapterContext = _mod_rxwmdm.buildAdapterContext;
|
|
5
|
+
exports.getAvailableAdapters = _mod_rxwmdm.getAvailableAdapters;
|
|
6
|
+
exports.selectAdapter = _mod_rxwmdm.selectAdapter;;
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
10
|
-
const
|
|
11
|
-
exports.FileCacher =
|
|
12
|
-
const
|
|
13
|
-
exports.UrlStore =
|
|
1
|
+
const _mod_19hka7 = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_19hka7.LRUCache;;
|
|
3
|
+
const _mod_dvaqns = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_dvaqns.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_dvaqns.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_dvaqns.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_dnu91e = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_dnu91e.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_dnu91e.normalizeResponseCacheConfig;;
|
|
10
|
+
const _mod_0174no = require('./file-cacher.cjs');
|
|
11
|
+
exports.FileCacher = _mod_0174no.FileCacher;;
|
|
12
|
+
const _mod_4o34b7 = require('./url-store.cjs');
|
|
13
|
+
exports.UrlStore = _mod_4o34b7.UrlStore;;
|
package/dist/entries/crawler.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
exports.Domain =
|
|
1
|
+
const _mod_z4k9gt = require('../plugin/crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_z4k9gt.Crawler;;
|
|
3
|
+
const _mod_7kj7fl = require('../plugin/crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_7kj7fl.CrawlerOptions;
|
|
5
|
+
exports.Domain = _mod_7kj7fl.Domain;;
|
package/dist/index.cjs
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Rezo =
|
|
3
|
-
exports.createRezoInstance =
|
|
4
|
-
exports.createDefaultInstance =
|
|
5
|
-
const
|
|
6
|
-
exports.RezoError =
|
|
7
|
-
exports.RezoErrorCode =
|
|
8
|
-
const
|
|
9
|
-
exports.RezoHeaders =
|
|
10
|
-
const
|
|
11
|
-
exports.RezoFormData =
|
|
12
|
-
const
|
|
13
|
-
exports.RezoCookieJar =
|
|
14
|
-
const
|
|
15
|
-
exports.createDefaultHooks =
|
|
16
|
-
exports.mergeHooks =
|
|
1
|
+
const _mod_pevlcv = require('./core/rezo.cjs');
|
|
2
|
+
exports.Rezo = _mod_pevlcv.Rezo;
|
|
3
|
+
exports.createRezoInstance = _mod_pevlcv.createRezoInstance;
|
|
4
|
+
exports.createDefaultInstance = _mod_pevlcv.createDefaultInstance;;
|
|
5
|
+
const _mod_82va86 = require('./errors/rezo-error.cjs');
|
|
6
|
+
exports.RezoError = _mod_82va86.RezoError;
|
|
7
|
+
exports.RezoErrorCode = _mod_82va86.RezoErrorCode;;
|
|
8
|
+
const _mod_q5wnhn = require('./utils/headers.cjs');
|
|
9
|
+
exports.RezoHeaders = _mod_q5wnhn.RezoHeaders;;
|
|
10
|
+
const _mod_pklwtk = require('./utils/form-data.cjs');
|
|
11
|
+
exports.RezoFormData = _mod_pklwtk.RezoFormData;;
|
|
12
|
+
const _mod_05uuoj = require('./utils/cookies.cjs');
|
|
13
|
+
exports.RezoCookieJar = _mod_05uuoj.RezoCookieJar;;
|
|
14
|
+
const _mod_yizls7 = require('./core/hooks.cjs');
|
|
15
|
+
exports.createDefaultHooks = _mod_yizls7.createDefaultHooks;
|
|
16
|
+
exports.mergeHooks = _mod_yizls7.mergeHooks;;
|
|
17
17
|
const { RezoError } = require('./errors/rezo-error.cjs');
|
|
18
18
|
const isRezoError = exports.isRezoError = RezoError.isRezoError;
|
|
19
19
|
const Cancel = exports.Cancel = RezoError;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports={};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
var c={};export{c as default};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports={};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
var c={};export{c as default};
|
package/dist/plugin/index.cjs
CHANGED
|
@@ -1,36 +1,36 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
const
|
|
6
|
-
exports.FileCacher =
|
|
7
|
-
const
|
|
8
|
-
exports.UrlStore =
|
|
9
|
-
const
|
|
10
|
-
exports.Oxylabs =
|
|
11
|
-
const
|
|
12
|
-
exports.OXYLABS_BROWSER_TYPES =
|
|
13
|
-
exports.OXYLABS_COMMON_LOCALES =
|
|
14
|
-
exports.OXYLABS_COMMON_GEO_LOCATIONS =
|
|
15
|
-
exports.OXYLABS_US_STATES =
|
|
16
|
-
exports.OXYLABS_EUROPEAN_COUNTRIES =
|
|
17
|
-
exports.OXYLABS_ASIAN_COUNTRIES =
|
|
18
|
-
exports.getRandomOxylabsBrowserType =
|
|
19
|
-
exports.getRandomOxylabsLocale =
|
|
20
|
-
exports.getRandomOxylabsGeoLocation =
|
|
21
|
-
const
|
|
22
|
-
exports.Decodo =
|
|
23
|
-
const
|
|
24
|
-
exports.DECODO_DEVICE_TYPES =
|
|
25
|
-
exports.DECODO_HEADLESS_MODES =
|
|
26
|
-
exports.DECODO_COMMON_LOCALES =
|
|
27
|
-
exports.DECODO_COMMON_COUNTRIES =
|
|
28
|
-
exports.DECODO_EUROPEAN_COUNTRIES =
|
|
29
|
-
exports.DECODO_ASIAN_COUNTRIES =
|
|
30
|
-
exports.DECODO_US_STATES =
|
|
31
|
-
exports.DECODO_COMMON_CITIES =
|
|
32
|
-
exports.getRandomDecodoDeviceType =
|
|
33
|
-
exports.getRandomDecodoLocale =
|
|
34
|
-
exports.getRandomDecodoCountry =
|
|
35
|
-
exports.getRandomDecodoCity =
|
|
36
|
-
exports.generateDecodoSessionId =
|
|
1
|
+
const _mod_t1dtd7 = require('./crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_t1dtd7.Crawler;;
|
|
3
|
+
const _mod_id8gq1 = require('./crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_id8gq1.CrawlerOptions;;
|
|
5
|
+
const _mod_nd5row = require('../cache/file-cacher.cjs');
|
|
6
|
+
exports.FileCacher = _mod_nd5row.FileCacher;;
|
|
7
|
+
const _mod_9cw32z = require('../cache/url-store.cjs');
|
|
8
|
+
exports.UrlStore = _mod_9cw32z.UrlStore;;
|
|
9
|
+
const _mod_w9mqpo = require('./addon/oxylabs/index.cjs');
|
|
10
|
+
exports.Oxylabs = _mod_w9mqpo.Oxylabs;;
|
|
11
|
+
const _mod_25im63 = require('./addon/oxylabs/options.cjs');
|
|
12
|
+
exports.OXYLABS_BROWSER_TYPES = _mod_25im63.OXYLABS_BROWSER_TYPES;
|
|
13
|
+
exports.OXYLABS_COMMON_LOCALES = _mod_25im63.OXYLABS_COMMON_LOCALES;
|
|
14
|
+
exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_25im63.OXYLABS_COMMON_GEO_LOCATIONS;
|
|
15
|
+
exports.OXYLABS_US_STATES = _mod_25im63.OXYLABS_US_STATES;
|
|
16
|
+
exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_25im63.OXYLABS_EUROPEAN_COUNTRIES;
|
|
17
|
+
exports.OXYLABS_ASIAN_COUNTRIES = _mod_25im63.OXYLABS_ASIAN_COUNTRIES;
|
|
18
|
+
exports.getRandomOxylabsBrowserType = _mod_25im63.getRandomBrowserType;
|
|
19
|
+
exports.getRandomOxylabsLocale = _mod_25im63.getRandomLocale;
|
|
20
|
+
exports.getRandomOxylabsGeoLocation = _mod_25im63.getRandomGeoLocation;;
|
|
21
|
+
const _mod_rwnxp1 = require('./addon/decodo/index.cjs');
|
|
22
|
+
exports.Decodo = _mod_rwnxp1.Decodo;;
|
|
23
|
+
const _mod_ztue8d = require('./addon/decodo/options.cjs');
|
|
24
|
+
exports.DECODO_DEVICE_TYPES = _mod_ztue8d.DECODO_DEVICE_TYPES;
|
|
25
|
+
exports.DECODO_HEADLESS_MODES = _mod_ztue8d.DECODO_HEADLESS_MODES;
|
|
26
|
+
exports.DECODO_COMMON_LOCALES = _mod_ztue8d.DECODO_COMMON_LOCALES;
|
|
27
|
+
exports.DECODO_COMMON_COUNTRIES = _mod_ztue8d.DECODO_COMMON_COUNTRIES;
|
|
28
|
+
exports.DECODO_EUROPEAN_COUNTRIES = _mod_ztue8d.DECODO_EUROPEAN_COUNTRIES;
|
|
29
|
+
exports.DECODO_ASIAN_COUNTRIES = _mod_ztue8d.DECODO_ASIAN_COUNTRIES;
|
|
30
|
+
exports.DECODO_US_STATES = _mod_ztue8d.DECODO_US_STATES;
|
|
31
|
+
exports.DECODO_COMMON_CITIES = _mod_ztue8d.DECODO_COMMON_CITIES;
|
|
32
|
+
exports.getRandomDecodoDeviceType = _mod_ztue8d.getRandomDeviceType;
|
|
33
|
+
exports.getRandomDecodoLocale = _mod_ztue8d.getRandomLocale;
|
|
34
|
+
exports.getRandomDecodoCountry = _mod_ztue8d.getRandomCountry;
|
|
35
|
+
exports.getRandomDecodoCity = _mod_ztue8d.getRandomCity;
|
|
36
|
+
exports.generateDecodoSessionId = _mod_ztue8d.generateSessionId;;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
var{parseHTML:w}=require("linkedom");class x extends Set{maxSize;constructor(e){super();this.maxSize=e}add(e){if(this.has(e))return this;if(this.size>=this.maxSize){let n=this.values().next().value;if(n)this.delete(n)}return super.add(e)}}class v{http;httpOptions;onEmailLeads;onEmailDiscovered;debug;discoveredEmails=new x(1e4);userAgents=[];fileExtensions=[];restrictedDomains=$();forbiddenProtocols=["mailto:","tel:","javascript:","data:","sms:","ftp:","file:","irc:","blob:","chrome:","about:","intent:"];constructor(e,n,a,o,t=!1){this.http=e,this.httpOptions=n,this.onEmailLeads=a,this.onEmailDiscovered=o,this.debug=t,this.userAgents=y()}sleep(e){return new Promise((n)=>setTimeout(n,e))}async executeHttp(e,n,a,o,t,i=0){let{getCache:r,saveCache:c,hasUrlInCache:u,saveUrl:l,httpConfig:s={}}=o;if(!e||e.length<3||this.forbiddenProtocols.some((d)=>e.startsWith(d)))return;try{let d=t?!1:await u(e),h=await r(e);if(d&&!h)return!1;if(d&&n!=="GET")return!1;let m=h&&n==="GET"?h:await(n==="GET"?this.http.get(e,s):n==="PATCH"?this.http.patch(e,a,s):n==="POST"?this.http.post(e,a,s):this.http.put(e,a,s));if(!h)await c(e,{data:m.data,contentType:m.contentType,finalUrl:m.finalUrl});if(!d)await l(e);if(!m.contentType||!m.contentType.includes("/html")||!m.contentType.includes("text/")||typeof m.data!=="string")return null;return{data:m.data,contentType:m.contentType,finalUrl:m.finalUrl}}catch(d){let h=d,m=this.httpOptions;if(h&&h.response){let g=h.response.status,p=m.retryDelay||100,b=m.maxRetryAttempts||3,f=m.retryWithoutProxyOnStatusCode||void 0,k=m.maxRetryOnProxyError||3;if(f&&s.proxy&&f.includes(g)&&i<b)return await this.sleep(p),delete s.proxy,await this.executeHttp(e,n,a,o,t,i+1);else if(m.retryOnStatusCode&&s.proxy&&m.retryOnStatusCode.includes(g)&&i<b)return await this.sleep(p),await this.executeHttp(e,n,a,o,t,i+1);else if(m.retryOnProxyError&&s.proxy&&i<k)return await this.sleep(p),await this.executeHttp(e,n,a,o,t,i+1)}if(this.debug){if(this.debug)console.log(`Error: unable to ${n} ${e}: ${d.message}`)}return null}}extractEmails(e,n,a,o,t){let i=this.extractEmailsFromContent(e?.replaceAll("mailto:"," ")),r=[];for(let c of i)if(this.handleEmailDiscovery(c,n,a,t))r.push(c);if(o&&o.length>0&&r.length>0)t.add(async()=>Promise.all(o.map((c)=>c(r))));i.length=0,r.length=0}async parseExternalWebsite(e,n,a,o,t,i=!0,r,c){let u=o.httpConfig?.headers?o.httpConfig.headers instanceof Headers?Object.fromEntries(o.httpConfig.headers.entries()):o.httpConfig.headers:{};o.httpConfig=o.httpConfig||{},o.httpConfig.headers={"user-agent":this.getRandomUserAgent(),accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","cache-control":"no-cache",pragma:"no-cache",...u},o.httpConfig.timeout=o.httpConfig.timeout||15000,o.depth=o.depth||0,o.allowCrossDomainTravel=o.allowCrossDomainTravel||!1,t=t&&i;let l=[];try{let s=new URL(e),d=this.extractRootDomain(e);if(this.isLinktreeUrl(e))return await this.parseLinktreeProfile(e,o,t);if(this.isRestrictedDomain(e)){if(this.debug)console.warn(`⚠️ Skipped URL (restricted url): ${e}`);return l}let h=await this.executeHttp(e,n,a,o,t);if(!h){if(this.debug&&h===null)console.warn(`⚠️ Failed to fetch page content: ${e}`);if(this.debug&&h===!1)console.warn(`⚠️ Skipped URL (already visited): ${e}`);return l}let m=this.extractEmailsFromContent(h.data?.replaceAll("mailto:"," "));for(let g of m)if(this.handleEmailDiscovery(g,e,o.onEmailDiscovered,o.queue))l.push(g);if(o.depth>0||!r){let g=w(h.data).document,p=this.extractRelevantLinks(g,s,d,o.depth,o.allowCrossDomainTravel);o.depth--;let b=await Promise.allSettled(p.map((f)=>this.parseExternalWebsite(f,"GET",null,{...o,depth:o.depth},t,!1,!0)));for(let f of b)if(f.status==="fulfilled")l.push(...f.value);else if(this.debug)console.warn("⚠️ Failed to parse child URL:",f.reason?.message)}}catch(s){if(this.debug)console.error(`❌ Error parsing external website: ${e}`,s?.message)}if(i){if(o.onEmails&&o.onEmails.length>0)o.queue.add(async()=>Promise.all(o.onEmails.map((s)=>s(l))))}return l}async parseLinktreeProfile(e,n,a){let o=[];try{let t=await this.executeHttp(e,"GET",null,n,a);if(!t){if(this.debug)console.warn(`⚠️ Failed to fetch Linktree profile: ${e}`);return o}let r=w(t).document.getElementById("links-container");if(!r){if(this.debug)console.warn(`\uD83D\uDD0D No links container found in Linktree profile: ${e}`);return o}let c=this.extractLinktreeExternalUrls(r,e);if(c.length===0){if(this.debug)console.info("\uD83D\uDCED No valid external links found in Linktree profile");return o}if(this.debug)console.info(`\uD83C\uDFAF Found ${c.length} external links in Linktree profile`);let u=await Promise.allSettled(c.map((l)=>this.parseExternalWebsite(l,"GET",null,n,a,!1)));for(let l of u)if(l.status==="fulfilled")o.push(...l.value);else if(this.debug)console.warn("⚠️ Failed to parse Linktree external URL:",l.reason?.message)}catch(t){if(this.debug)console.error(`❌ Error parsing Linktree profile: ${e}`,t?.message)}return o}extractLinktreeExternalUrls(e,n){let a=new Set,o=e.querySelectorAll("a[href][target='_blank']");for(let t of o){let i=t.getAttribute("href");if(!i||i.length<3||this.forbiddenProtocols.some((r)=>i.startsWith(r)))continue;try{let r=new URL(i,n).href,c=this.extractRootDomain(r);if(c!=="linktr.ee"&&!this.isRestrictedDomain(r)&&c.length>3)a.add(r)}catch(r){if(this.debug)console.warn(`\uD83D\uDD17 Invalid URL in Linktree: ${i}`)}}return Array.from(a)}handleEmailDiscovery(e,n,a,o){if(!this.discoveredEmails.has(e)){this.discoveredEmails.add(e);let t={email:e,discoveredAt:n,timestamp:new Date};if(a&&a.length>0)o.add(async()=>Promise.all(a.map((i)=>i(t))));if(this.debug)console.info(`\uD83D\uDCE7 New email discovered: ${e} at ${n}`);return!0}return!1}isDomainAccessAllowed(e,n,a,o){if(o)return!0;if(a===0)return e===n;return e===n||e.endsWith(`.${n}`)||n.endsWith(`.${e}`)}extractRelevantLinks(e,n,a,o,t){let i=[],r=["about","contact","help","support","reach","email","mail","message","company","team","staff","info","inquiry","feedback","service","assistance","connect","touch"],c=e.querySelectorAll("a[href]");for(let u of c){let l=u.getAttribute("href");if(!l||l.length<2)continue;try{let s=this.normalizeUrl(l,n),d=this.extractRootDomain(s);if(!this.isDomainAccessAllowed(d,a,o,t))continue;if(r.some((m)=>s.toLowerCase().includes(m))||this.isLinktreeUrl(s))i.push(s)}catch(s){if(this.debug)console.warn(`\uD83D\uDD17 Invalid link found: ${l}`,s?.message)}}return i}extractEmailsFromContent(e){let n=e.replace(/[^\w@.-\s]/g," "),a=/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g,o=(c)=>{let u=/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/,l=c.split("@")[1]?.toLowerCase(),s=c.split(".").pop()?.toLowerCase();return u.test(c)&&l!==void 0&&s!==void 0&&!this.fileExtensions.includes(`.${s}`)&&!this.isRestrictedDomain(`https://${l}`)},t=(c)=>{return(c.match(a)||[]).filter(o)},i=e.replace(/<[^>]*>/g," "),r=[...t(i),...t(n)];return[...new Set(r)]}isRestrictedDomain(e){try{let n=new URL(e).host.toLowerCase();return this.restrictedDomains.some((a)=>n===a.toLowerCase()||n.endsWith(`.${a.toLowerCase()}`))}catch{return!0}}isLinktreeUrl(e){try{return this.extractRootDomain(e)==="linktr.ee"}catch{return!1}}extractRootDomain(e){try{let a=new URL(e).hostname.toLowerCase();return a.startsWith("www.")?a.slice(4):a}catch{return""}}normalizeUrl(e,n){if(e.startsWith("http://")||e.startsWith("https://"))return e;if(e.startsWith("//"))return`${n.protocol}${e}`;return new URL(e,n.href).href}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function y(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],n=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],a=[];for(let o=0;o<200;o++){let t=e[Math.floor(Math.random()*e.length)],i=n[Math.floor(Math.random()*n.length)],r="";switch(t.name){case"Chrome":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":r=`Mozilla/5.0 (${i}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}a.push(r)}return a}function $(){return["facebook.com","fb.com","messenger.com","instagram.com","threads.net","twitter.com","x.com","linkedin.com","pinterest.com","pin.it","reddit.com","tumblr.com","snapchat.com","tiktok.com","douyin.com","youtube.com","youtu.be","whatsapp.com","telegram.org","t.me","medium.com","quora.com","flickr.com","vimeo.com","vk.com","weibo.com","sina.com.cn","line.me","discord.com","discordapp.com","twitch.tv","meetup.com","nextdoor.com","xing.com","yelp.com","zalo.me","mastodon.social","clubhouse.com","patreon.com","onlyfans.com","douban.com","goodreads.com","soundcloud.com","spotify.com","last.fm","behance.net","dribbble.com","deviantart.com","pixiv.net","slideshare.net","tinder.com","bumble.com","etsy.com","indeed.com","glassdoor.com","monster.com","careerbuilder.com","dice.com","ziprecruiter.com","simplyhired.com","upwork.com","freelancer.com","fiverr.com","stackoverflow.com","stackoverflow.co","angel.co","wellfound.com","quora.com","stackexchange.com","yahoo.com","answers.microsoft.com","askubuntu.com","superuser.com","serverfault.com","mathoverflow.net","xda-developers.com","gamespot.com","ign.com","4chan.org","9gag.com","gizmodo.com","slashdot.org","hacker-news.news","ycombinator.com","producthunt.com","discourse.org","google.com","google.co.uk","google.de","google.fr","google.co.jp","bing.com","yahoo.com","search.yahoo.com","duckduckgo.com","baidu.com","yandex.com","yandex.ru","ask.com","wolframalpha.com","ecosia.org","startpage.com","qwant.com","searx.me","gibiru.com","swisscows.com","gmail.com","googlemail.com","outlook.com","hotmail.com","live.com","msn.com","yahoo.com","ymail.com","aol.com","icloud.com","me.com","mac.com","protonmail.com","pm.me","zoho.com","mail.com","gmx.com","gmx.net","yandex.com","yandex.ru","tutanota.com","tutanota.de","fastmail.com","hushmail.com","mailbox.org","posteo.de","runbox.com","disroot.org","163.com","qq.com","rambler.ru","mail.ru","yelp.com","yelp.ca","yelp.co.uk","yelp.com.au","yellowpages.com","yellowpages.ca","yell.com","tripadvisor.com","tripadvisor.co.uk","tripadvisor.ca","foursquare.com","angieslist.com","bbb.org","manta.com","thumbtack.com","homeadvisor.com","superpages.com","whitepages.com","local.com","citysearch.com","merchantcircle.com","insiderpages.com","kudzu.com","hotfrog.com","buildzoom.com","houzz.com","porch.com","mapquest.com","zagat.com","zomato.com","opentable.com","viator.com","expedia.com","booking.com","airbnb.com","vrbo.com","homeaway.com","craigslist.org","nextdoor.com","patch.com","meetup.com","eventbrite.com","groupon.com","livingsocial.com","gumtree.com","gumtree.com.au","kijiji.ca","leboncoin.fr","finn.no","blocket.se","58.com","dianping.com","tabelog.com","ypcdn.com"]}exports.Scraper=v;exports.CappedSet=x;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{parseHTML as x}from"linkedom";class w extends Set{maxSize;constructor(e){super();this.maxSize=e}add(e){if(this.has(e))return this;if(this.size>=this.maxSize){let n=this.values().next().value;if(n)this.delete(n)}return super.add(e)}}class k{http;httpOptions;onEmailLeads;onEmailDiscovered;debug;discoveredEmails=new w(1e4);userAgents=[];fileExtensions=[];restrictedDomains=$();forbiddenProtocols=["mailto:","tel:","javascript:","data:","sms:","ftp:","file:","irc:","blob:","chrome:","about:","intent:"];constructor(e,n,a,o,t=!1){this.http=e,this.httpOptions=n,this.onEmailLeads=a,this.onEmailDiscovered=o,this.debug=t,this.userAgents=y()}sleep(e){return new Promise((n)=>setTimeout(n,e))}async executeHttp(e,n,a,o,t,i=0){let{getCache:r,saveCache:c,hasUrlInCache:u,saveUrl:l,httpConfig:s={}}=o;if(!e||e.length<3||this.forbiddenProtocols.some((d)=>e.startsWith(d)))return;try{let d=t?!1:await u(e),h=await r(e);if(d&&!h)return!1;if(d&&n!=="GET")return!1;let m=h&&n==="GET"?h:await(n==="GET"?this.http.get(e,s):n==="PATCH"?this.http.patch(e,a,s):n==="POST"?this.http.post(e,a,s):this.http.put(e,a,s));if(!h)await c(e,{data:m.data,contentType:m.contentType,finalUrl:m.finalUrl});if(!d)await l(e);if(!m.contentType||!m.contentType.includes("/html")||!m.contentType.includes("text/")||typeof m.data!=="string")return null;return{data:m.data,contentType:m.contentType,finalUrl:m.finalUrl}}catch(d){let h=d,m=this.httpOptions;if(h&&h.response){let g=h.response.status,p=m.retryDelay||100,b=m.maxRetryAttempts||3,f=m.retryWithoutProxyOnStatusCode||void 0,v=m.maxRetryOnProxyError||3;if(f&&s.proxy&&f.includes(g)&&i<b)return await this.sleep(p),delete s.proxy,await this.executeHttp(e,n,a,o,t,i+1);else if(m.retryOnStatusCode&&s.proxy&&m.retryOnStatusCode.includes(g)&&i<b)return await this.sleep(p),await this.executeHttp(e,n,a,o,t,i+1);else if(m.retryOnProxyError&&s.proxy&&i<v)return await this.sleep(p),await this.executeHttp(e,n,a,o,t,i+1)}if(this.debug){if(this.debug)console.log(`Error: unable to ${n} ${e}: ${d.message}`)}return null}}extractEmails(e,n,a,o,t){let i=this.extractEmailsFromContent(e?.replaceAll("mailto:"," ")),r=[];for(let c of i)if(this.handleEmailDiscovery(c,n,a,t))r.push(c);if(o&&o.length>0&&r.length>0)t.add(async()=>Promise.all(o.map((c)=>c(r))));i.length=0,r.length=0}async parseExternalWebsite(e,n,a,o,t,i=!0,r,c){let u=o.httpConfig?.headers?o.httpConfig.headers instanceof Headers?Object.fromEntries(o.httpConfig.headers.entries()):o.httpConfig.headers:{};o.httpConfig=o.httpConfig||{},o.httpConfig.headers={"user-agent":this.getRandomUserAgent(),accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","cache-control":"no-cache",pragma:"no-cache",...u},o.httpConfig.timeout=o.httpConfig.timeout||15000,o.depth=o.depth||0,o.allowCrossDomainTravel=o.allowCrossDomainTravel||!1,t=t&&i;let l=[];try{let s=new URL(e),d=this.extractRootDomain(e);if(this.isLinktreeUrl(e))return await this.parseLinktreeProfile(e,o,t);if(this.isRestrictedDomain(e)){if(this.debug)console.warn(`⚠️ Skipped URL (restricted url): ${e}`);return l}let h=await this.executeHttp(e,n,a,o,t);if(!h){if(this.debug&&h===null)console.warn(`⚠️ Failed to fetch page content: ${e}`);if(this.debug&&h===!1)console.warn(`⚠️ Skipped URL (already visited): ${e}`);return l}let m=this.extractEmailsFromContent(h.data?.replaceAll("mailto:"," "));for(let g of m)if(this.handleEmailDiscovery(g,e,o.onEmailDiscovered,o.queue))l.push(g);if(o.depth>0||!r){let g=x(h.data).document,p=this.extractRelevantLinks(g,s,d,o.depth,o.allowCrossDomainTravel);o.depth--;let b=await Promise.allSettled(p.map((f)=>this.parseExternalWebsite(f,"GET",null,{...o,depth:o.depth},t,!1,!0)));for(let f of b)if(f.status==="fulfilled")l.push(...f.value);else if(this.debug)console.warn("⚠️ Failed to parse child URL:",f.reason?.message)}}catch(s){if(this.debug)console.error(`❌ Error parsing external website: ${e}`,s?.message)}if(i){if(o.onEmails&&o.onEmails.length>0)o.queue.add(async()=>Promise.all(o.onEmails.map((s)=>s(l))))}return l}async parseLinktreeProfile(e,n,a){let o=[];try{let t=await this.executeHttp(e,"GET",null,n,a);if(!t){if(this.debug)console.warn(`⚠️ Failed to fetch Linktree profile: ${e}`);return o}let r=x(t).document.getElementById("links-container");if(!r){if(this.debug)console.warn(`\uD83D\uDD0D No links container found in Linktree profile: ${e}`);return o}let c=this.extractLinktreeExternalUrls(r,e);if(c.length===0){if(this.debug)console.info("\uD83D\uDCED No valid external links found in Linktree profile");return o}if(this.debug)console.info(`\uD83C\uDFAF Found ${c.length} external links in Linktree profile`);let u=await Promise.allSettled(c.map((l)=>this.parseExternalWebsite(l,"GET",null,n,a,!1)));for(let l of u)if(l.status==="fulfilled")o.push(...l.value);else if(this.debug)console.warn("⚠️ Failed to parse Linktree external URL:",l.reason?.message)}catch(t){if(this.debug)console.error(`❌ Error parsing Linktree profile: ${e}`,t?.message)}return o}extractLinktreeExternalUrls(e,n){let a=new Set,o=e.querySelectorAll("a[href][target='_blank']");for(let t of o){let i=t.getAttribute("href");if(!i||i.length<3||this.forbiddenProtocols.some((r)=>i.startsWith(r)))continue;try{let r=new URL(i,n).href,c=this.extractRootDomain(r);if(c!=="linktr.ee"&&!this.isRestrictedDomain(r)&&c.length>3)a.add(r)}catch(r){if(this.debug)console.warn(`\uD83D\uDD17 Invalid URL in Linktree: ${i}`)}}return Array.from(a)}handleEmailDiscovery(e,n,a,o){if(!this.discoveredEmails.has(e)){this.discoveredEmails.add(e);let t={email:e,discoveredAt:n,timestamp:new Date};if(a&&a.length>0)o.add(async()=>Promise.all(a.map((i)=>i(t))));if(this.debug)console.info(`\uD83D\uDCE7 New email discovered: ${e} at ${n}`);return!0}return!1}isDomainAccessAllowed(e,n,a,o){if(o)return!0;if(a===0)return e===n;return e===n||e.endsWith(`.${n}`)||n.endsWith(`.${e}`)}extractRelevantLinks(e,n,a,o,t){let i=[],r=["about","contact","help","support","reach","email","mail","message","company","team","staff","info","inquiry","feedback","service","assistance","connect","touch"],c=e.querySelectorAll("a[href]");for(let u of c){let l=u.getAttribute("href");if(!l||l.length<2)continue;try{let s=this.normalizeUrl(l,n),d=this.extractRootDomain(s);if(!this.isDomainAccessAllowed(d,a,o,t))continue;if(r.some((m)=>s.toLowerCase().includes(m))||this.isLinktreeUrl(s))i.push(s)}catch(s){if(this.debug)console.warn(`\uD83D\uDD17 Invalid link found: ${l}`,s?.message)}}return i}extractEmailsFromContent(e){let n=e.replace(/[^\w@.-\s]/g," "),a=/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g,o=(c)=>{let u=/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/,l=c.split("@")[1]?.toLowerCase(),s=c.split(".").pop()?.toLowerCase();return u.test(c)&&l!==void 0&&s!==void 0&&!this.fileExtensions.includes(`.${s}`)&&!this.isRestrictedDomain(`https://${l}`)},t=(c)=>{return(c.match(a)||[]).filter(o)},i=e.replace(/<[^>]*>/g," "),r=[...t(i),...t(n)];return[...new Set(r)]}isRestrictedDomain(e){try{let n=new URL(e).host.toLowerCase();return this.restrictedDomains.some((a)=>n===a.toLowerCase()||n.endsWith(`.${a.toLowerCase()}`))}catch{return!0}}isLinktreeUrl(e){try{return this.extractRootDomain(e)==="linktr.ee"}catch{return!1}}extractRootDomain(e){try{let a=new URL(e).hostname.toLowerCase();return a.startsWith("www.")?a.slice(4):a}catch{return""}}normalizeUrl(e,n){if(e.startsWith("http://")||e.startsWith("https://"))return e;if(e.startsWith("//"))return`${n.protocol}${e}`;return new URL(e,n.href).href}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function y(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],n=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],a=[];for(let o=0;o<200;o++){let t=e[Math.floor(Math.random()*e.length)],i=n[Math.floor(Math.random()*n.length)],r="";switch(t.name){case"Chrome":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":r=`Mozilla/5.0 (${i}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}a.push(r)}return a}function $(){return["facebook.com","fb.com","messenger.com","instagram.com","threads.net","twitter.com","x.com","linkedin.com","pinterest.com","pin.it","reddit.com","tumblr.com","snapchat.com","tiktok.com","douyin.com","youtube.com","youtu.be","whatsapp.com","telegram.org","t.me","medium.com","quora.com","flickr.com","vimeo.com","vk.com","weibo.com","sina.com.cn","line.me","discord.com","discordapp.com","twitch.tv","meetup.com","nextdoor.com","xing.com","yelp.com","zalo.me","mastodon.social","clubhouse.com","patreon.com","onlyfans.com","douban.com","goodreads.com","soundcloud.com","spotify.com","last.fm","behance.net","dribbble.com","deviantart.com","pixiv.net","slideshare.net","tinder.com","bumble.com","etsy.com","indeed.com","glassdoor.com","monster.com","careerbuilder.com","dice.com","ziprecruiter.com","simplyhired.com","upwork.com","freelancer.com","fiverr.com","stackoverflow.com","stackoverflow.co","angel.co","wellfound.com","quora.com","stackexchange.com","yahoo.com","answers.microsoft.com","askubuntu.com","superuser.com","serverfault.com","mathoverflow.net","xda-developers.com","gamespot.com","ign.com","4chan.org","9gag.com","gizmodo.com","slashdot.org","hacker-news.news","ycombinator.com","producthunt.com","discourse.org","google.com","google.co.uk","google.de","google.fr","google.co.jp","bing.com","yahoo.com","search.yahoo.com","duckduckgo.com","baidu.com","yandex.com","yandex.ru","ask.com","wolframalpha.com","ecosia.org","startpage.com","qwant.com","searx.me","gibiru.com","swisscows.com","gmail.com","googlemail.com","outlook.com","hotmail.com","live.com","msn.com","yahoo.com","ymail.com","aol.com","icloud.com","me.com","mac.com","protonmail.com","pm.me","zoho.com","mail.com","gmx.com","gmx.net","yandex.com","yandex.ru","tutanota.com","tutanota.de","fastmail.com","hushmail.com","mailbox.org","posteo.de","runbox.com","disroot.org","163.com","qq.com","rambler.ru","mail.ru","yelp.com","yelp.ca","yelp.co.uk","yelp.com.au","yellowpages.com","yellowpages.ca","yell.com","tripadvisor.com","tripadvisor.co.uk","tripadvisor.ca","foursquare.com","angieslist.com","bbb.org","manta.com","thumbtack.com","homeadvisor.com","superpages.com","whitepages.com","local.com","citysearch.com","merchantcircle.com","insiderpages.com","kudzu.com","hotfrog.com","buildzoom.com","houzz.com","porch.com","mapquest.com","zagat.com","zomato.com","opentable.com","viator.com","expedia.com","booking.com","airbnb.com","vrbo.com","homeaway.com","craigslist.org","nextdoor.com","patch.com","meetup.com","eventbrite.com","groupon.com","livingsocial.com","gumtree.com","gumtree.com.au","kijiji.ca","leboncoin.fr","finn.no","blocket.se","58.com","dianping.com","tabelog.com","ypcdn.com"]}export{k as Scraper,w as CappedSet};
|
package/dist/utils/index.cjs
CHANGED
package/dist/utils/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rezo",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
|
|
5
5
|
"main": "dist/index.cjs",
|
|
6
6
|
"module": "dist/index.js",
|