rezo 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -25
- package/dist/adapters/entries/curl.cjs +4 -4
- package/dist/adapters/entries/fetch.cjs +2 -2
- package/dist/adapters/entries/http.cjs +2 -2
- package/dist/adapters/entries/http2.cjs +4 -4
- package/dist/adapters/entries/react-native.cjs +2 -2
- package/dist/adapters/entries/xhr.cjs +2 -2
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +13 -13
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +16 -16
- package/dist/plugin/addon/decodo/types.cjs +1 -0
- package/dist/plugin/addon/decodo/types.js +1 -0
- package/dist/plugin/addon/oxylabs/types.cjs +1 -0
- package/dist/plugin/addon/oxylabs/types.js +1 -0
- package/dist/plugin/index.cjs +36 -36
- package/dist/plugin/scraper.cjs +1 -0
- package/dist/plugin/scraper.js +1 -0
- package/dist/utils/index.cjs +0 -1
- package/dist/utils/index.js +0 -1
- package/package.json +62 -17
package/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<img src="assets/logo.svg" alt="Rezo HTTP Client" width="400">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/yuniqsolutions/rezo/main/assets/logo.svg" alt="Rezo HTTP Client" width="400">
|
|
3
3
|
</p>
|
|
4
4
|
|
|
5
5
|
<h1 align="center">Rezo HTTP Client</h1>
|
|
@@ -122,14 +122,17 @@ bun add rezo
|
|
|
122
122
|
```typescript
|
|
123
123
|
import rezo from 'rezo';
|
|
124
124
|
|
|
125
|
-
// Simple GET request
|
|
126
|
-
const
|
|
127
|
-
console.log(
|
|
125
|
+
// Simple GET request with destructured response
|
|
126
|
+
const { data, status, headers } = await rezo.get('https://api.example.com/users');
|
|
127
|
+
console.log(data); // Response body (auto-parsed JSON)
|
|
128
|
+
console.log(status); // 200
|
|
129
|
+
console.log(headers); // Response headers
|
|
128
130
|
|
|
129
131
|
// With query parameters
|
|
130
|
-
const users = await rezo.get('https://api.example.com/users', {
|
|
132
|
+
const { data: users } = await rezo.get('https://api.example.com/users', {
|
|
131
133
|
params: { page: 1, limit: 10 }
|
|
132
134
|
});
|
|
135
|
+
console.log(users);
|
|
133
136
|
```
|
|
134
137
|
|
|
135
138
|
### POST Request with JSON
|
|
@@ -137,13 +140,14 @@ const users = await rezo.get('https://api.example.com/users', {
|
|
|
137
140
|
```typescript
|
|
138
141
|
import rezo from 'rezo';
|
|
139
142
|
|
|
140
|
-
|
|
143
|
+
// Destructure the response for cleaner code
|
|
144
|
+
const { data, status, headers, config } = await rezo.post('https://api.example.com/users', {
|
|
141
145
|
name: 'John Doe',
|
|
142
146
|
email: 'john@example.com'
|
|
143
147
|
});
|
|
144
148
|
|
|
145
|
-
console.log(
|
|
146
|
-
console.log(
|
|
149
|
+
console.log(data); // Created user object
|
|
150
|
+
console.log(status); // 201
|
|
147
151
|
```
|
|
148
152
|
|
|
149
153
|
### Creating an Instance
|
|
@@ -1335,32 +1339,66 @@ await crawler.crawl('https://example.com');
|
|
|
1335
1339
|
|
|
1336
1340
|
## DOM Module
|
|
1337
1341
|
|
|
1338
|
-
Rezo
|
|
1342
|
+
Rezo provides a lightweight DOM parsing module for server-side HTML manipulation. Built on [linkedom](https://github.com/WebReflection/linkedom), it enables fast, standards-compliant DOM operations in Node, Bun, Deno or Edge environments - perfect for web scraping, HTML transformation, and testing.
|
|
1339
1343
|
|
|
1340
|
-
|
|
1341
|
-
import { parseHTML, DOMParser } from 'rezo/dom';
|
|
1344
|
+
### Parsing HTML
|
|
1342
1345
|
|
|
1343
|
-
|
|
1344
|
-
|
|
1346
|
+
```typescript
|
|
1347
|
+
import { parseHTML } from 'rezo/dom';
|
|
1348
|
+
|
|
1349
|
+
// Parse HTML and work with the document
|
|
1350
|
+
const { document, window } = parseHTML(`
|
|
1351
|
+
<html>
|
|
1352
|
+
<body>
|
|
1353
|
+
<h1>Hello World</h1>
|
|
1354
|
+
<ul id="items">
|
|
1355
|
+
<li>Item 1</li>
|
|
1356
|
+
<li>Item 2</li>
|
|
1357
|
+
</ul>
|
|
1358
|
+
</body>
|
|
1359
|
+
</html>
|
|
1360
|
+
`);
|
|
1361
|
+
|
|
1362
|
+
// Use familiar DOM APIs
|
|
1345
1363
|
console.log(document.querySelector('h1')?.textContent); // 'Hello World'
|
|
1364
|
+
console.log(document.querySelectorAll('li').length); // 2
|
|
1365
|
+
```
|
|
1366
|
+
|
|
1367
|
+
### Using with Rezo HTTP Client
|
|
1368
|
+
|
|
1369
|
+
```typescript
|
|
1370
|
+
import rezo from 'rezo';
|
|
1371
|
+
import { parseHTML } from 'rezo/dom';
|
|
1372
|
+
|
|
1373
|
+
// Fetch and parse a webpage
|
|
1374
|
+
const { data: html } = await rezo.get('https://example.com');
|
|
1375
|
+
const { document } = parseHTML(html);
|
|
1376
|
+
|
|
1377
|
+
// Extract data from the page
|
|
1378
|
+
const title = document.querySelector('title')?.textContent;
|
|
1379
|
+
const links = [...document.querySelectorAll('a')].map(a => a.getAttribute('href'));
|
|
1380
|
+
```
|
|
1381
|
+
|
|
1382
|
+
### DOMParser Interface
|
|
1383
|
+
|
|
1384
|
+
```typescript
|
|
1385
|
+
import { DOMParser } from 'rezo/dom';
|
|
1346
1386
|
|
|
1347
|
-
// Using DOMParser
|
|
1348
1387
|
const parser = new DOMParser();
|
|
1349
1388
|
const doc = parser.parseFromString('<div class="content">Text</div>', 'text/html');
|
|
1350
1389
|
console.log(doc.querySelector('.content')?.textContent); // 'Text'
|
|
1351
1390
|
```
|
|
1352
1391
|
|
|
1353
|
-
### Available
|
|
1354
|
-
|
|
1355
|
-
All linkedom exports are available from `rezo/dom`:
|
|
1392
|
+
### Available APIs
|
|
1356
1393
|
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1394
|
+
| Export | Description |
|
|
1395
|
+
|--------|-------------|
|
|
1396
|
+
| `parseHTML` | Parse HTML string into window/document objects |
|
|
1397
|
+
| `DOMParser` | Standard W3C DOMParser interface |
|
|
1398
|
+
| `Document` | Document constructor for creating new documents |
|
|
1399
|
+
| `Event` / `CustomEvent` | DOM Event interfaces |
|
|
1400
|
+
| `EventTarget` | Event handling interface |
|
|
1401
|
+
| `NodeList` | Standard NodeList interface |
|
|
1364
1402
|
|
|
1365
1403
|
---
|
|
1366
1404
|
|
|
@@ -1498,7 +1536,7 @@ See the [LICENSE](LICENSE) file for the complete license text.
|
|
|
1498
1536
|
---
|
|
1499
1537
|
|
|
1500
1538
|
<p align="center">
|
|
1501
|
-
<img src="assets/icon.svg" alt="Rezo Icon" width="60">
|
|
1539
|
+
<img src="https://raw.githubusercontent.com/yuniqsolutions/rezo/main/assets/icon.svg" alt="Rezo Icon" width="60">
|
|
1502
1540
|
</p>
|
|
1503
1541
|
|
|
1504
1542
|
<p align="center">
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeCurlRequest =
|
|
3
|
-
const
|
|
4
|
-
exports.executeHttpRequest =
|
|
1
|
+
const _mod_cuox1p = require('../curl.cjs');
|
|
2
|
+
exports.executeCurlRequest = _mod_cuox1p.executeRequest;;
|
|
3
|
+
const _mod_h6eiws = require('../http.cjs');
|
|
4
|
+
exports.executeHttpRequest = _mod_h6eiws.executeRequest;;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeRequest =
|
|
1
|
+
const _mod_g7jn2g = require('../fetch.cjs');
|
|
2
|
+
exports.executeRequest = _mod_g7jn2g.executeRequest;;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeRequest =
|
|
1
|
+
const _mod_3hcg7b = require('../http.cjs');
|
|
2
|
+
exports.executeRequest = _mod_3hcg7b.executeRequest;;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeHttp2Request =
|
|
3
|
-
const
|
|
4
|
-
exports.executeHttpRequest =
|
|
1
|
+
const _mod_0wgnjy = require('../http2.cjs');
|
|
2
|
+
exports.executeHttp2Request = _mod_0wgnjy.executeRequest;;
|
|
3
|
+
const _mod_ugdz7r = require('../http.cjs');
|
|
4
|
+
exports.executeHttpRequest = _mod_ugdz7r.executeRequest;;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeRequest =
|
|
1
|
+
const _mod_mo4x9u = require('../react-native.cjs');
|
|
2
|
+
exports.executeRequest = _mod_mo4x9u.executeRequest;;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.executeRequest =
|
|
1
|
+
const _mod_1604ms = require('../xhr.cjs');
|
|
2
|
+
exports.executeRequest = _mod_1604ms.executeRequest;;
|
package/dist/adapters/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.detectRuntime =
|
|
3
|
-
exports.getAdapterCapabilities =
|
|
4
|
-
exports.buildAdapterContext =
|
|
5
|
-
exports.getAvailableAdapters =
|
|
6
|
-
exports.selectAdapter =
|
|
1
|
+
const _mod_3xqpwj = require('./picker.cjs');
|
|
2
|
+
exports.detectRuntime = _mod_3xqpwj.detectRuntime;
|
|
3
|
+
exports.getAdapterCapabilities = _mod_3xqpwj.getAdapterCapabilities;
|
|
4
|
+
exports.buildAdapterContext = _mod_3xqpwj.buildAdapterContext;
|
|
5
|
+
exports.getAvailableAdapters = _mod_3xqpwj.getAvailableAdapters;
|
|
6
|
+
exports.selectAdapter = _mod_3xqpwj.selectAdapter;;
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
10
|
-
const
|
|
11
|
-
exports.FileCacher =
|
|
12
|
-
const
|
|
13
|
-
exports.UrlStore =
|
|
1
|
+
const _mod_dxws8j = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_dxws8j.LRUCache;;
|
|
3
|
+
const _mod_hons1h = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_hons1h.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_hons1h.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_hons1h.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_99mq9u = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_99mq9u.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_99mq9u.normalizeResponseCacheConfig;;
|
|
10
|
+
const _mod_pz0bcx = require('./file-cacher.cjs');
|
|
11
|
+
exports.FileCacher = _mod_pz0bcx.FileCacher;;
|
|
12
|
+
const _mod_kc26ws = require('./url-store.cjs');
|
|
13
|
+
exports.UrlStore = _mod_kc26ws.UrlStore;;
|
package/dist/entries/crawler.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
exports.Domain =
|
|
1
|
+
const _mod_0hj10x = require('../plugin/crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_0hj10x.Crawler;;
|
|
3
|
+
const _mod_mxmkoz = require('../plugin/crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_mxmkoz.CrawlerOptions;
|
|
5
|
+
exports.Domain = _mod_mxmkoz.Domain;;
|
package/dist/index.cjs
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Rezo =
|
|
3
|
-
exports.createRezoInstance =
|
|
4
|
-
exports.createDefaultInstance =
|
|
5
|
-
const
|
|
6
|
-
exports.RezoError =
|
|
7
|
-
exports.RezoErrorCode =
|
|
8
|
-
const
|
|
9
|
-
exports.RezoHeaders =
|
|
10
|
-
const
|
|
11
|
-
exports.RezoFormData =
|
|
12
|
-
const
|
|
13
|
-
exports.RezoCookieJar =
|
|
14
|
-
const
|
|
15
|
-
exports.createDefaultHooks =
|
|
16
|
-
exports.mergeHooks =
|
|
1
|
+
const _mod_7sze83 = require('./core/rezo.cjs');
|
|
2
|
+
exports.Rezo = _mod_7sze83.Rezo;
|
|
3
|
+
exports.createRezoInstance = _mod_7sze83.createRezoInstance;
|
|
4
|
+
exports.createDefaultInstance = _mod_7sze83.createDefaultInstance;;
|
|
5
|
+
const _mod_j9i84j = require('./errors/rezo-error.cjs');
|
|
6
|
+
exports.RezoError = _mod_j9i84j.RezoError;
|
|
7
|
+
exports.RezoErrorCode = _mod_j9i84j.RezoErrorCode;;
|
|
8
|
+
const _mod_y5piv0 = require('./utils/headers.cjs');
|
|
9
|
+
exports.RezoHeaders = _mod_y5piv0.RezoHeaders;;
|
|
10
|
+
const _mod_l0prwp = require('./utils/form-data.cjs');
|
|
11
|
+
exports.RezoFormData = _mod_l0prwp.RezoFormData;;
|
|
12
|
+
const _mod_t9pqxu = require('./utils/cookies.cjs');
|
|
13
|
+
exports.RezoCookieJar = _mod_t9pqxu.RezoCookieJar;;
|
|
14
|
+
const _mod_7d3hfy = require('./core/hooks.cjs');
|
|
15
|
+
exports.createDefaultHooks = _mod_7d3hfy.createDefaultHooks;
|
|
16
|
+
exports.mergeHooks = _mod_7d3hfy.mergeHooks;;
|
|
17
17
|
const { RezoError } = require('./errors/rezo-error.cjs');
|
|
18
18
|
const isRezoError = exports.isRezoError = RezoError.isRezoError;
|
|
19
19
|
const Cancel = exports.Cancel = RezoError;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports={};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
var c={};export{c as default};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports={};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
var c={};export{c as default};
|
package/dist/plugin/index.cjs
CHANGED
|
@@ -1,36 +1,36 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
const
|
|
6
|
-
exports.FileCacher =
|
|
7
|
-
const
|
|
8
|
-
exports.UrlStore =
|
|
9
|
-
const
|
|
10
|
-
exports.Oxylabs =
|
|
11
|
-
const
|
|
12
|
-
exports.OXYLABS_BROWSER_TYPES =
|
|
13
|
-
exports.OXYLABS_COMMON_LOCALES =
|
|
14
|
-
exports.OXYLABS_COMMON_GEO_LOCATIONS =
|
|
15
|
-
exports.OXYLABS_US_STATES =
|
|
16
|
-
exports.OXYLABS_EUROPEAN_COUNTRIES =
|
|
17
|
-
exports.OXYLABS_ASIAN_COUNTRIES =
|
|
18
|
-
exports.getRandomOxylabsBrowserType =
|
|
19
|
-
exports.getRandomOxylabsLocale =
|
|
20
|
-
exports.getRandomOxylabsGeoLocation =
|
|
21
|
-
const
|
|
22
|
-
exports.Decodo =
|
|
23
|
-
const
|
|
24
|
-
exports.DECODO_DEVICE_TYPES =
|
|
25
|
-
exports.DECODO_HEADLESS_MODES =
|
|
26
|
-
exports.DECODO_COMMON_LOCALES =
|
|
27
|
-
exports.DECODO_COMMON_COUNTRIES =
|
|
28
|
-
exports.DECODO_EUROPEAN_COUNTRIES =
|
|
29
|
-
exports.DECODO_ASIAN_COUNTRIES =
|
|
30
|
-
exports.DECODO_US_STATES =
|
|
31
|
-
exports.DECODO_COMMON_CITIES =
|
|
32
|
-
exports.getRandomDecodoDeviceType =
|
|
33
|
-
exports.getRandomDecodoLocale =
|
|
34
|
-
exports.getRandomDecodoCountry =
|
|
35
|
-
exports.getRandomDecodoCity =
|
|
36
|
-
exports.generateDecodoSessionId =
|
|
1
|
+
const _mod_khxd8a = require('./crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_khxd8a.Crawler;;
|
|
3
|
+
const _mod_vm96it = require('./crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_vm96it.CrawlerOptions;;
|
|
5
|
+
const _mod_enfdda = require('../cache/file-cacher.cjs');
|
|
6
|
+
exports.FileCacher = _mod_enfdda.FileCacher;;
|
|
7
|
+
const _mod_p51bkd = require('../cache/url-store.cjs');
|
|
8
|
+
exports.UrlStore = _mod_p51bkd.UrlStore;;
|
|
9
|
+
const _mod_q47c3j = require('./addon/oxylabs/index.cjs');
|
|
10
|
+
exports.Oxylabs = _mod_q47c3j.Oxylabs;;
|
|
11
|
+
const _mod_rfrfyo = require('./addon/oxylabs/options.cjs');
|
|
12
|
+
exports.OXYLABS_BROWSER_TYPES = _mod_rfrfyo.OXYLABS_BROWSER_TYPES;
|
|
13
|
+
exports.OXYLABS_COMMON_LOCALES = _mod_rfrfyo.OXYLABS_COMMON_LOCALES;
|
|
14
|
+
exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_rfrfyo.OXYLABS_COMMON_GEO_LOCATIONS;
|
|
15
|
+
exports.OXYLABS_US_STATES = _mod_rfrfyo.OXYLABS_US_STATES;
|
|
16
|
+
exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_rfrfyo.OXYLABS_EUROPEAN_COUNTRIES;
|
|
17
|
+
exports.OXYLABS_ASIAN_COUNTRIES = _mod_rfrfyo.OXYLABS_ASIAN_COUNTRIES;
|
|
18
|
+
exports.getRandomOxylabsBrowserType = _mod_rfrfyo.getRandomBrowserType;
|
|
19
|
+
exports.getRandomOxylabsLocale = _mod_rfrfyo.getRandomLocale;
|
|
20
|
+
exports.getRandomOxylabsGeoLocation = _mod_rfrfyo.getRandomGeoLocation;;
|
|
21
|
+
const _mod_5qyxa5 = require('./addon/decodo/index.cjs');
|
|
22
|
+
exports.Decodo = _mod_5qyxa5.Decodo;;
|
|
23
|
+
const _mod_38klqg = require('./addon/decodo/options.cjs');
|
|
24
|
+
exports.DECODO_DEVICE_TYPES = _mod_38klqg.DECODO_DEVICE_TYPES;
|
|
25
|
+
exports.DECODO_HEADLESS_MODES = _mod_38klqg.DECODO_HEADLESS_MODES;
|
|
26
|
+
exports.DECODO_COMMON_LOCALES = _mod_38klqg.DECODO_COMMON_LOCALES;
|
|
27
|
+
exports.DECODO_COMMON_COUNTRIES = _mod_38klqg.DECODO_COMMON_COUNTRIES;
|
|
28
|
+
exports.DECODO_EUROPEAN_COUNTRIES = _mod_38klqg.DECODO_EUROPEAN_COUNTRIES;
|
|
29
|
+
exports.DECODO_ASIAN_COUNTRIES = _mod_38klqg.DECODO_ASIAN_COUNTRIES;
|
|
30
|
+
exports.DECODO_US_STATES = _mod_38klqg.DECODO_US_STATES;
|
|
31
|
+
exports.DECODO_COMMON_CITIES = _mod_38klqg.DECODO_COMMON_CITIES;
|
|
32
|
+
exports.getRandomDecodoDeviceType = _mod_38klqg.getRandomDeviceType;
|
|
33
|
+
exports.getRandomDecodoLocale = _mod_38klqg.getRandomLocale;
|
|
34
|
+
exports.getRandomDecodoCountry = _mod_38klqg.getRandomCountry;
|
|
35
|
+
exports.getRandomDecodoCity = _mod_38klqg.getRandomCity;
|
|
36
|
+
exports.generateDecodoSessionId = _mod_38klqg.generateSessionId;;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
var{parseHTML:w}=require("linkedom");class x extends Set{maxSize;constructor(e){super();this.maxSize=e}add(e){if(this.has(e))return this;if(this.size>=this.maxSize){let n=this.values().next().value;if(n)this.delete(n)}return super.add(e)}}class v{http;httpOptions;onEmailLeads;onEmailDiscovered;debug;discoveredEmails=new x(1e4);userAgents=[];fileExtensions=[];restrictedDomains=$();forbiddenProtocols=["mailto:","tel:","javascript:","data:","sms:","ftp:","file:","irc:","blob:","chrome:","about:","intent:"];constructor(e,n,a,o,t=!1){this.http=e,this.httpOptions=n,this.onEmailLeads=a,this.onEmailDiscovered=o,this.debug=t,this.userAgents=y()}sleep(e){return new Promise((n)=>setTimeout(n,e))}async executeHttp(e,n,a,o,t,i=0){let{getCache:r,saveCache:c,hasUrlInCache:u,saveUrl:l,httpConfig:s={}}=o;if(!e||e.length<3||this.forbiddenProtocols.some((d)=>e.startsWith(d)))return;try{let d=t?!1:await u(e),h=await r(e);if(d&&!h)return!1;if(d&&n!=="GET")return!1;let m=h&&n==="GET"?h:await(n==="GET"?this.http.get(e,s):n==="PATCH"?this.http.patch(e,a,s):n==="POST"?this.http.post(e,a,s):this.http.put(e,a,s));if(!h)await c(e,{data:m.data,contentType:m.contentType,finalUrl:m.finalUrl});if(!d)await l(e);if(!m.contentType||!m.contentType.includes("/html")||!m.contentType.includes("text/")||typeof m.data!=="string")return null;return{data:m.data,contentType:m.contentType,finalUrl:m.finalUrl}}catch(d){let h=d,m=this.httpOptions;if(h&&h.response){let g=h.response.status,p=m.retryDelay||100,b=m.maxRetryAttempts||3,f=m.retryWithoutProxyOnStatusCode||void 0,k=m.maxRetryOnProxyError||3;if(f&&s.proxy&&f.includes(g)&&i<b)return await this.sleep(p),delete s.proxy,await this.executeHttp(e,n,a,o,t,i+1);else if(m.retryOnStatusCode&&s.proxy&&m.retryOnStatusCode.includes(g)&&i<b)return await this.sleep(p),await this.executeHttp(e,n,a,o,t,i+1);else if(m.retryOnProxyError&&s.proxy&&i<k)return await this.sleep(p),await this.executeHttp(e,n,a,o,t,i+1)}if(this.debug){if(this.debug)console.log(`Error: unable to ${n} ${e}: ${d.message}`)}return null}}extractEmails(e,n,a,o,t){let i=this.extractEmailsFromContent(e?.replaceAll("mailto:"," ")),r=[];for(let c of i)if(this.handleEmailDiscovery(c,n,a,t))r.push(c);if(o&&o.length>0&&r.length>0)t.add(async()=>Promise.all(o.map((c)=>c(r))));i.length=0,r.length=0}async parseExternalWebsite(e,n,a,o,t,i=!0,r,c){let u=o.httpConfig?.headers?o.httpConfig.headers instanceof Headers?Object.fromEntries(o.httpConfig.headers.entries()):o.httpConfig.headers:{};o.httpConfig=o.httpConfig||{},o.httpConfig.headers={"user-agent":this.getRandomUserAgent(),accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","cache-control":"no-cache",pragma:"no-cache",...u},o.httpConfig.timeout=o.httpConfig.timeout||15000,o.depth=o.depth||0,o.allowCrossDomainTravel=o.allowCrossDomainTravel||!1,t=t&&i;let l=[];try{let s=new URL(e),d=this.extractRootDomain(e);if(this.isLinktreeUrl(e))return await this.parseLinktreeProfile(e,o,t);if(this.isRestrictedDomain(e)){if(this.debug)console.warn(`⚠️ Skipped URL (restricted url): ${e}`);return l}let h=await this.executeHttp(e,n,a,o,t);if(!h){if(this.debug&&h===null)console.warn(`⚠️ Failed to fetch page content: ${e}`);if(this.debug&&h===!1)console.warn(`⚠️ Skipped URL (already visited): ${e}`);return l}let m=this.extractEmailsFromContent(h.data?.replaceAll("mailto:"," "));for(let g of m)if(this.handleEmailDiscovery(g,e,o.onEmailDiscovered,o.queue))l.push(g);if(o.depth>0||!r){let g=w(h.data).document,p=this.extractRelevantLinks(g,s,d,o.depth,o.allowCrossDomainTravel);o.depth--;let b=await Promise.allSettled(p.map((f)=>this.parseExternalWebsite(f,"GET",null,{...o,depth:o.depth},t,!1,!0)));for(let f of b)if(f.status==="fulfilled")l.push(...f.value);else if(this.debug)console.warn("⚠️ Failed to parse child URL:",f.reason?.message)}}catch(s){if(this.debug)console.error(`❌ Error parsing external website: ${e}`,s?.message)}if(i){if(o.onEmails&&o.onEmails.length>0)o.queue.add(async()=>Promise.all(o.onEmails.map((s)=>s(l))))}return l}async parseLinktreeProfile(e,n,a){let o=[];try{let t=await this.executeHttp(e,"GET",null,n,a);if(!t){if(this.debug)console.warn(`⚠️ Failed to fetch Linktree profile: ${e}`);return o}let r=w(t).document.getElementById("links-container");if(!r){if(this.debug)console.warn(`\uD83D\uDD0D No links container found in Linktree profile: ${e}`);return o}let c=this.extractLinktreeExternalUrls(r,e);if(c.length===0){if(this.debug)console.info("\uD83D\uDCED No valid external links found in Linktree profile");return o}if(this.debug)console.info(`\uD83C\uDFAF Found ${c.length} external links in Linktree profile`);let u=await Promise.allSettled(c.map((l)=>this.parseExternalWebsite(l,"GET",null,n,a,!1)));for(let l of u)if(l.status==="fulfilled")o.push(...l.value);else if(this.debug)console.warn("⚠️ Failed to parse Linktree external URL:",l.reason?.message)}catch(t){if(this.debug)console.error(`❌ Error parsing Linktree profile: ${e}`,t?.message)}return o}extractLinktreeExternalUrls(e,n){let a=new Set,o=e.querySelectorAll("a[href][target='_blank']");for(let t of o){let i=t.getAttribute("href");if(!i||i.length<3||this.forbiddenProtocols.some((r)=>i.startsWith(r)))continue;try{let r=new URL(i,n).href,c=this.extractRootDomain(r);if(c!=="linktr.ee"&&!this.isRestrictedDomain(r)&&c.length>3)a.add(r)}catch(r){if(this.debug)console.warn(`\uD83D\uDD17 Invalid URL in Linktree: ${i}`)}}return Array.from(a)}handleEmailDiscovery(e,n,a,o){if(!this.discoveredEmails.has(e)){this.discoveredEmails.add(e);let t={email:e,discoveredAt:n,timestamp:new Date};if(a&&a.length>0)o.add(async()=>Promise.all(a.map((i)=>i(t))));if(this.debug)console.info(`\uD83D\uDCE7 New email discovered: ${e} at ${n}`);return!0}return!1}isDomainAccessAllowed(e,n,a,o){if(o)return!0;if(a===0)return e===n;return e===n||e.endsWith(`.${n}`)||n.endsWith(`.${e}`)}extractRelevantLinks(e,n,a,o,t){let i=[],r=["about","contact","help","support","reach","email","mail","message","company","team","staff","info","inquiry","feedback","service","assistance","connect","touch"],c=e.querySelectorAll("a[href]");for(let u of c){let l=u.getAttribute("href");if(!l||l.length<2)continue;try{let s=this.normalizeUrl(l,n),d=this.extractRootDomain(s);if(!this.isDomainAccessAllowed(d,a,o,t))continue;if(r.some((m)=>s.toLowerCase().includes(m))||this.isLinktreeUrl(s))i.push(s)}catch(s){if(this.debug)console.warn(`\uD83D\uDD17 Invalid link found: ${l}`,s?.message)}}return i}extractEmailsFromContent(e){let n=e.replace(/[^\w@.-\s]/g," "),a=/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g,o=(c)=>{let u=/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/,l=c.split("@")[1]?.toLowerCase(),s=c.split(".").pop()?.toLowerCase();return u.test(c)&&l!==void 0&&s!==void 0&&!this.fileExtensions.includes(`.${s}`)&&!this.isRestrictedDomain(`https://${l}`)},t=(c)=>{return(c.match(a)||[]).filter(o)},i=e.replace(/<[^>]*>/g," "),r=[...t(i),...t(n)];return[...new Set(r)]}isRestrictedDomain(e){try{let n=new URL(e).host.toLowerCase();return this.restrictedDomains.some((a)=>n===a.toLowerCase()||n.endsWith(`.${a.toLowerCase()}`))}catch{return!0}}isLinktreeUrl(e){try{return this.extractRootDomain(e)==="linktr.ee"}catch{return!1}}extractRootDomain(e){try{let a=new URL(e).hostname.toLowerCase();return a.startsWith("www.")?a.slice(4):a}catch{return""}}normalizeUrl(e,n){if(e.startsWith("http://")||e.startsWith("https://"))return e;if(e.startsWith("//"))return`${n.protocol}${e}`;return new URL(e,n.href).href}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function y(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],n=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],a=[];for(let o=0;o<200;o++){let t=e[Math.floor(Math.random()*e.length)],i=n[Math.floor(Math.random()*n.length)],r="";switch(t.name){case"Chrome":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":r=`Mozilla/5.0 (${i}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}a.push(r)}return a}function $(){return["facebook.com","fb.com","messenger.com","instagram.com","threads.net","twitter.com","x.com","linkedin.com","pinterest.com","pin.it","reddit.com","tumblr.com","snapchat.com","tiktok.com","douyin.com","youtube.com","youtu.be","whatsapp.com","telegram.org","t.me","medium.com","quora.com","flickr.com","vimeo.com","vk.com","weibo.com","sina.com.cn","line.me","discord.com","discordapp.com","twitch.tv","meetup.com","nextdoor.com","xing.com","yelp.com","zalo.me","mastodon.social","clubhouse.com","patreon.com","onlyfans.com","douban.com","goodreads.com","soundcloud.com","spotify.com","last.fm","behance.net","dribbble.com","deviantart.com","pixiv.net","slideshare.net","tinder.com","bumble.com","etsy.com","indeed.com","glassdoor.com","monster.com","careerbuilder.com","dice.com","ziprecruiter.com","simplyhired.com","upwork.com","freelancer.com","fiverr.com","stackoverflow.com","stackoverflow.co","angel.co","wellfound.com","quora.com","stackexchange.com","yahoo.com","answers.microsoft.com","askubuntu.com","superuser.com","serverfault.com","mathoverflow.net","xda-developers.com","gamespot.com","ign.com","4chan.org","9gag.com","gizmodo.com","slashdot.org","hacker-news.news","ycombinator.com","producthunt.com","discourse.org","google.com","google.co.uk","google.de","google.fr","google.co.jp","bing.com","yahoo.com","search.yahoo.com","duckduckgo.com","baidu.com","yandex.com","yandex.ru","ask.com","wolframalpha.com","ecosia.org","startpage.com","qwant.com","searx.me","gibiru.com","swisscows.com","gmail.com","googlemail.com","outlook.com","hotmail.com","live.com","msn.com","yahoo.com","ymail.com","aol.com","icloud.com","me.com","mac.com","protonmail.com","pm.me","zoho.com","mail.com","gmx.com","gmx.net","yandex.com","yandex.ru","tutanota.com","tutanota.de","fastmail.com","hushmail.com","mailbox.org","posteo.de","runbox.com","disroot.org","163.com","qq.com","rambler.ru","mail.ru","yelp.com","yelp.ca","yelp.co.uk","yelp.com.au","yellowpages.com","yellowpages.ca","yell.com","tripadvisor.com","tripadvisor.co.uk","tripadvisor.ca","foursquare.com","angieslist.com","bbb.org","manta.com","thumbtack.com","homeadvisor.com","superpages.com","whitepages.com","local.com","citysearch.com","merchantcircle.com","insiderpages.com","kudzu.com","hotfrog.com","buildzoom.com","houzz.com","porch.com","mapquest.com","zagat.com","zomato.com","opentable.com","viator.com","expedia.com","booking.com","airbnb.com","vrbo.com","homeaway.com","craigslist.org","nextdoor.com","patch.com","meetup.com","eventbrite.com","groupon.com","livingsocial.com","gumtree.com","gumtree.com.au","kijiji.ca","leboncoin.fr","finn.no","blocket.se","58.com","dianping.com","tabelog.com","ypcdn.com"]}exports.Scraper=v;exports.CappedSet=x;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{parseHTML as x}from"linkedom";class w extends Set{maxSize;constructor(e){super();this.maxSize=e}add(e){if(this.has(e))return this;if(this.size>=this.maxSize){let n=this.values().next().value;if(n)this.delete(n)}return super.add(e)}}class k{http;httpOptions;onEmailLeads;onEmailDiscovered;debug;discoveredEmails=new w(1e4);userAgents=[];fileExtensions=[];restrictedDomains=$();forbiddenProtocols=["mailto:","tel:","javascript:","data:","sms:","ftp:","file:","irc:","blob:","chrome:","about:","intent:"];constructor(e,n,a,o,t=!1){this.http=e,this.httpOptions=n,this.onEmailLeads=a,this.onEmailDiscovered=o,this.debug=t,this.userAgents=y()}sleep(e){return new Promise((n)=>setTimeout(n,e))}async executeHttp(e,n,a,o,t,i=0){let{getCache:r,saveCache:c,hasUrlInCache:u,saveUrl:l,httpConfig:s={}}=o;if(!e||e.length<3||this.forbiddenProtocols.some((d)=>e.startsWith(d)))return;try{let d=t?!1:await u(e),h=await r(e);if(d&&!h)return!1;if(d&&n!=="GET")return!1;let m=h&&n==="GET"?h:await(n==="GET"?this.http.get(e,s):n==="PATCH"?this.http.patch(e,a,s):n==="POST"?this.http.post(e,a,s):this.http.put(e,a,s));if(!h)await c(e,{data:m.data,contentType:m.contentType,finalUrl:m.finalUrl});if(!d)await l(e);if(!m.contentType||!m.contentType.includes("/html")||!m.contentType.includes("text/")||typeof m.data!=="string")return null;return{data:m.data,contentType:m.contentType,finalUrl:m.finalUrl}}catch(d){let h=d,m=this.httpOptions;if(h&&h.response){let g=h.response.status,p=m.retryDelay||100,b=m.maxRetryAttempts||3,f=m.retryWithoutProxyOnStatusCode||void 0,v=m.maxRetryOnProxyError||3;if(f&&s.proxy&&f.includes(g)&&i<b)return await this.sleep(p),delete s.proxy,await this.executeHttp(e,n,a,o,t,i+1);else if(m.retryOnStatusCode&&s.proxy&&m.retryOnStatusCode.includes(g)&&i<b)return await this.sleep(p),await this.executeHttp(e,n,a,o,t,i+1);else if(m.retryOnProxyError&&s.proxy&&i<v)return await this.sleep(p),await this.executeHttp(e,n,a,o,t,i+1)}if(this.debug){if(this.debug)console.log(`Error: unable to ${n} ${e}: ${d.message}`)}return null}}extractEmails(e,n,a,o,t){let i=this.extractEmailsFromContent(e?.replaceAll("mailto:"," ")),r=[];for(let c of i)if(this.handleEmailDiscovery(c,n,a,t))r.push(c);if(o&&o.length>0&&r.length>0)t.add(async()=>Promise.all(o.map((c)=>c(r))));i.length=0,r.length=0}async parseExternalWebsite(e,n,a,o,t,i=!0,r,c){let u=o.httpConfig?.headers?o.httpConfig.headers instanceof Headers?Object.fromEntries(o.httpConfig.headers.entries()):o.httpConfig.headers:{};o.httpConfig=o.httpConfig||{},o.httpConfig.headers={"user-agent":this.getRandomUserAgent(),accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","accept-language":"en-US,en;q=0.9","cache-control":"no-cache",pragma:"no-cache",...u},o.httpConfig.timeout=o.httpConfig.timeout||15000,o.depth=o.depth||0,o.allowCrossDomainTravel=o.allowCrossDomainTravel||!1,t=t&&i;let l=[];try{let s=new URL(e),d=this.extractRootDomain(e);if(this.isLinktreeUrl(e))return await this.parseLinktreeProfile(e,o,t);if(this.isRestrictedDomain(e)){if(this.debug)console.warn(`⚠️ Skipped URL (restricted url): ${e}`);return l}let h=await this.executeHttp(e,n,a,o,t);if(!h){if(this.debug&&h===null)console.warn(`⚠️ Failed to fetch page content: ${e}`);if(this.debug&&h===!1)console.warn(`⚠️ Skipped URL (already visited): ${e}`);return l}let m=this.extractEmailsFromContent(h.data?.replaceAll("mailto:"," "));for(let g of m)if(this.handleEmailDiscovery(g,e,o.onEmailDiscovered,o.queue))l.push(g);if(o.depth>0||!r){let g=x(h.data).document,p=this.extractRelevantLinks(g,s,d,o.depth,o.allowCrossDomainTravel);o.depth--;let b=await Promise.allSettled(p.map((f)=>this.parseExternalWebsite(f,"GET",null,{...o,depth:o.depth},t,!1,!0)));for(let f of b)if(f.status==="fulfilled")l.push(...f.value);else if(this.debug)console.warn("⚠️ Failed to parse child URL:",f.reason?.message)}}catch(s){if(this.debug)console.error(`❌ Error parsing external website: ${e}`,s?.message)}if(i){if(o.onEmails&&o.onEmails.length>0)o.queue.add(async()=>Promise.all(o.onEmails.map((s)=>s(l))))}return l}async parseLinktreeProfile(e,n,a){let o=[];try{let t=await this.executeHttp(e,"GET",null,n,a);if(!t){if(this.debug)console.warn(`⚠️ Failed to fetch Linktree profile: ${e}`);return o}let r=x(t).document.getElementById("links-container");if(!r){if(this.debug)console.warn(`\uD83D\uDD0D No links container found in Linktree profile: ${e}`);return o}let c=this.extractLinktreeExternalUrls(r,e);if(c.length===0){if(this.debug)console.info("\uD83D\uDCED No valid external links found in Linktree profile");return o}if(this.debug)console.info(`\uD83C\uDFAF Found ${c.length} external links in Linktree profile`);let u=await Promise.allSettled(c.map((l)=>this.parseExternalWebsite(l,"GET",null,n,a,!1)));for(let l of u)if(l.status==="fulfilled")o.push(...l.value);else if(this.debug)console.warn("⚠️ Failed to parse Linktree external URL:",l.reason?.message)}catch(t){if(this.debug)console.error(`❌ Error parsing Linktree profile: ${e}`,t?.message)}return o}extractLinktreeExternalUrls(e,n){let a=new Set,o=e.querySelectorAll("a[href][target='_blank']");for(let t of o){let i=t.getAttribute("href");if(!i||i.length<3||this.forbiddenProtocols.some((r)=>i.startsWith(r)))continue;try{let r=new URL(i,n).href,c=this.extractRootDomain(r);if(c!=="linktr.ee"&&!this.isRestrictedDomain(r)&&c.length>3)a.add(r)}catch(r){if(this.debug)console.warn(`\uD83D\uDD17 Invalid URL in Linktree: ${i}`)}}return Array.from(a)}handleEmailDiscovery(e,n,a,o){if(!this.discoveredEmails.has(e)){this.discoveredEmails.add(e);let t={email:e,discoveredAt:n,timestamp:new Date};if(a&&a.length>0)o.add(async()=>Promise.all(a.map((i)=>i(t))));if(this.debug)console.info(`\uD83D\uDCE7 New email discovered: ${e} at ${n}`);return!0}return!1}isDomainAccessAllowed(e,n,a,o){if(o)return!0;if(a===0)return e===n;return e===n||e.endsWith(`.${n}`)||n.endsWith(`.${e}`)}extractRelevantLinks(e,n,a,o,t){let i=[],r=["about","contact","help","support","reach","email","mail","message","company","team","staff","info","inquiry","feedback","service","assistance","connect","touch"],c=e.querySelectorAll("a[href]");for(let u of c){let l=u.getAttribute("href");if(!l||l.length<2)continue;try{let s=this.normalizeUrl(l,n),d=this.extractRootDomain(s);if(!this.isDomainAccessAllowed(d,a,o,t))continue;if(r.some((m)=>s.toLowerCase().includes(m))||this.isLinktreeUrl(s))i.push(s)}catch(s){if(this.debug)console.warn(`\uD83D\uDD17 Invalid link found: ${l}`,s?.message)}}return i}extractEmailsFromContent(e){let n=e.replace(/[^\w@.-\s]/g," "),a=/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g,o=(c)=>{let u=/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/,l=c.split("@")[1]?.toLowerCase(),s=c.split(".").pop()?.toLowerCase();return u.test(c)&&l!==void 0&&s!==void 0&&!this.fileExtensions.includes(`.${s}`)&&!this.isRestrictedDomain(`https://${l}`)},t=(c)=>{return(c.match(a)||[]).filter(o)},i=e.replace(/<[^>]*>/g," "),r=[...t(i),...t(n)];return[...new Set(r)]}isRestrictedDomain(e){try{let n=new URL(e).host.toLowerCase();return this.restrictedDomains.some((a)=>n===a.toLowerCase()||n.endsWith(`.${a.toLowerCase()}`))}catch{return!0}}isLinktreeUrl(e){try{return this.extractRootDomain(e)==="linktr.ee"}catch{return!1}}extractRootDomain(e){try{let a=new URL(e).hostname.toLowerCase();return a.startsWith("www.")?a.slice(4):a}catch{return""}}normalizeUrl(e,n){if(e.startsWith("http://")||e.startsWith("https://"))return e;if(e.startsWith("//"))return`${n.protocol}${e}`;return new URL(e,n.href).href}getRandomUserAgent(){return this.userAgents[Math.floor(Math.random()*this.userAgents.length)]}}function y(){let e=[{name:"Chrome",version:"91.0.4472.124",engine:"AppleWebKit/537.36"},{name:"Firefox",version:"89.0",engine:"Gecko/20100101"},{name:"Safari",version:"14.1.1",engine:"AppleWebKit/605.1.15"},{name:"Edge",version:"91.0.864.59",engine:"AppleWebKit/537.36"},{name:"Opera",version:"77.0.4054.277",engine:"AppleWebKit/537.36"},{name:"Vivaldi",version:"3.8.2259.42",engine:"AppleWebKit/537.36"},{name:"Brave",version:"1.26.74",engine:"AppleWebKit/537.36"},{name:"Chromium",version:"91.0.4472.101",engine:"AppleWebKit/537.36"},{name:"Yandex",version:"21.5.3.742",engine:"AppleWebKit/537.36"},{name:"Maxthon",version:"5.3.8.2000",engine:"AppleWebKit/537.36"}],n=["Windows NT 10.0","Windows NT 6.1","Macintosh; Intel Mac OS X 10_15_7","Macintosh; Intel Mac OS X 11_4_0","X11; Linux x86_64","X11; Ubuntu; Linux x86_64"],a=[];for(let o=0;o<200;o++){let t=e[Math.floor(Math.random()*e.length)],i=n[Math.floor(Math.random()*n.length)],r="";switch(t.name){case"Chrome":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36`;break;case"Firefox":r=`Mozilla/5.0 (${i}; rv:${t.version}) ${t.engine} Firefox/${t.version}`;break;case"Safari":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Version/${t.version} Safari/605.1.15`;break;case"Edge":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Edg/${t.version}`;break;case"Opera":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 OPR/${t.version}`;break;case"Vivaldi":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Vivaldi/${t.version}`;break;case"Brave":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Brave/${t.version}`;break;case"Chromium":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chromium/${t.version} Chrome/${t.version} Safari/537.36`;break;case"Yandex":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} YaBrowser/${t.version} Safari/537.36`;break;case"Maxthon":r=`Mozilla/5.0 (${i}) ${t.engine} (KHTML, like Gecko) Chrome/${t.version} Safari/537.36 Maxthon/${t.version}`;break}a.push(r)}return a}function $(){return["facebook.com","fb.com","messenger.com","instagram.com","threads.net","twitter.com","x.com","linkedin.com","pinterest.com","pin.it","reddit.com","tumblr.com","snapchat.com","tiktok.com","douyin.com","youtube.com","youtu.be","whatsapp.com","telegram.org","t.me","medium.com","quora.com","flickr.com","vimeo.com","vk.com","weibo.com","sina.com.cn","line.me","discord.com","discordapp.com","twitch.tv","meetup.com","nextdoor.com","xing.com","yelp.com","zalo.me","mastodon.social","clubhouse.com","patreon.com","onlyfans.com","douban.com","goodreads.com","soundcloud.com","spotify.com","last.fm","behance.net","dribbble.com","deviantart.com","pixiv.net","slideshare.net","tinder.com","bumble.com","etsy.com","indeed.com","glassdoor.com","monster.com","careerbuilder.com","dice.com","ziprecruiter.com","simplyhired.com","upwork.com","freelancer.com","fiverr.com","stackoverflow.com","stackoverflow.co","angel.co","wellfound.com","quora.com","stackexchange.com","yahoo.com","answers.microsoft.com","askubuntu.com","superuser.com","serverfault.com","mathoverflow.net","xda-developers.com","gamespot.com","ign.com","4chan.org","9gag.com","gizmodo.com","slashdot.org","hacker-news.news","ycombinator.com","producthunt.com","discourse.org","google.com","google.co.uk","google.de","google.fr","google.co.jp","bing.com","yahoo.com","search.yahoo.com","duckduckgo.com","baidu.com","yandex.com","yandex.ru","ask.com","wolframalpha.com","ecosia.org","startpage.com","qwant.com","searx.me","gibiru.com","swisscows.com","gmail.com","googlemail.com","outlook.com","hotmail.com","live.com","msn.com","yahoo.com","ymail.com","aol.com","icloud.com","me.com","mac.com","protonmail.com","pm.me","zoho.com","mail.com","gmx.com","gmx.net","yandex.com","yandex.ru","tutanota.com","tutanota.de","fastmail.com","hushmail.com","mailbox.org","posteo.de","runbox.com","disroot.org","163.com","qq.com","rambler.ru","mail.ru","yelp.com","yelp.ca","yelp.co.uk","yelp.com.au","yellowpages.com","yellowpages.ca","yell.com","tripadvisor.com","tripadvisor.co.uk","tripadvisor.ca","foursquare.com","angieslist.com","bbb.org","manta.com","thumbtack.com","homeadvisor.com","superpages.com","whitepages.com","local.com","citysearch.com","merchantcircle.com","insiderpages.com","kudzu.com","hotfrog.com","buildzoom.com","houzz.com","porch.com","mapquest.com","zagat.com","zomato.com","opentable.com","viator.com","expedia.com","booking.com","airbnb.com","vrbo.com","homeaway.com","craigslist.org","nextdoor.com","patch.com","meetup.com","eventbrite.com","groupon.com","livingsocial.com","gumtree.com","gumtree.com.au","kijiji.ca","leboncoin.fr","finn.no","blocket.se","58.com","dianping.com","tabelog.com","ypcdn.com"]}export{k as Scraper,w as CappedSet};
|
package/dist/utils/index.cjs
CHANGED
package/dist/utils/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rezo",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.2",
|
|
4
4
|
"description": "Lightning-fast, enterprise-grade HTTP client for modern JavaScript. Full HTTP/2 support, intelligent cookie management, multiple adapters (HTTP, Fetch, cURL, XHR), streaming, proxy support (HTTP/HTTPS/SOCKS), and cross-environment compatibility.",
|
|
5
5
|
"main": "dist/index.cjs",
|
|
6
6
|
"module": "dist/index.js",
|
|
@@ -70,22 +70,58 @@
|
|
|
70
70
|
"linkedom": "^0.18.11",
|
|
71
71
|
"agent-base": "^7.1.4"
|
|
72
72
|
},
|
|
73
|
+
"typesVersions": {
|
|
74
|
+
"*": {
|
|
75
|
+
"platform/node": [
|
|
76
|
+
"dist/platform/node.d.ts"
|
|
77
|
+
],
|
|
78
|
+
"platform/browser": [
|
|
79
|
+
"dist/platform/browser.d.ts"
|
|
80
|
+
],
|
|
81
|
+
"platform/bun": [
|
|
82
|
+
"dist/platform/bun.d.ts"
|
|
83
|
+
],
|
|
84
|
+
"platform/deno": [
|
|
85
|
+
"dist/platform/deno.d.ts"
|
|
86
|
+
],
|
|
87
|
+
"platform/worker": [
|
|
88
|
+
"dist/platform/worker.d.ts"
|
|
89
|
+
],
|
|
90
|
+
"platform/react-native": [
|
|
91
|
+
"dist/platform/react-native.d.ts"
|
|
92
|
+
],
|
|
93
|
+
"adapters/http": [
|
|
94
|
+
"dist/adapters/entries/http.d.ts"
|
|
95
|
+
],
|
|
96
|
+
"adapters/http2": [
|
|
97
|
+
"dist/adapters/entries/http2.d.ts"
|
|
98
|
+
],
|
|
99
|
+
"adapters/curl": [
|
|
100
|
+
"dist/adapters/entries/curl.d.ts"
|
|
101
|
+
],
|
|
102
|
+
"adapters/fetch": [
|
|
103
|
+
"dist/adapters/entries/fetch.d.ts"
|
|
104
|
+
],
|
|
105
|
+
"adapters/xhr": [
|
|
106
|
+
"dist/adapters/entries/xhr.d.ts"
|
|
107
|
+
],
|
|
108
|
+
"adapters/react-native": [
|
|
109
|
+
"dist/adapters/entries/react-native.d.ts"
|
|
110
|
+
],
|
|
111
|
+
"crawler": [
|
|
112
|
+
"dist/crawler.d.ts"
|
|
113
|
+
],
|
|
114
|
+
"dom": [
|
|
115
|
+
"dist/dom/index.d.ts"
|
|
116
|
+
]
|
|
117
|
+
}
|
|
118
|
+
},
|
|
73
119
|
"exports": {
|
|
74
120
|
".": {
|
|
75
121
|
"types": "./dist/index.d.ts",
|
|
76
122
|
"import": "./dist/index.js",
|
|
77
123
|
"require": "./dist/index.cjs"
|
|
78
124
|
},
|
|
79
|
-
"./crawler": {
|
|
80
|
-
"types": "./dist/crawler.d.ts",
|
|
81
|
-
"import": "./dist/entries/crawler.js",
|
|
82
|
-
"require": "./dist/entries/crawler.cjs"
|
|
83
|
-
},
|
|
84
|
-
"./dom": {
|
|
85
|
-
"types": "./dist/dom/index.d.ts",
|
|
86
|
-
"import": "./dist/dom/index.js",
|
|
87
|
-
"require": "./dist/dom/index.cjs"
|
|
88
|
-
},
|
|
89
125
|
"./platform/node": {
|
|
90
126
|
"types": "./dist/platform/node.d.ts",
|
|
91
127
|
"import": "./dist/platform/node.js",
|
|
@@ -126,16 +162,16 @@
|
|
|
126
162
|
"import": "./dist/adapters/entries/http2.js",
|
|
127
163
|
"require": "./dist/adapters/entries/http2.cjs"
|
|
128
164
|
},
|
|
129
|
-
"./adapters/fetch": {
|
|
130
|
-
"types": "./dist/adapters/entries/fetch.d.ts",
|
|
131
|
-
"import": "./dist/adapters/entries/fetch.js",
|
|
132
|
-
"require": "./dist/adapters/entries/fetch.cjs"
|
|
133
|
-
},
|
|
134
165
|
"./adapters/curl": {
|
|
135
166
|
"types": "./dist/adapters/entries/curl.d.ts",
|
|
136
167
|
"import": "./dist/adapters/entries/curl.js",
|
|
137
168
|
"require": "./dist/adapters/entries/curl.cjs"
|
|
138
169
|
},
|
|
170
|
+
"./adapters/fetch": {
|
|
171
|
+
"types": "./dist/adapters/entries/fetch.d.ts",
|
|
172
|
+
"import": "./dist/adapters/entries/fetch.js",
|
|
173
|
+
"require": "./dist/adapters/entries/fetch.cjs"
|
|
174
|
+
},
|
|
139
175
|
"./adapters/xhr": {
|
|
140
176
|
"types": "./dist/adapters/entries/xhr.d.ts",
|
|
141
177
|
"import": "./dist/adapters/entries/xhr.js",
|
|
@@ -146,7 +182,16 @@
|
|
|
146
182
|
"import": "./dist/adapters/entries/react-native.js",
|
|
147
183
|
"require": "./dist/adapters/entries/react-native.cjs"
|
|
148
184
|
},
|
|
149
|
-
"./
|
|
185
|
+
"./crawler": {
|
|
186
|
+
"types": "./dist/crawler.d.ts",
|
|
187
|
+
"import": "./dist/entries/crawler.js",
|
|
188
|
+
"require": "./dist/entries/crawler.cjs"
|
|
189
|
+
},
|
|
190
|
+
"./dom": {
|
|
191
|
+
"types": "./dist/dom/index.d.ts",
|
|
192
|
+
"import": "./dist/dom/index.js",
|
|
193
|
+
"require": "./dist/dom/index.cjs"
|
|
194
|
+
}
|
|
150
195
|
},
|
|
151
196
|
"browser": {
|
|
152
197
|
"./dist/platform/node.js": "./dist/platform/browser.js",
|