thordata-js-sdk 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -20
- package/README.md +282 -88
- package/dist/examples/basic_serp.js +3 -3
- package/dist/examples/basic_serp.js.map +1 -1
- package/dist/examples/location_api.js +47 -0
- package/dist/examples/location_api.js.map +1 -0
- package/dist/examples/proxy_datacenter.d.ts +1 -0
- package/dist/examples/proxy_datacenter.js +18 -0
- package/dist/examples/proxy_datacenter.js.map +1 -0
- package/dist/examples/proxy_isp.d.ts +1 -0
- package/dist/examples/proxy_isp.js +21 -0
- package/dist/examples/proxy_isp.js.map +1 -0
- package/dist/examples/proxy_mobile.d.ts +1 -0
- package/dist/examples/proxy_mobile.js +18 -0
- package/dist/examples/proxy_mobile.js.map +1 -0
- package/dist/examples/proxy_residential.d.ts +1 -0
- package/dist/examples/proxy_residential.js +27 -0
- package/dist/examples/proxy_residential.js.map +1 -0
- package/dist/examples/serp_google_news.d.ts +0 -6
- package/dist/examples/serp_google_news.js +7 -10
- package/dist/examples/serp_google_news.js.map +1 -1
- package/dist/src/client.d.ts +114 -5
- package/dist/src/client.js +192 -42
- package/dist/src/client.js.map +1 -1
- package/dist/src/enums.d.ts +221 -3
- package/dist/src/enums.js +291 -2
- package/dist/src/enums.js.map +1 -1
- package/dist/src/errors.d.ts +37 -5
- package/dist/src/errors.js +61 -17
- package/dist/src/errors.js.map +1 -1
- package/dist/src/index.d.ts +8 -6
- package/dist/src/index.js +22 -6
- package/dist/src/index.js.map +1 -1
- package/dist/src/models.d.ts +122 -83
- package/dist/src/models.js +1 -90
- package/dist/src/models.js.map +1 -1
- package/dist/src/proxy.d.ts +100 -0
- package/dist/src/proxy.js +227 -0
- package/dist/src/proxy.js.map +1 -0
- package/dist/src/retry.d.ts +25 -3
- package/dist/src/retry.js +16 -0
- package/dist/src/retry.js.map +1 -1
- package/dist/src/thordata.d.ts +10 -0
- package/dist/src/thordata.js +46 -0
- package/dist/src/thordata.js.map +1 -0
- package/dist/src/utils.d.ts +18 -8
- package/dist/src/utils.js +58 -34
- package/dist/src/utils.js.map +1 -1
- package/package.json +5 -6
- package/dist/examples/proxy_ip_check.js +0 -37
- package/dist/examples/proxy_ip_check.js.map +0 -1
- /package/dist/examples/{proxy_ip_check.d.ts → location_api.d.ts} +0 -0
package/LICENSE
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2025 Thordata
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Thordata
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
21
|
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
# Thordata JS SDK (Node.js / TypeScript)
|
|
2
2
|
|
|
3
|
-
Official JavaScript/TypeScript SDK for
|
|
3
|
+
Official JavaScript/TypeScript SDK for Thordata APIs.
|
|
4
4
|
|
|
5
5
|
This SDK supports:
|
|
6
6
|
|
|
7
|
-
- **SERP API** (Google / Bing / Yandex / DuckDuckGo
|
|
7
|
+
- **SERP API** (Google / Bing / Yandex / DuckDuckGo)
|
|
8
8
|
- **Web Unlocker / Universal API**
|
|
9
9
|
- **Web Scraper API** (task-based scraping)
|
|
10
|
+
- **Proxy Network** (Residential / Datacenter / Mobile / ISP)
|
|
11
|
+
- **Location API** (Geo-targeting options)
|
|
10
12
|
|
|
11
13
|
It is designed to be:
|
|
12
14
|
|
|
13
|
-
- **TypeScript-first**
|
|
14
|
-
- **ESM-ready**
|
|
15
|
-
- **Offline-test friendly** (base URLs can be overridden
|
|
15
|
+
- **TypeScript-first** with full type definitions
|
|
16
|
+
- **ESM-ready** with CommonJS fallback
|
|
17
|
+
- **Offline-test friendly** (base URLs can be overridden)
|
|
16
18
|
|
|
17
19
|
---
|
|
18
20
|
|
|
@@ -22,7 +24,7 @@ It is designed to be:
|
|
|
22
24
|
npm install thordata-js-sdk
|
|
23
25
|
```
|
|
24
26
|
|
|
25
|
-
|
|
27
|
+
For local development:
|
|
26
28
|
|
|
27
29
|
```bash
|
|
28
30
|
git clone https://github.com/Thordata/thordata-js-sdk.git
|
|
@@ -38,15 +40,15 @@ npm run build
|
|
|
38
40
|
Set environment variables:
|
|
39
41
|
|
|
40
42
|
```bash
|
|
41
|
-
export
|
|
43
|
+
export THORDATA_TOKEN=your_scraper_token
|
|
42
44
|
export THORDATA_PUBLIC_TOKEN=your_public_token
|
|
43
45
|
export THORDATA_PUBLIC_KEY=your_public_key
|
|
44
46
|
```
|
|
45
47
|
|
|
46
|
-
Or create a `.env` file
|
|
48
|
+
Or create a `.env` file:
|
|
47
49
|
|
|
48
50
|
```env
|
|
49
|
-
|
|
51
|
+
THORDATA_TOKEN=your_scraper_token
|
|
50
52
|
THORDATA_PUBLIC_TOKEN=your_public_token
|
|
51
53
|
THORDATA_PUBLIC_KEY=your_public_key
|
|
52
54
|
```
|
|
@@ -55,97 +57,129 @@ THORDATA_PUBLIC_KEY=your_public_key
|
|
|
55
57
|
|
|
56
58
|
## 🚀 Quick Start
|
|
57
59
|
|
|
58
|
-
|
|
60
|
+
Create a client:
|
|
59
61
|
|
|
60
62
|
```typescript
|
|
61
63
|
import { ThordataClient } from "thordata-js-sdk";
|
|
62
64
|
|
|
63
65
|
const client = new ThordataClient({
|
|
64
|
-
scraperToken: process.env.
|
|
66
|
+
scraperToken: process.env.THORDATA_TOKEN!,
|
|
65
67
|
publicToken: process.env.THORDATA_PUBLIC_TOKEN,
|
|
66
68
|
publicKey: process.env.THORDATA_PUBLIC_KEY,
|
|
67
69
|
});
|
|
68
70
|
```
|
|
69
71
|
|
|
72
|
+
Or use the convenience wrapper:
|
|
73
|
+
|
|
74
|
+
```typescript
|
|
75
|
+
import { Thordata } from "thordata-js-sdk";
|
|
76
|
+
|
|
77
|
+
// Automatically reads from environment variables
|
|
78
|
+
const client = new Thordata();
|
|
79
|
+
```
|
|
80
|
+
|
|
70
81
|
---
|
|
71
82
|
|
|
72
83
|
## 🔍 SERP API
|
|
73
84
|
|
|
74
|
-
###
|
|
85
|
+
### Supported Engines
|
|
86
|
+
|
|
87
|
+
| Engine | Identifier | Notes |
|
|
88
|
+
| ---------- | ---------- | ---------------------- |
|
|
89
|
+
| Google | google | Base web search |
|
|
90
|
+
| Bing | bing | Base web search |
|
|
91
|
+
| Yandex | yandex | Uses text instead of q |
|
|
92
|
+
| DuckDuckGo | duckduckgo | Base web search |
|
|
93
|
+
|
|
94
|
+
### Google Specialized Engines
|
|
95
|
+
|
|
96
|
+
For Google verticals, use dedicated engines for best results:
|
|
97
|
+
|
|
98
|
+
| Engine | Identifier |
|
|
99
|
+
| --------------- | --------------- |
|
|
100
|
+
| Google News | google_news |
|
|
101
|
+
| Google Shopping | google_shopping |
|
|
102
|
+
| Google Images | google_images |
|
|
103
|
+
| Google Videos | google_videos |
|
|
104
|
+
| Google Maps | google_maps |
|
|
105
|
+
| Google Scholar | google_scholar |
|
|
106
|
+
| Google Patents | google_patents |
|
|
107
|
+
| Google Jobs | google_jobs |
|
|
108
|
+
| Google Flights | google_flights |
|
|
109
|
+
| Google Finance | google_finance |
|
|
110
|
+
| Google Product | google_product |
|
|
111
|
+
|
|
112
|
+
### Basic Search
|
|
75
113
|
|
|
76
114
|
```typescript
|
|
77
115
|
import { ThordataClient, Engine } from "thordata-js-sdk";
|
|
78
116
|
|
|
79
|
-
const client = new ThordataClient({ scraperToken: process.env.
|
|
117
|
+
const client = new ThordataClient({ scraperToken: process.env.THORDATA_TOKEN! });
|
|
80
118
|
|
|
81
119
|
const data = await client.serpSearch({
|
|
82
120
|
query: "Thordata proxy network",
|
|
83
121
|
engine: Engine.GOOGLE,
|
|
84
|
-
|
|
122
|
+
country: "us",
|
|
123
|
+
language: "en",
|
|
124
|
+
num: 10,
|
|
85
125
|
});
|
|
86
126
|
|
|
87
127
|
const organic = data?.organic ?? [];
|
|
88
128
|
console.log(`Found ${organic.length} organic results`);
|
|
89
|
-
|
|
90
|
-
for (const item of organic.slice(0, 3)) {
|
|
91
|
-
console.log("-", item.title, "->", item.link);
|
|
92
|
-
}
|
|
93
129
|
```
|
|
94
130
|
|
|
95
|
-
###
|
|
96
|
-
|
|
97
|
-
Thordata supports both:
|
|
98
|
-
|
|
99
|
-
- Dedicated engines (recommended): `google_news`, `google_shopping`
|
|
100
|
-
- Generic Google + tbm via `searchType` (alternative)
|
|
101
|
-
|
|
102
|
-
#### Google News (recommended):
|
|
131
|
+
### Google News (Recommended: Dedicated Engine)
|
|
103
132
|
|
|
104
133
|
```typescript
|
|
105
|
-
const
|
|
134
|
+
const news = await client.serpSearch({
|
|
106
135
|
query: "AI regulation",
|
|
107
|
-
engine:
|
|
136
|
+
engine: Engine.GOOGLE_NEWS,
|
|
108
137
|
country: "us",
|
|
109
138
|
language: "en",
|
|
110
139
|
num: 10,
|
|
111
|
-
so: 1, // 0=relevance, 1=date (Google News)
|
|
112
140
|
});
|
|
113
141
|
```
|
|
114
142
|
|
|
115
|
-
|
|
143
|
+
### Google Shopping
|
|
116
144
|
|
|
117
145
|
```typescript
|
|
118
|
-
const
|
|
146
|
+
const shopping = await client.serpSearch({
|
|
119
147
|
query: "iPhone 15",
|
|
120
|
-
engine:
|
|
148
|
+
engine: Engine.GOOGLE_SHOPPING,
|
|
121
149
|
country: "us",
|
|
122
150
|
language: "en",
|
|
123
151
|
num: 10,
|
|
124
|
-
min_price: 500,
|
|
125
|
-
max_price: 1500,
|
|
126
152
|
});
|
|
127
153
|
```
|
|
128
154
|
|
|
129
|
-
|
|
155
|
+
### Alternative: Using tbm Parameter
|
|
156
|
+
|
|
157
|
+
For some Google engines, you can also use the searchType (tbm) parameter:
|
|
130
158
|
|
|
131
159
|
```typescript
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
num: 10,
|
|
160
|
+
import { GoogleTbm } from "thordata-js-sdk";
|
|
161
|
+
|
|
162
|
+
const images = await client.serpSearch({
|
|
163
|
+
query: "cats",
|
|
164
|
+
engine: Engine.GOOGLE,
|
|
165
|
+
searchType: GoogleTbm.IMAGES, // or just "isch"
|
|
139
166
|
});
|
|
140
167
|
```
|
|
141
168
|
|
|
142
|
-
|
|
169
|
+
> **Note:** Using dedicated engines (e.g., `Engine.GOOGLE_NEWS`) is recommended over `Engine.GOOGLE + searchType` for clearer parameter contracts.
|
|
143
170
|
|
|
144
171
|
---
|
|
145
172
|
|
|
146
|
-
##
|
|
173
|
+
## 🌐 Web Unlocker / Universal API
|
|
147
174
|
|
|
148
|
-
###
|
|
175
|
+
### Supported Output Formats
|
|
176
|
+
|
|
177
|
+
| Format | Description |
|
|
178
|
+
| ------ | ---------------------- |
|
|
179
|
+
| html | HTML content (default) |
|
|
180
|
+
| png | PNG screenshot |
|
|
181
|
+
|
|
182
|
+
### Basic HTML Scraping
|
|
149
183
|
|
|
150
184
|
```typescript
|
|
151
185
|
const html = await client.universalScrape({
|
|
@@ -157,14 +191,25 @@ const html = await client.universalScrape({
|
|
|
157
191
|
console.log(String(html).slice(0, 300));
|
|
158
192
|
```
|
|
159
193
|
|
|
160
|
-
### JS
|
|
194
|
+
### JS Rendering + Wait for Selector
|
|
161
195
|
|
|
162
196
|
```typescript
|
|
163
197
|
const html = await client.universalScrape({
|
|
164
198
|
url: "https://example.com/spa",
|
|
165
199
|
jsRender: true,
|
|
166
200
|
outputFormat: "html",
|
|
167
|
-
waitFor: ".main-content",
|
|
201
|
+
waitFor: ".main-content", // Wait for element to appear
|
|
202
|
+
});
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
### Block Resources for Performance
|
|
206
|
+
|
|
207
|
+
```typescript
|
|
208
|
+
const html = await client.universalScrape({
|
|
209
|
+
url: "https://example.com",
|
|
210
|
+
jsRender: true,
|
|
211
|
+
blockResources: "image,media", // Block images and media
|
|
212
|
+
cleanContent: "js,css", // Remove JS and CSS from output
|
|
168
213
|
});
|
|
169
214
|
```
|
|
170
215
|
|
|
@@ -182,6 +227,131 @@ const pngBytes = await client.universalScrape({
|
|
|
182
227
|
writeFileSync("screenshot.png", pngBytes as Buffer);
|
|
183
228
|
```
|
|
184
229
|
|
|
230
|
+
### Custom Headers and Cookies
|
|
231
|
+
|
|
232
|
+
```typescript
|
|
233
|
+
const html = await client.universalScrape({
|
|
234
|
+
url: "https://example.com/account",
|
|
235
|
+
jsRender: true,
|
|
236
|
+
headers: [
|
|
237
|
+
{ name: "User-Agent", value: "Mozilla/5.0 (ThordataBot)" },
|
|
238
|
+
{ name: "Accept-Language", value: "en-US,en;q=0.9" },
|
|
239
|
+
],
|
|
240
|
+
cookies: [{ name: "sessionid", value: "abc123" }],
|
|
241
|
+
});
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
## 🌍 Proxy Network
|
|
247
|
+
|
|
248
|
+
Each proxy product requires separate credentials from Thordata Dashboard.
|
|
249
|
+
|
|
250
|
+
### Environment Variables
|
|
251
|
+
|
|
252
|
+
```env
|
|
253
|
+
# Residential Proxy (port 9999)
|
|
254
|
+
THORDATA_RESIDENTIAL_USERNAME=your_residential_username
|
|
255
|
+
THORDATA_RESIDENTIAL_PASSWORD=your_residential_password
|
|
256
|
+
|
|
257
|
+
# Datacenter Proxy (port 7777)
|
|
258
|
+
THORDATA_DATACENTER_USERNAME=your_datacenter_username
|
|
259
|
+
THORDATA_DATACENTER_PASSWORD=your_datacenter_password
|
|
260
|
+
|
|
261
|
+
# Mobile Proxy (port 5555)
|
|
262
|
+
THORDATA_MOBILE_USERNAME=your_mobile_username
|
|
263
|
+
THORDATA_MOBILE_PASSWORD=your_mobile_password
|
|
264
|
+
|
|
265
|
+
# Static ISP Proxy (port 6666)
|
|
266
|
+
THORDATA_ISP_HOST=your_static_ip_address
|
|
267
|
+
THORDATA_ISP_USERNAME=your_isp_username
|
|
268
|
+
THORDATA_ISP_PASSWORD=your_isp_password
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
### Residential Proxy
|
|
272
|
+
|
|
273
|
+
```typescript
|
|
274
|
+
import { Thordata } from "thordata-js-sdk";
|
|
275
|
+
|
|
276
|
+
const client = new Thordata();
|
|
277
|
+
|
|
278
|
+
// Basic US residential
|
|
279
|
+
const proxy = Thordata.Proxy.residentialFromEnv().country("us");
|
|
280
|
+
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
281
|
+
console.log("IP:", result.origin);
|
|
282
|
+
|
|
283
|
+
// Sticky session (same IP for 30 minutes)
|
|
284
|
+
const stickyProxy = Thordata.Proxy.residentialFromEnv()
|
|
285
|
+
.country("jp")
|
|
286
|
+
.city("tokyo")
|
|
287
|
+
.session("my_session")
|
|
288
|
+
.sticky(30);
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
### Datacenter Proxy
|
|
292
|
+
|
|
293
|
+
```typescript
|
|
294
|
+
const proxy = Thordata.Proxy.datacenterFromEnv();
|
|
295
|
+
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
### Mobile Proxy
|
|
299
|
+
|
|
300
|
+
```typescript
|
|
301
|
+
const proxy = Thordata.Proxy.mobileFromEnv().country("gb");
|
|
302
|
+
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
### Static ISP Proxy
|
|
306
|
+
|
|
307
|
+
```typescript
|
|
308
|
+
const proxy = Thordata.Proxy.ispFromEnv();
|
|
309
|
+
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
---
|
|
313
|
+
|
|
314
|
+
## 📍 Location API
|
|
315
|
+
|
|
316
|
+
Query available geo-targeting options. Requires `THORDATA_PUBLIC_TOKEN` and `THORDATA_PUBLIC_KEY`.
|
|
317
|
+
|
|
318
|
+
### List Countries
|
|
319
|
+
|
|
320
|
+
```typescript
|
|
321
|
+
// Using string parameter (recommended)
|
|
322
|
+
const countries = await client.listCountries("residential");
|
|
323
|
+
|
|
324
|
+
// Or numeric parameter (1 = residential, 2 = unlimited)
|
|
325
|
+
const countries2 = await client.listCountries(1);
|
|
326
|
+
|
|
327
|
+
console.log(`Found ${countries.length} countries`);
|
|
328
|
+
// [{ country_code: "US", country_name: "United States" }, ...]
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
### List States
|
|
332
|
+
|
|
333
|
+
```typescript
|
|
334
|
+
const states = await client.listStates("US", "residential");
|
|
335
|
+
console.log(`Found ${states.length} states`);
|
|
336
|
+
// [{ state_code: "california", state_name: "California" }, ...]
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
### List Cities
|
|
340
|
+
|
|
341
|
+
```typescript
|
|
342
|
+
const cities = await client.listCities("US", "california", "residential");
|
|
343
|
+
console.log(`Found ${cities.length} cities`);
|
|
344
|
+
// [{ city_name: "Los Angeles" }, { city_name: "San Francisco" }, ...]
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
### List ASNs
|
|
348
|
+
|
|
349
|
+
```typescript
|
|
350
|
+
const asns = await client.listAsns("US", "residential");
|
|
351
|
+
console.log(`Found ${asns.length} ASNs`);
|
|
352
|
+
// [{ asn_code: "AS7922", asn_name: "Comcast" }, ...]
|
|
353
|
+
```
|
|
354
|
+
|
|
185
355
|
---
|
|
186
356
|
|
|
187
357
|
## 🕷️ Web Scraper API (Task-based)
|
|
@@ -190,11 +360,12 @@ Requires `THORDATA_PUBLIC_TOKEN` and `THORDATA_PUBLIC_KEY`.
|
|
|
190
360
|
|
|
191
361
|
```typescript
|
|
192
362
|
const client = new ThordataClient({
|
|
193
|
-
scraperToken: process.env.
|
|
363
|
+
scraperToken: process.env.THORDATA_TOKEN!,
|
|
194
364
|
publicToken: process.env.THORDATA_PUBLIC_TOKEN,
|
|
195
365
|
publicKey: process.env.THORDATA_PUBLIC_KEY,
|
|
196
366
|
});
|
|
197
367
|
|
|
368
|
+
// Create task
|
|
198
369
|
const taskId = await client.createScraperTask({
|
|
199
370
|
fileName: "demo_task",
|
|
200
371
|
spiderId: "example-spider-id",
|
|
@@ -204,6 +375,7 @@ const taskId = await client.createScraperTask({
|
|
|
204
375
|
|
|
205
376
|
console.log("Task created:", taskId);
|
|
206
377
|
|
|
378
|
+
// Wait for completion
|
|
207
379
|
const status = await client.waitForTask(taskId, {
|
|
208
380
|
pollIntervalMs: 5000,
|
|
209
381
|
maxWaitMs: 60000,
|
|
@@ -211,6 +383,7 @@ const status = await client.waitForTask(taskId, {
|
|
|
211
383
|
|
|
212
384
|
console.log("Final status:", status);
|
|
213
385
|
|
|
386
|
+
// Download results
|
|
214
387
|
if (["ready", "success", "finished"].includes(status.toLowerCase())) {
|
|
215
388
|
const downloadUrl = await client.getTaskResult(taskId, "json");
|
|
216
389
|
console.log("Download URL:", downloadUrl);
|
|
@@ -219,20 +392,22 @@ if (["ready", "success", "finished"].includes(status.toLowerCase())) {
|
|
|
219
392
|
|
|
220
393
|
---
|
|
221
394
|
|
|
222
|
-
## 🔧
|
|
395
|
+
## 🔧 Error Handling
|
|
396
|
+
|
|
397
|
+
The SDK throws typed errors based on API response codes:
|
|
223
398
|
|
|
224
|
-
|
|
399
|
+
| Code | Meaning | Error Class |
|
|
400
|
+
| ------- | ------------- | ------------------------- |
|
|
401
|
+
| 200 | Success | - |
|
|
402
|
+
| 300 | Not collected | ThordataNotCollectedError |
|
|
403
|
+
| 400 | Bad request | ThordataValidationError |
|
|
404
|
+
| 401/403 | Auth error | ThordataAuthError |
|
|
405
|
+
| 402/429 | Rate limit | ThordataRateLimitError |
|
|
406
|
+
| 5xx | Server error | ThordataServerError |
|
|
225
407
|
|
|
226
|
-
|
|
227
|
-
| ------- | --------------------- | ---------------------------------------------- |
|
|
228
|
-
| 200 | Success | - |
|
|
229
|
-
| 300 | Not collected | `ThordataNotCollectedError` |
|
|
230
|
-
| 400 | Bad request | `ThordataValidationError` |
|
|
231
|
-
| 401/403 | Auth/Forbidden | `ThordataAuthError` |
|
|
232
|
-
| 402/429 | Quota/Rate limit | `ThordataRateLimitError` |
|
|
233
|
-
| 5xx | Server/timeout issues | `ThordataServerError` / `ThordataTimeoutError` |
|
|
408
|
+
> **Billing Note:** Only requests with code == 200 are billed.
|
|
234
409
|
|
|
235
|
-
### Example
|
|
410
|
+
### Example
|
|
236
411
|
|
|
237
412
|
```typescript
|
|
238
413
|
import {
|
|
@@ -262,18 +437,37 @@ try {
|
|
|
262
437
|
|
|
263
438
|
---
|
|
264
439
|
|
|
265
|
-
##
|
|
440
|
+
## ⚙️ Advanced Configuration
|
|
266
441
|
|
|
267
|
-
|
|
442
|
+
### SSL Verification
|
|
443
|
+
|
|
444
|
+
```typescript
|
|
445
|
+
const client = new ThordataClient({
|
|
446
|
+
scraperToken: process.env.THORDATA_TOKEN!,
|
|
447
|
+
verifySsl: false, // Only for testing with self-signed certs
|
|
448
|
+
});
|
|
449
|
+
```
|
|
450
|
+
|
|
451
|
+
### Retry Configuration
|
|
452
|
+
|
|
453
|
+
```typescript
|
|
454
|
+
const client = new ThordataClient({
|
|
455
|
+
scraperToken: process.env.THORDATA_TOKEN!,
|
|
456
|
+
maxRetries: 3, // Retry up to 3 times on transient failures
|
|
457
|
+
timeoutMs: 60000, // 60 second timeout
|
|
458
|
+
});
|
|
459
|
+
```
|
|
460
|
+
|
|
461
|
+
### Base URL Overrides
|
|
462
|
+
|
|
463
|
+
For testing or custom routing:
|
|
268
464
|
|
|
269
465
|
```bash
|
|
270
466
|
export THORDATA_SCRAPERAPI_BASE_URL=http://127.0.0.1:12345
|
|
271
467
|
export THORDATA_UNIVERSALAPI_BASE_URL=http://127.0.0.1:12345
|
|
272
|
-
export THORDATA_WEB_SCRAPER_API_BASE_URL=http://127.0.0.1:12345
|
|
273
|
-
export THORDATA_LOCATIONS_BASE_URL=http://127.0.0.1:12345
|
|
274
468
|
```
|
|
275
469
|
|
|
276
|
-
Or via
|
|
470
|
+
Or via config:
|
|
277
471
|
|
|
278
472
|
```typescript
|
|
279
473
|
const client = new ThordataClient({
|
|
@@ -290,9 +484,11 @@ const client = new ThordataClient({
|
|
|
290
484
|
npm install
|
|
291
485
|
npm run build
|
|
292
486
|
npm test
|
|
487
|
+
npm run lint
|
|
488
|
+
npm run format
|
|
293
489
|
```
|
|
294
490
|
|
|
295
|
-
### Run
|
|
491
|
+
### Run Examples
|
|
296
492
|
|
|
297
493
|
```bash
|
|
298
494
|
node dist/examples/basic_serp.js
|
|
@@ -306,37 +502,35 @@ node dist/examples/basic_universal.js
|
|
|
306
502
|
```
|
|
307
503
|
thordata-js-sdk/
|
|
308
504
|
├── src/
|
|
309
|
-
│ ├── index.ts
|
|
310
|
-
│ ├── client.ts
|
|
311
|
-
│ ├──
|
|
312
|
-
│ ├──
|
|
313
|
-
│ ├──
|
|
314
|
-
│ ├──
|
|
315
|
-
│ ├──
|
|
316
|
-
│
|
|
505
|
+
│ ├── index.ts # Main exports
|
|
506
|
+
│ ├── client.ts # ThordataClient class
|
|
507
|
+
│ ├── thordata.ts # Convenience wrapper
|
|
508
|
+
│ ├── proxy.ts # Proxy configuration
|
|
509
|
+
│ ├── models.ts # TypeScript interfaces
|
|
510
|
+
│ ├── enums.ts # Enumerations
|
|
511
|
+
│ ├── errors.ts # Error classes
|
|
512
|
+
│ ├── utils.ts # Helper functions
|
|
513
|
+
│ └── endpoints.ts # API endpoints
|
|
317
514
|
├── examples/
|
|
318
|
-
│ ├── basic_serp.ts
|
|
319
|
-
│ ├── basic_universal.ts
|
|
320
|
-
│ ├── basic_scraper_task.ts
|
|
321
|
-
│ └── serp_google_news.ts
|
|
322
515
|
├── tests/
|
|
323
|
-
│ ├── serp.offline.test.ts
|
|
324
|
-
│ ├── mockServer.ts
|
|
325
|
-
│ └── examples.e2e.test.ts
|
|
326
|
-
├── .github/workflows/ci.yml
|
|
327
516
|
├── package.json
|
|
328
|
-
├── tsconfig.json
|
|
329
|
-
├── tsconfig.build.json
|
|
330
517
|
└── README.md
|
|
331
518
|
```
|
|
332
519
|
|
|
333
520
|
---
|
|
334
521
|
|
|
335
|
-
##
|
|
522
|
+
## 📚 API Reference
|
|
336
523
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
-
|
|
340
|
-
-
|
|
524
|
+
For detailed API documentation, visit:
|
|
525
|
+
|
|
526
|
+
- [Google Search Parameters](#)
|
|
527
|
+
- [Google News Parameters](#)
|
|
528
|
+
- [Google Shopping Parameters](#)
|
|
529
|
+
- [Yandex Parameters](#)
|
|
530
|
+
- [Universal API Parameters](#)
|
|
341
531
|
|
|
342
532
|
---
|
|
533
|
+
|
|
534
|
+
## 📄 License
|
|
535
|
+
|
|
536
|
+
MIT License - see LICENSE for details.
|
|
@@ -14,10 +14,10 @@ async function main() {
|
|
|
14
14
|
engine: Engine.GOOGLE,
|
|
15
15
|
num: 5,
|
|
16
16
|
});
|
|
17
|
-
//
|
|
17
|
+
// Debug: show actual response structure
|
|
18
18
|
console.dir(results, { depth: 4 });
|
|
19
|
-
//
|
|
20
|
-
const organic = results?.organic ?? results?.organic_results ?? [];
|
|
19
|
+
// Type-safe access with fallback
|
|
20
|
+
const organic = (results?.organic ?? results?.organic_results ?? []);
|
|
21
21
|
console.log(`Found ${organic.length} organic results`);
|
|
22
22
|
for (const item of organic.slice(0, 3)) {
|
|
23
23
|
console.log("-", item.title, "->", item.link);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"basic_serp.js","sourceRoot":"","sources":["../../examples/basic_serp.ts"],"names":[],"mappings":"AAAA,yBAAyB;AAEzB,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzD,KAAK,UAAU,IAAI;IACjB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACjD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC;QACtC,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,CAAC;KACP,CAAC,CAAC;IAEH,
|
|
1
|
+
{"version":3,"file":"basic_serp.js","sourceRoot":"","sources":["../../examples/basic_serp.ts"],"names":[],"mappings":"AAAA,yBAAyB;AAEzB,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzD,KAAK,UAAU,IAAI;IACjB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACjD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC;QACtC,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,CAAC;KACP,CAAC,CAAC;IAEH,wCAAwC;IACxC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEnC,iCAAiC;IACjC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,eAAe,IAAI,EAAE,CAGjE,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;IACvD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC/B,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// examples/location_api.ts
|
|
2
|
+
import "dotenv/config";
|
|
3
|
+
import { Thordata } from "../src/thordata.js";
|
|
4
|
+
async function main() {
|
|
5
|
+
if (!process.env.THORDATA_PUBLIC_TOKEN || !process.env.THORDATA_PUBLIC_KEY) {
|
|
6
|
+
console.log("Location API Demo - Skipped");
|
|
7
|
+
console.log("Set THORDATA_PUBLIC_TOKEN and THORDATA_PUBLIC_KEY in .env");
|
|
8
|
+
return;
|
|
9
|
+
}
|
|
10
|
+
const client = new Thordata();
|
|
11
|
+
console.log("Location API Demo\n");
|
|
12
|
+
// List countries (now using string parameter)
|
|
13
|
+
console.log("1. Listing countries...");
|
|
14
|
+
const countries = await client.listCountries("residential");
|
|
15
|
+
console.log(` Found ${countries.length} countries`);
|
|
16
|
+
console.log(` First 5: ${countries
|
|
17
|
+
.slice(0, 5)
|
|
18
|
+
.map((c) => c.country_code)
|
|
19
|
+
.join(", ")}`);
|
|
20
|
+
// List states for US
|
|
21
|
+
console.log("\n2. Listing US states...");
|
|
22
|
+
const states = await client.listStates("US", "residential");
|
|
23
|
+
console.log(` Found ${states.length} states`);
|
|
24
|
+
console.log(` First 5: ${states
|
|
25
|
+
.slice(0, 5)
|
|
26
|
+
.map((s) => s.state_name)
|
|
27
|
+
.join(", ")}`);
|
|
28
|
+
// List cities for California
|
|
29
|
+
console.log("\n3. Listing California cities...");
|
|
30
|
+
const cities = await client.listCities("US", "california", "residential");
|
|
31
|
+
console.log(` Found ${cities.length} cities`);
|
|
32
|
+
console.log(` First 5: ${cities
|
|
33
|
+
.slice(0, 5)
|
|
34
|
+
.map((c) => c.city_name)
|
|
35
|
+
.join(", ")}`);
|
|
36
|
+
// List ASNs for US
|
|
37
|
+
console.log("\n4. Listing US ASNs...");
|
|
38
|
+
const asns = await client.listAsns("US", "residential");
|
|
39
|
+
console.log(` Found ${asns.length} ASNs`);
|
|
40
|
+
console.log(` First 3: ${asns
|
|
41
|
+
.slice(0, 3)
|
|
42
|
+
.map((a) => `${a.asn_code}: ${a.asn_name}`)
|
|
43
|
+
.join(", ")}`);
|
|
44
|
+
console.log("\nLocation API demo completed.");
|
|
45
|
+
}
|
|
46
|
+
main().catch(console.error);
|
|
47
|
+
//# sourceMappingURL=location_api.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"location_api.js","sourceRoot":"","sources":["../../examples/location_api.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAE3B,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,KAAK,UAAU,IAAI;IACjB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,CAAC;QAC3E,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;QACzE,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;IAE9B,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IAEnC,8CAA8C;IAC9C,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,CAAC,MAAM,YAAY,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CACT,eAAe,SAAS;SACrB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;SAC1B,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;IAEF,qBAAqB;IACrB,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CACT,eAAe,MAAM;SAClB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;SACxB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;IAEF,6BAA6B;IAC7B,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CACT,eAAe,MAAM;SAClB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;SACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;IAEF,mBAAmB;IACnB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,MAAM,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CACT,eAAe,IAAI;SAChB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;SAC1C,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;AAChD,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import "dotenv/config";
|