thordata-js-sdk 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -20
- package/README.md +235 -123
- package/dist/examples/basic_serp.js +3 -3
- package/dist/examples/basic_serp.js.map +1 -1
- package/dist/examples/location_api.js +47 -0
- package/dist/examples/location_api.js.map +1 -0
- package/dist/examples/proxy_datacenter.js +1 -1
- package/dist/examples/proxy_datacenter.js.map +1 -1
- package/dist/examples/proxy_isp.js +1 -1
- package/dist/examples/proxy_isp.js.map +1 -1
- package/dist/examples/proxy_mobile.js +1 -1
- package/dist/examples/proxy_mobile.js.map +1 -1
- package/dist/examples/proxy_residential.js +2 -2
- package/dist/examples/proxy_residential.js.map +1 -1
- package/dist/examples/serp_google_news.d.ts +0 -6
- package/dist/examples/serp_google_news.js +7 -10
- package/dist/examples/serp_google_news.js.map +1 -1
- package/dist/src/client.d.ts +110 -6
- package/dist/src/client.js +185 -42
- package/dist/src/client.js.map +1 -1
- package/dist/src/enums.d.ts +221 -3
- package/dist/src/enums.js +291 -2
- package/dist/src/enums.js.map +1 -1
- package/dist/src/errors.d.ts +37 -5
- package/dist/src/errors.js +61 -17
- package/dist/src/errors.js.map +1 -1
- package/dist/src/index.d.ts +7 -5
- package/dist/src/index.js +21 -5
- package/dist/src/index.js.map +1 -1
- package/dist/src/models.d.ts +137 -12
- package/dist/src/models.js +1 -0
- package/dist/src/models.js.map +1 -1
- package/dist/src/proxy.d.ts +59 -0
- package/dist/src/proxy.js +83 -11
- package/dist/src/proxy.js.map +1 -1
- package/dist/src/retry.d.ts +25 -3
- package/dist/src/retry.js +16 -0
- package/dist/src/retry.js.map +1 -1
- package/dist/src/thordata.d.ts +5 -2
- package/dist/src/thordata.js +36 -6
- package/dist/src/thordata.js.map +1 -1
- package/dist/src/utils.d.ts +18 -8
- package/dist/src/utils.js +58 -34
- package/dist/src/utils.js.map +1 -1
- package/package.json +5 -6
- package/dist/examples/proxy_demo.js +0 -69
- package/dist/examples/proxy_demo.js.map +0 -1
- package/dist/examples/proxy_ip_check.d.ts +0 -1
- package/dist/examples/proxy_ip_check.js +0 -37
- package/dist/examples/proxy_ip_check.js.map +0 -1
- /package/dist/examples/{proxy_demo.d.ts → location_api.d.ts} +0 -0
package/LICENSE
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2025 Thordata
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Thordata
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
21
|
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
# Thordata JS SDK (Node.js / TypeScript)
|
|
2
2
|
|
|
3
|
-
Official JavaScript/TypeScript SDK for
|
|
3
|
+
Official JavaScript/TypeScript SDK for Thordata APIs.
|
|
4
4
|
|
|
5
5
|
This SDK supports:
|
|
6
6
|
|
|
7
|
-
- **SERP API** (Google / Bing / Yandex / DuckDuckGo
|
|
7
|
+
- **SERP API** (Google / Bing / Yandex / DuckDuckGo)
|
|
8
8
|
- **Web Unlocker / Universal API**
|
|
9
9
|
- **Web Scraper API** (task-based scraping)
|
|
10
|
+
- **Proxy Network** (Residential / Datacenter / Mobile / ISP)
|
|
11
|
+
- **Location API** (Geo-targeting options)
|
|
10
12
|
|
|
11
13
|
It is designed to be:
|
|
12
14
|
|
|
13
|
-
- **TypeScript-first**
|
|
14
|
-
- **ESM-ready**
|
|
15
|
-
- **Offline-test friendly** (base URLs can be overridden
|
|
15
|
+
- **TypeScript-first** with full type definitions
|
|
16
|
+
- **ESM-ready** with CommonJS fallback
|
|
17
|
+
- **Offline-test friendly** (base URLs can be overridden)
|
|
16
18
|
|
|
17
19
|
---
|
|
18
20
|
|
|
@@ -22,7 +24,7 @@ It is designed to be:
|
|
|
22
24
|
npm install thordata-js-sdk
|
|
23
25
|
```
|
|
24
26
|
|
|
25
|
-
|
|
27
|
+
For local development:
|
|
26
28
|
|
|
27
29
|
```bash
|
|
28
30
|
git clone https://github.com/Thordata/thordata-js-sdk.git
|
|
@@ -38,15 +40,15 @@ npm run build
|
|
|
38
40
|
Set environment variables:
|
|
39
41
|
|
|
40
42
|
```bash
|
|
41
|
-
export
|
|
43
|
+
export THORDATA_TOKEN=your_scraper_token
|
|
42
44
|
export THORDATA_PUBLIC_TOKEN=your_public_token
|
|
43
45
|
export THORDATA_PUBLIC_KEY=your_public_key
|
|
44
46
|
```
|
|
45
47
|
|
|
46
|
-
Or create a `.env` file
|
|
48
|
+
Or create a `.env` file:
|
|
47
49
|
|
|
48
50
|
```env
|
|
49
|
-
|
|
51
|
+
THORDATA_TOKEN=your_scraper_token
|
|
50
52
|
THORDATA_PUBLIC_TOKEN=your_public_token
|
|
51
53
|
THORDATA_PUBLIC_KEY=your_public_key
|
|
52
54
|
```
|
|
@@ -55,95 +57,193 @@ THORDATA_PUBLIC_KEY=your_public_key
|
|
|
55
57
|
|
|
56
58
|
## 🚀 Quick Start
|
|
57
59
|
|
|
58
|
-
|
|
60
|
+
Create a client:
|
|
59
61
|
|
|
60
62
|
```typescript
|
|
61
63
|
import { ThordataClient } from "thordata-js-sdk";
|
|
62
64
|
|
|
63
65
|
const client = new ThordataClient({
|
|
64
|
-
scraperToken: process.env.
|
|
66
|
+
scraperToken: process.env.THORDATA_TOKEN!,
|
|
65
67
|
publicToken: process.env.THORDATA_PUBLIC_TOKEN,
|
|
66
68
|
publicKey: process.env.THORDATA_PUBLIC_KEY,
|
|
67
69
|
});
|
|
68
70
|
```
|
|
69
71
|
|
|
72
|
+
Or use the convenience wrapper:
|
|
73
|
+
|
|
74
|
+
```typescript
|
|
75
|
+
import { Thordata } from "thordata-js-sdk";
|
|
76
|
+
|
|
77
|
+
// Automatically reads from environment variables
|
|
78
|
+
const client = new Thordata();
|
|
79
|
+
```
|
|
80
|
+
|
|
70
81
|
---
|
|
71
82
|
|
|
72
83
|
## 🔍 SERP API
|
|
73
84
|
|
|
74
|
-
###
|
|
85
|
+
### Supported Engines
|
|
86
|
+
|
|
87
|
+
| Engine | Identifier | Notes |
|
|
88
|
+
| ---------- | ---------- | ---------------------- |
|
|
89
|
+
| Google | google | Base web search |
|
|
90
|
+
| Bing | bing | Base web search |
|
|
91
|
+
| Yandex | yandex | Uses text instead of q |
|
|
92
|
+
| DuckDuckGo | duckduckgo | Base web search |
|
|
93
|
+
|
|
94
|
+
### Google Specialized Engines
|
|
95
|
+
|
|
96
|
+
For Google verticals, use dedicated engines for best results:
|
|
97
|
+
|
|
98
|
+
| Engine | Identifier |
|
|
99
|
+
| --------------- | --------------- |
|
|
100
|
+
| Google News | google_news |
|
|
101
|
+
| Google Shopping | google_shopping |
|
|
102
|
+
| Google Images | google_images |
|
|
103
|
+
| Google Videos | google_videos |
|
|
104
|
+
| Google Maps | google_maps |
|
|
105
|
+
| Google Scholar | google_scholar |
|
|
106
|
+
| Google Patents | google_patents |
|
|
107
|
+
| Google Jobs | google_jobs |
|
|
108
|
+
| Google Flights | google_flights |
|
|
109
|
+
| Google Finance | google_finance |
|
|
110
|
+
| Google Product | google_product |
|
|
111
|
+
|
|
112
|
+
### Basic Search
|
|
75
113
|
|
|
76
114
|
```typescript
|
|
77
115
|
import { ThordataClient, Engine } from "thordata-js-sdk";
|
|
78
116
|
|
|
79
|
-
const client = new ThordataClient({ scraperToken: process.env.
|
|
117
|
+
const client = new ThordataClient({ scraperToken: process.env.THORDATA_TOKEN! });
|
|
80
118
|
|
|
81
119
|
const data = await client.serpSearch({
|
|
82
120
|
query: "Thordata proxy network",
|
|
83
121
|
engine: Engine.GOOGLE,
|
|
84
|
-
|
|
122
|
+
country: "us",
|
|
123
|
+
language: "en",
|
|
124
|
+
num: 10,
|
|
85
125
|
});
|
|
86
126
|
|
|
87
127
|
const organic = data?.organic ?? [];
|
|
88
128
|
console.log(`Found ${organic.length} organic results`);
|
|
89
|
-
|
|
90
|
-
for (const item of organic.slice(0, 3)) {
|
|
91
|
-
console.log("-", item.title, "->", item.link);
|
|
92
|
-
}
|
|
93
129
|
```
|
|
94
130
|
|
|
95
|
-
###
|
|
96
|
-
|
|
97
|
-
Thordata supports both:
|
|
98
|
-
|
|
99
|
-
- Dedicated engines (recommended): `google_news`, `google_shopping`
|
|
100
|
-
- Generic Google + tbm via `searchType` (alternative)
|
|
101
|
-
|
|
102
|
-
#### Google News (recommended):
|
|
131
|
+
### Google News (Recommended: Dedicated Engine)
|
|
103
132
|
|
|
104
133
|
```typescript
|
|
105
|
-
const
|
|
134
|
+
const news = await client.serpSearch({
|
|
106
135
|
query: "AI regulation",
|
|
107
|
-
engine:
|
|
136
|
+
engine: Engine.GOOGLE_NEWS,
|
|
108
137
|
country: "us",
|
|
109
138
|
language: "en",
|
|
110
139
|
num: 10,
|
|
111
|
-
so: 1, // 0=relevance, 1=date (Google News)
|
|
112
140
|
});
|
|
113
141
|
```
|
|
114
142
|
|
|
115
|
-
|
|
143
|
+
### Google Shopping
|
|
116
144
|
|
|
117
145
|
```typescript
|
|
118
|
-
const
|
|
146
|
+
const shopping = await client.serpSearch({
|
|
119
147
|
query: "iPhone 15",
|
|
120
|
-
engine:
|
|
148
|
+
engine: Engine.GOOGLE_SHOPPING,
|
|
121
149
|
country: "us",
|
|
122
150
|
language: "en",
|
|
123
151
|
num: 10,
|
|
124
|
-
min_price: 500,
|
|
125
|
-
max_price: 1500,
|
|
126
152
|
});
|
|
127
153
|
```
|
|
128
154
|
|
|
129
|
-
|
|
155
|
+
### Alternative: Using tbm Parameter
|
|
156
|
+
|
|
157
|
+
For some Google engines, you can also use the searchType (tbm) parameter:
|
|
130
158
|
|
|
131
159
|
```typescript
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
160
|
+
import { GoogleTbm } from "thordata-js-sdk";
|
|
161
|
+
|
|
162
|
+
const images = await client.serpSearch({
|
|
163
|
+
query: "cats",
|
|
164
|
+
engine: Engine.GOOGLE,
|
|
165
|
+
searchType: GoogleTbm.IMAGES, // or just "isch"
|
|
166
|
+
});
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
> **Note:** Using dedicated engines (e.g., `Engine.GOOGLE_NEWS`) is recommended over `Engine.GOOGLE + searchType` for clearer parameter contracts.
|
|
170
|
+
|
|
171
|
+
---
|
|
172
|
+
|
|
173
|
+
## 🌐 Web Unlocker / Universal API
|
|
174
|
+
|
|
175
|
+
### Supported Output Formats
|
|
176
|
+
|
|
177
|
+
| Format | Description |
|
|
178
|
+
| ------ | ---------------------- |
|
|
179
|
+
| html | HTML content (default) |
|
|
180
|
+
| png | PNG screenshot |
|
|
181
|
+
|
|
182
|
+
### Basic HTML Scraping
|
|
183
|
+
|
|
184
|
+
```typescript
|
|
185
|
+
const html = await client.universalScrape({
|
|
186
|
+
url: "https://httpbin.org/html",
|
|
187
|
+
jsRender: false,
|
|
188
|
+
outputFormat: "html",
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
console.log(String(html).slice(0, 300));
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
### JS Rendering + Wait for Selector
|
|
195
|
+
|
|
196
|
+
```typescript
|
|
197
|
+
const html = await client.universalScrape({
|
|
198
|
+
url: "https://example.com/spa",
|
|
199
|
+
jsRender: true,
|
|
200
|
+
outputFormat: "html",
|
|
201
|
+
waitFor: ".main-content", // Wait for element to appear
|
|
202
|
+
});
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
### Block Resources for Performance
|
|
206
|
+
|
|
207
|
+
```typescript
|
|
208
|
+
const html = await client.universalScrape({
|
|
209
|
+
url: "https://example.com",
|
|
210
|
+
jsRender: true,
|
|
211
|
+
blockResources: "image,media", // Block images and media
|
|
212
|
+
cleanContent: "js,css", // Remove JS and CSS from output
|
|
139
213
|
});
|
|
140
214
|
```
|
|
141
215
|
|
|
142
|
-
|
|
216
|
+
### Screenshot (PNG)
|
|
217
|
+
|
|
218
|
+
```typescript
|
|
219
|
+
import { writeFileSync } from "node:fs";
|
|
220
|
+
|
|
221
|
+
const pngBytes = await client.universalScrape({
|
|
222
|
+
url: "https://example.com",
|
|
223
|
+
jsRender: true,
|
|
224
|
+
outputFormat: "png",
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
writeFileSync("screenshot.png", pngBytes as Buffer);
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
### Custom Headers and Cookies
|
|
231
|
+
|
|
232
|
+
```typescript
|
|
233
|
+
const html = await client.universalScrape({
|
|
234
|
+
url: "https://example.com/account",
|
|
235
|
+
jsRender: true,
|
|
236
|
+
headers: [
|
|
237
|
+
{ name: "User-Agent", value: "Mozilla/5.0 (ThordataBot)" },
|
|
238
|
+
{ name: "Accept-Language", value: "en-US,en;q=0.9" },
|
|
239
|
+
],
|
|
240
|
+
cookies: [{ name: "sessionid", value: "abc123" }],
|
|
241
|
+
});
|
|
242
|
+
```
|
|
143
243
|
|
|
144
244
|
---
|
|
145
245
|
|
|
146
|
-
##
|
|
246
|
+
## 🌍 Proxy Network
|
|
147
247
|
|
|
148
248
|
Each proxy product requires separate credentials from Thordata Dashboard.
|
|
149
249
|
|
|
@@ -162,7 +262,7 @@ THORDATA_DATACENTER_PASSWORD=your_datacenter_password
|
|
|
162
262
|
THORDATA_MOBILE_USERNAME=your_mobile_username
|
|
163
263
|
THORDATA_MOBILE_PASSWORD=your_mobile_password
|
|
164
264
|
|
|
165
|
-
# Static ISP Proxy (port 6666
|
|
265
|
+
# Static ISP Proxy (port 6666)
|
|
166
266
|
THORDATA_ISP_HOST=your_static_ip_address
|
|
167
267
|
THORDATA_ISP_USERNAME=your_isp_username
|
|
168
268
|
THORDATA_ISP_PASSWORD=your_isp_password
|
|
@@ -186,7 +286,6 @@ const stickyProxy = Thordata.Proxy.residentialFromEnv()
|
|
|
186
286
|
.city("tokyo")
|
|
187
287
|
.session("my_session")
|
|
188
288
|
.sticky(30);
|
|
189
|
-
const result2 = await client.request("http://httpbin.org/ip", { proxy: stickyProxy });
|
|
190
289
|
```
|
|
191
290
|
|
|
192
291
|
### Datacenter Proxy
|
|
@@ -194,7 +293,6 @@ const result2 = await client.request("http://httpbin.org/ip", { proxy: stickyPro
|
|
|
194
293
|
```typescript
|
|
195
294
|
const proxy = Thordata.Proxy.datacenterFromEnv();
|
|
196
295
|
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
197
|
-
console.log("Datacenter IP:", result.origin);
|
|
198
296
|
```
|
|
199
297
|
|
|
200
298
|
### Mobile Proxy
|
|
@@ -202,7 +300,6 @@ console.log("Datacenter IP:", result.origin);
|
|
|
202
300
|
```typescript
|
|
203
301
|
const proxy = Thordata.Proxy.mobileFromEnv().country("gb");
|
|
204
302
|
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
205
|
-
console.log("UK Mobile IP:", result.origin);
|
|
206
303
|
```
|
|
207
304
|
|
|
208
305
|
### Static ISP Proxy
|
|
@@ -210,58 +307,49 @@ console.log("UK Mobile IP:", result.origin);
|
|
|
210
307
|
```typescript
|
|
211
308
|
const proxy = Thordata.Proxy.ispFromEnv();
|
|
212
309
|
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
213
|
-
console.log("Static ISP IP:", result.origin);
|
|
214
|
-
// Returns your purchased static IP address
|
|
215
|
-
```
|
|
216
|
-
|
|
217
|
-
### Proxy Examples
|
|
218
|
-
|
|
219
|
-
```bash
|
|
220
|
-
node dist/examples/proxy_residential.js
|
|
221
|
-
node dist/examples/proxy_datacenter.js
|
|
222
|
-
node dist/examples/proxy_mobile.js
|
|
223
|
-
node dist/examples/proxy_isp.js
|
|
224
310
|
```
|
|
225
311
|
|
|
226
312
|
---
|
|
227
313
|
|
|
228
|
-
##
|
|
314
|
+
## 📍 Location API
|
|
229
315
|
|
|
230
|
-
|
|
316
|
+
Query available geo-targeting options. Requires `THORDATA_PUBLIC_TOKEN` and `THORDATA_PUBLIC_KEY`.
|
|
317
|
+
|
|
318
|
+
### List Countries
|
|
231
319
|
|
|
232
320
|
```typescript
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
jsRender: false,
|
|
236
|
-
outputFormat: "html",
|
|
237
|
-
});
|
|
321
|
+
// Using string parameter (recommended)
|
|
322
|
+
const countries = await client.listCountries("residential");
|
|
238
323
|
|
|
239
|
-
|
|
324
|
+
// Or numeric parameter (1 = residential, 2 = unlimited)
|
|
325
|
+
const countries2 = await client.listCountries(1);
|
|
326
|
+
|
|
327
|
+
console.log(`Found ${countries.length} countries`);
|
|
328
|
+
// [{ country_code: "US", country_name: "United States" }, ...]
|
|
240
329
|
```
|
|
241
330
|
|
|
242
|
-
###
|
|
331
|
+
### List States
|
|
243
332
|
|
|
244
333
|
```typescript
|
|
245
|
-
const
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
outputFormat: "html",
|
|
249
|
-
waitFor: ".main-content",
|
|
250
|
-
});
|
|
334
|
+
const states = await client.listStates("US", "residential");
|
|
335
|
+
console.log(`Found ${states.length} states`);
|
|
336
|
+
// [{ state_code: "california", state_name: "California" }, ...]
|
|
251
337
|
```
|
|
252
338
|
|
|
253
|
-
###
|
|
339
|
+
### List Cities
|
|
254
340
|
|
|
255
341
|
```typescript
|
|
256
|
-
|
|
342
|
+
const cities = await client.listCities("US", "california", "residential");
|
|
343
|
+
console.log(`Found ${cities.length} cities`);
|
|
344
|
+
// [{ city_name: "Los Angeles" }, { city_name: "San Francisco" }, ...]
|
|
345
|
+
```
|
|
257
346
|
|
|
258
|
-
|
|
259
|
-
url: "https://example.com",
|
|
260
|
-
jsRender: true,
|
|
261
|
-
outputFormat: "png",
|
|
262
|
-
});
|
|
347
|
+
### List ASNs
|
|
263
348
|
|
|
264
|
-
|
|
349
|
+
```typescript
|
|
350
|
+
const asns = await client.listAsns("US", "residential");
|
|
351
|
+
console.log(`Found ${asns.length} ASNs`);
|
|
352
|
+
// [{ asn_code: "AS7922", asn_name: "Comcast" }, ...]
|
|
265
353
|
```
|
|
266
354
|
|
|
267
355
|
---
|
|
@@ -272,11 +360,12 @@ Requires `THORDATA_PUBLIC_TOKEN` and `THORDATA_PUBLIC_KEY`.
|
|
|
272
360
|
|
|
273
361
|
```typescript
|
|
274
362
|
const client = new ThordataClient({
|
|
275
|
-
scraperToken: process.env.
|
|
363
|
+
scraperToken: process.env.THORDATA_TOKEN!,
|
|
276
364
|
publicToken: process.env.THORDATA_PUBLIC_TOKEN,
|
|
277
365
|
publicKey: process.env.THORDATA_PUBLIC_KEY,
|
|
278
366
|
});
|
|
279
367
|
|
|
368
|
+
// Create task
|
|
280
369
|
const taskId = await client.createScraperTask({
|
|
281
370
|
fileName: "demo_task",
|
|
282
371
|
spiderId: "example-spider-id",
|
|
@@ -286,6 +375,7 @@ const taskId = await client.createScraperTask({
|
|
|
286
375
|
|
|
287
376
|
console.log("Task created:", taskId);
|
|
288
377
|
|
|
378
|
+
// Wait for completion
|
|
289
379
|
const status = await client.waitForTask(taskId, {
|
|
290
380
|
pollIntervalMs: 5000,
|
|
291
381
|
maxWaitMs: 60000,
|
|
@@ -293,6 +383,7 @@ const status = await client.waitForTask(taskId, {
|
|
|
293
383
|
|
|
294
384
|
console.log("Final status:", status);
|
|
295
385
|
|
|
386
|
+
// Download results
|
|
296
387
|
if (["ready", "success", "finished"].includes(status.toLowerCase())) {
|
|
297
388
|
const downloadUrl = await client.getTaskResult(taskId, "json");
|
|
298
389
|
console.log("Download URL:", downloadUrl);
|
|
@@ -301,20 +392,22 @@ if (["ready", "success", "finished"].includes(status.toLowerCase())) {
|
|
|
301
392
|
|
|
302
393
|
---
|
|
303
394
|
|
|
304
|
-
## 🔧
|
|
395
|
+
## 🔧 Error Handling
|
|
396
|
+
|
|
397
|
+
The SDK throws typed errors based on API response codes:
|
|
305
398
|
|
|
306
|
-
|
|
399
|
+
| Code | Meaning | Error Class |
|
|
400
|
+
| ------- | ------------- | ------------------------- |
|
|
401
|
+
| 200 | Success | - |
|
|
402
|
+
| 300 | Not collected | ThordataNotCollectedError |
|
|
403
|
+
| 400 | Bad request | ThordataValidationError |
|
|
404
|
+
| 401/403 | Auth error | ThordataAuthError |
|
|
405
|
+
| 402/429 | Rate limit | ThordataRateLimitError |
|
|
406
|
+
| 5xx | Server error | ThordataServerError |
|
|
307
407
|
|
|
308
|
-
|
|
309
|
-
| ------- | --------------------- | ---------------------------------------------- |
|
|
310
|
-
| 200 | Success | - |
|
|
311
|
-
| 300 | Not collected | `ThordataNotCollectedError` |
|
|
312
|
-
| 400 | Bad request | `ThordataValidationError` |
|
|
313
|
-
| 401/403 | Auth/Forbidden | `ThordataAuthError` |
|
|
314
|
-
| 402/429 | Quota/Rate limit | `ThordataRateLimitError` |
|
|
315
|
-
| 5xx | Server/timeout issues | `ThordataServerError` / `ThordataTimeoutError` |
|
|
408
|
+
> **Billing Note:** Only requests with code == 200 are billed.
|
|
316
409
|
|
|
317
|
-
### Example
|
|
410
|
+
### Example
|
|
318
411
|
|
|
319
412
|
```typescript
|
|
320
413
|
import {
|
|
@@ -344,18 +437,37 @@ try {
|
|
|
344
437
|
|
|
345
438
|
---
|
|
346
439
|
|
|
347
|
-
##
|
|
440
|
+
## ⚙️ Advanced Configuration
|
|
348
441
|
|
|
349
|
-
|
|
442
|
+
### SSL Verification
|
|
443
|
+
|
|
444
|
+
```typescript
|
|
445
|
+
const client = new ThordataClient({
|
|
446
|
+
scraperToken: process.env.THORDATA_TOKEN!,
|
|
447
|
+
verifySsl: false, // Only for testing with self-signed certs
|
|
448
|
+
});
|
|
449
|
+
```
|
|
450
|
+
|
|
451
|
+
### Retry Configuration
|
|
452
|
+
|
|
453
|
+
```typescript
|
|
454
|
+
const client = new ThordataClient({
|
|
455
|
+
scraperToken: process.env.THORDATA_TOKEN!,
|
|
456
|
+
maxRetries: 3, // Retry up to 3 times on transient failures
|
|
457
|
+
timeoutMs: 60000, // 60 second timeout
|
|
458
|
+
});
|
|
459
|
+
```
|
|
460
|
+
|
|
461
|
+
### Base URL Overrides
|
|
462
|
+
|
|
463
|
+
For testing or custom routing:
|
|
350
464
|
|
|
351
465
|
```bash
|
|
352
466
|
export THORDATA_SCRAPERAPI_BASE_URL=http://127.0.0.1:12345
|
|
353
467
|
export THORDATA_UNIVERSALAPI_BASE_URL=http://127.0.0.1:12345
|
|
354
|
-
export THORDATA_WEB_SCRAPER_API_BASE_URL=http://127.0.0.1:12345
|
|
355
|
-
export THORDATA_LOCATIONS_BASE_URL=http://127.0.0.1:12345
|
|
356
468
|
```
|
|
357
469
|
|
|
358
|
-
Or via
|
|
470
|
+
Or via config:
|
|
359
471
|
|
|
360
472
|
```typescript
|
|
361
473
|
const client = new ThordataClient({
|
|
@@ -372,9 +484,11 @@ const client = new ThordataClient({
|
|
|
372
484
|
npm install
|
|
373
485
|
npm run build
|
|
374
486
|
npm test
|
|
487
|
+
npm run lint
|
|
488
|
+
npm run format
|
|
375
489
|
```
|
|
376
490
|
|
|
377
|
-
### Run
|
|
491
|
+
### Run Examples
|
|
378
492
|
|
|
379
493
|
```bash
|
|
380
494
|
node dist/examples/basic_serp.js
|
|
@@ -388,37 +502,35 @@ node dist/examples/basic_universal.js
|
|
|
388
502
|
```
|
|
389
503
|
thordata-js-sdk/
|
|
390
504
|
├── src/
|
|
391
|
-
│ ├── index.ts
|
|
392
|
-
│ ├── client.ts
|
|
393
|
-
│ ├──
|
|
394
|
-
│ ├──
|
|
395
|
-
│ ├──
|
|
396
|
-
│ ├──
|
|
397
|
-
│ ├──
|
|
398
|
-
│
|
|
505
|
+
│ ├── index.ts # Main exports
|
|
506
|
+
│ ├── client.ts # ThordataClient class
|
|
507
|
+
│ ├── thordata.ts # Convenience wrapper
|
|
508
|
+
│ ├── proxy.ts # Proxy configuration
|
|
509
|
+
│ ├── models.ts # TypeScript interfaces
|
|
510
|
+
│ ├── enums.ts # Enumerations
|
|
511
|
+
│ ├── errors.ts # Error classes
|
|
512
|
+
│ ├── utils.ts # Helper functions
|
|
513
|
+
│ └── endpoints.ts # API endpoints
|
|
399
514
|
├── examples/
|
|
400
|
-
│ ├── basic_serp.ts
|
|
401
|
-
│ ├── basic_universal.ts
|
|
402
|
-
│ ├── basic_scraper_task.ts
|
|
403
|
-
│ └── serp_google_news.ts
|
|
404
515
|
├── tests/
|
|
405
|
-
│ ├── serp.offline.test.ts
|
|
406
|
-
│ ├── mockServer.ts
|
|
407
|
-
│ └── examples.e2e.test.ts
|
|
408
|
-
├── .github/workflows/ci.yml
|
|
409
516
|
├── package.json
|
|
410
|
-
├── tsconfig.json
|
|
411
|
-
├── tsconfig.build.json
|
|
412
517
|
└── README.md
|
|
413
518
|
```
|
|
414
519
|
|
|
415
520
|
---
|
|
416
521
|
|
|
417
|
-
##
|
|
522
|
+
## 📚 API Reference
|
|
523
|
+
|
|
524
|
+
For detailed API documentation, visit:
|
|
418
525
|
|
|
419
|
-
-
|
|
420
|
-
-
|
|
421
|
-
-
|
|
422
|
-
-
|
|
526
|
+
- [Google Search Parameters](#)
|
|
527
|
+
- [Google News Parameters](#)
|
|
528
|
+
- [Google Shopping Parameters](#)
|
|
529
|
+
- [Yandex Parameters](#)
|
|
530
|
+
- [Universal API Parameters](#)
|
|
423
531
|
|
|
424
532
|
---
|
|
533
|
+
|
|
534
|
+
## 📄 License
|
|
535
|
+
|
|
536
|
+
MIT License - see LICENSE for details.
|
|
@@ -14,10 +14,10 @@ async function main() {
|
|
|
14
14
|
engine: Engine.GOOGLE,
|
|
15
15
|
num: 5,
|
|
16
16
|
});
|
|
17
|
-
//
|
|
17
|
+
// Debug: show actual response structure
|
|
18
18
|
console.dir(results, { depth: 4 });
|
|
19
|
-
//
|
|
20
|
-
const organic = results?.organic ?? results?.organic_results ?? [];
|
|
19
|
+
// Type-safe access with fallback
|
|
20
|
+
const organic = (results?.organic ?? results?.organic_results ?? []);
|
|
21
21
|
console.log(`Found ${organic.length} organic results`);
|
|
22
22
|
for (const item of organic.slice(0, 3)) {
|
|
23
23
|
console.log("-", item.title, "->", item.link);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"basic_serp.js","sourceRoot":"","sources":["../../examples/basic_serp.ts"],"names":[],"mappings":"AAAA,yBAAyB;AAEzB,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzD,KAAK,UAAU,IAAI;IACjB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACjD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC;QACtC,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,CAAC;KACP,CAAC,CAAC;IAEH,
|
|
1
|
+
{"version":3,"file":"basic_serp.js","sourceRoot":"","sources":["../../examples/basic_serp.ts"],"names":[],"mappings":"AAAA,yBAAyB;AAEzB,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzD,KAAK,UAAU,IAAI;IACjB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACjD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC;QACtC,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,CAAC;KACP,CAAC,CAAC;IAEH,wCAAwC;IACxC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEnC,iCAAiC;IACjC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,eAAe,IAAI,EAAE,CAGjE,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;IACvD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC/B,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// examples/location_api.ts
|
|
2
|
+
import "dotenv/config";
|
|
3
|
+
import { Thordata } from "../src/thordata.js";
|
|
4
|
+
async function main() {
|
|
5
|
+
if (!process.env.THORDATA_PUBLIC_TOKEN || !process.env.THORDATA_PUBLIC_KEY) {
|
|
6
|
+
console.log("Location API Demo - Skipped");
|
|
7
|
+
console.log("Set THORDATA_PUBLIC_TOKEN and THORDATA_PUBLIC_KEY in .env");
|
|
8
|
+
return;
|
|
9
|
+
}
|
|
10
|
+
const client = new Thordata();
|
|
11
|
+
console.log("Location API Demo\n");
|
|
12
|
+
// List countries (now using string parameter)
|
|
13
|
+
console.log("1. Listing countries...");
|
|
14
|
+
const countries = await client.listCountries("residential");
|
|
15
|
+
console.log(` Found ${countries.length} countries`);
|
|
16
|
+
console.log(` First 5: ${countries
|
|
17
|
+
.slice(0, 5)
|
|
18
|
+
.map((c) => c.country_code)
|
|
19
|
+
.join(", ")}`);
|
|
20
|
+
// List states for US
|
|
21
|
+
console.log("\n2. Listing US states...");
|
|
22
|
+
const states = await client.listStates("US", "residential");
|
|
23
|
+
console.log(` Found ${states.length} states`);
|
|
24
|
+
console.log(` First 5: ${states
|
|
25
|
+
.slice(0, 5)
|
|
26
|
+
.map((s) => s.state_name)
|
|
27
|
+
.join(", ")}`);
|
|
28
|
+
// List cities for California
|
|
29
|
+
console.log("\n3. Listing California cities...");
|
|
30
|
+
const cities = await client.listCities("US", "california", "residential");
|
|
31
|
+
console.log(` Found ${cities.length} cities`);
|
|
32
|
+
console.log(` First 5: ${cities
|
|
33
|
+
.slice(0, 5)
|
|
34
|
+
.map((c) => c.city_name)
|
|
35
|
+
.join(", ")}`);
|
|
36
|
+
// List ASNs for US
|
|
37
|
+
console.log("\n4. Listing US ASNs...");
|
|
38
|
+
const asns = await client.listAsns("US", "residential");
|
|
39
|
+
console.log(` Found ${asns.length} ASNs`);
|
|
40
|
+
console.log(` First 3: ${asns
|
|
41
|
+
.slice(0, 3)
|
|
42
|
+
.map((a) => `${a.asn_code}: ${a.asn_name}`)
|
|
43
|
+
.join(", ")}`);
|
|
44
|
+
console.log("\nLocation API demo completed.");
|
|
45
|
+
}
|
|
46
|
+
main().catch(console.error);
|
|
47
|
+
//# sourceMappingURL=location_api.js.map
|