thordata-js-sdk 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -459
- package/dist/examples/basic_scraper_task.js +49 -24
- package/dist/examples/basic_scraper_task.js.map +1 -1
- package/dist/examples/internal/example.d.ts +5 -0
- package/dist/examples/internal/example.js +25 -0
- package/dist/examples/internal/example.js.map +1 -0
- package/dist/examples/locations_basic.d.ts +1 -0
- package/dist/examples/locations_basic.js +21 -0
- package/dist/examples/locations_basic.js.map +1 -0
- package/dist/examples/serp_basic.d.ts +1 -0
- package/dist/examples/serp_basic.js +25 -0
- package/dist/examples/serp_basic.js.map +1 -0
- package/dist/examples/universal_basic.d.ts +1 -0
- package/dist/examples/universal_basic.js +29 -0
- package/dist/examples/universal_basic.js.map +1 -0
- package/dist/examples/verify_new_features.js +103 -0
- package/dist/examples/verify_new_features.js.map +1 -0
- package/dist/src/auth.d.ts +22 -0
- package/dist/src/auth.js +39 -0
- package/dist/src/auth.js.map +1 -0
- package/dist/src/client.d.ts +22 -5
- package/dist/src/client.js +177 -4
- package/dist/src/client.js.map +1 -1
- package/dist/src/endpoints.js +10 -10
- package/dist/src/endpoints.js.map +1 -1
- package/dist/src/enums.d.ts +3 -5
- package/dist/src/enums.js +1 -3
- package/dist/src/enums.js.map +1 -1
- package/dist/src/models.d.ts +59 -0
- package/dist/src/proxy.d.ts +1 -0
- package/dist/src/proxy.js +23 -1
- package/dist/src/proxy.js.map +1 -1
- package/dist/src/utils.d.ts +1 -6
- package/dist/src/utils.js +36 -9
- package/dist/src/utils.js.map +1 -1
- package/package.json +13 -2
- package/dist/examples/basic_serp.js +0 -29
- package/dist/examples/basic_serp.js.map +0 -1
- package/dist/examples/basic_universal.d.ts +0 -1
- package/dist/examples/basic_universal.js +0 -23
- package/dist/examples/basic_universal.js.map +0 -1
- /package/dist/examples/{basic_serp.d.ts → verify_new_features.d.ts} +0 -0
package/README.md
CHANGED
|
@@ -2,19 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
Official JavaScript/TypeScript SDK for Thordata APIs.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
[](https://www.npmjs.com/package/thordata-js-sdk)
|
|
6
|
+
[](LICENSE)
|
|
6
7
|
|
|
7
|
-
|
|
8
|
-
- **Web Unlocker / Universal API**
|
|
9
|
-
- **Web Scraper API** (task-based scraping)
|
|
10
|
-
- **Proxy Network** (Residential / Datacenter / Mobile / ISP)
|
|
11
|
-
- **Location API** (Geo-targeting options)
|
|
12
|
-
|
|
13
|
-
It is designed to be:
|
|
8
|
+
Supports:
|
|
14
9
|
|
|
15
|
-
- **
|
|
16
|
-
- **
|
|
17
|
-
- **
|
|
10
|
+
- **SERP API** (Google / Bing / Yandex)
|
|
11
|
+
- **Web Unlocker** (Universal API)
|
|
12
|
+
- **Web Scraper API** (Text & Video Tasks)
|
|
13
|
+
- **Proxy Network** (Residential / Datacenter / Mobile / ISP)
|
|
14
|
+
- **Account Management** (Usage, Users, Whitelist)
|
|
18
15
|
|
|
19
16
|
---
|
|
20
17
|
|
|
@@ -24,513 +21,110 @@ It is designed to be:
|
|
|
24
21
|
npm install thordata-js-sdk
|
|
25
22
|
```
|
|
26
23
|
|
|
27
|
-
For local development:
|
|
28
|
-
|
|
29
|
-
```bash
|
|
30
|
-
git clone https://github.com/Thordata/thordata-js-sdk.git
|
|
31
|
-
cd thordata-js-sdk
|
|
32
|
-
npm install
|
|
33
|
-
npm run build
|
|
34
|
-
```
|
|
35
|
-
|
|
36
24
|
---
|
|
37
25
|
|
|
38
26
|
## 🔐 Configuration
|
|
39
27
|
|
|
40
|
-
Set environment variables:
|
|
41
|
-
|
|
42
28
|
```bash
|
|
43
|
-
export
|
|
29
|
+
export THORDATA_SCRAPER_TOKEN=your_token
|
|
44
30
|
export THORDATA_PUBLIC_TOKEN=your_public_token
|
|
45
31
|
export THORDATA_PUBLIC_KEY=your_public_key
|
|
46
32
|
```
|
|
47
33
|
|
|
48
|
-
Or create a `.env` file:
|
|
49
|
-
|
|
50
|
-
```env
|
|
51
|
-
THORDATA_TOKEN=your_scraper_token
|
|
52
|
-
THORDATA_PUBLIC_TOKEN=your_public_token
|
|
53
|
-
THORDATA_PUBLIC_KEY=your_public_key
|
|
54
|
-
```
|
|
55
|
-
|
|
56
34
|
---
|
|
57
35
|
|
|
58
36
|
## 🚀 Quick Start
|
|
59
37
|
|
|
60
|
-
Create a client:
|
|
61
|
-
|
|
62
|
-
```typescript
|
|
63
|
-
import { ThordataClient } from "thordata-js-sdk";
|
|
64
|
-
|
|
65
|
-
const client = new ThordataClient({
|
|
66
|
-
scraperToken: process.env.THORDATA_TOKEN!,
|
|
67
|
-
publicToken: process.env.THORDATA_PUBLIC_TOKEN,
|
|
68
|
-
publicKey: process.env.THORDATA_PUBLIC_KEY,
|
|
69
|
-
});
|
|
70
|
-
```
|
|
71
|
-
|
|
72
|
-
Or use the convenience wrapper:
|
|
73
|
-
|
|
74
38
|
```typescript
|
|
75
39
|
import { Thordata } from "thordata-js-sdk";
|
|
76
40
|
|
|
77
|
-
//
|
|
78
|
-
const client = new Thordata();
|
|
79
|
-
```
|
|
80
|
-
|
|
81
|
-
---
|
|
82
|
-
|
|
83
|
-
## 🔍 SERP API
|
|
84
|
-
|
|
85
|
-
### Supported Engines
|
|
86
|
-
|
|
87
|
-
| Engine | Identifier | Notes |
|
|
88
|
-
| ---------- | ---------- | ---------------------- |
|
|
89
|
-
| Google | google | Base web search |
|
|
90
|
-
| Bing | bing | Base web search |
|
|
91
|
-
| Yandex | yandex | Uses text instead of q |
|
|
92
|
-
| DuckDuckGo | duckduckgo | Base web search |
|
|
41
|
+
const client = new Thordata(); // Reads from env vars
|
|
93
42
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
| Engine | Identifier |
|
|
99
|
-
| --------------- | --------------- |
|
|
100
|
-
| Google News | google_news |
|
|
101
|
-
| Google Shopping | google_shopping |
|
|
102
|
-
| Google Images | google_images |
|
|
103
|
-
| Google Videos | google_videos |
|
|
104
|
-
| Google Maps | google_maps |
|
|
105
|
-
| Google Scholar | google_scholar |
|
|
106
|
-
| Google Patents | google_patents |
|
|
107
|
-
| Google Jobs | google_jobs |
|
|
108
|
-
| Google Flights | google_flights |
|
|
109
|
-
| Google Finance | google_finance |
|
|
110
|
-
| Google Product | google_product |
|
|
111
|
-
|
|
112
|
-
### Basic Search
|
|
113
|
-
|
|
114
|
-
```typescript
|
|
115
|
-
import { ThordataClient, Engine } from "thordata-js-sdk";
|
|
116
|
-
|
|
117
|
-
const client = new ThordataClient({ scraperToken: process.env.THORDATA_TOKEN! });
|
|
118
|
-
|
|
119
|
-
const data = await client.serpSearch({
|
|
120
|
-
query: "Thordata proxy network",
|
|
121
|
-
engine: Engine.GOOGLE,
|
|
122
|
-
country: "us",
|
|
123
|
-
language: "en",
|
|
124
|
-
num: 10,
|
|
125
|
-
});
|
|
126
|
-
|
|
127
|
-
const organic = data?.organic ?? [];
|
|
128
|
-
console.log(`Found ${organic.length} organic results`);
|
|
129
|
-
```
|
|
130
|
-
|
|
131
|
-
### Google News (Recommended: Dedicated Engine)
|
|
132
|
-
|
|
133
|
-
```typescript
|
|
134
|
-
const news = await client.serpSearch({
|
|
135
|
-
query: "AI regulation",
|
|
136
|
-
engine: Engine.GOOGLE_NEWS,
|
|
137
|
-
country: "us",
|
|
138
|
-
language: "en",
|
|
139
|
-
num: 10,
|
|
140
|
-
});
|
|
141
|
-
```
|
|
142
|
-
|
|
143
|
-
### Google Shopping
|
|
144
|
-
|
|
145
|
-
```typescript
|
|
146
|
-
const shopping = await client.serpSearch({
|
|
147
|
-
query: "iPhone 15",
|
|
148
|
-
engine: Engine.GOOGLE_SHOPPING,
|
|
43
|
+
// SERP Search
|
|
44
|
+
const results = await client.serpSearch({
|
|
45
|
+
query: "Thordata SDK",
|
|
46
|
+
engine: "google",
|
|
149
47
|
country: "us",
|
|
150
|
-
language: "en",
|
|
151
|
-
num: 10,
|
|
152
48
|
});
|
|
49
|
+
console.log(results.organic?.[0]?.link);
|
|
153
50
|
```
|
|
154
51
|
|
|
155
|
-
### Alternative: Using tbm Parameter
|
|
156
|
-
|
|
157
|
-
For some Google engines, you can also use the searchType (tbm) parameter:
|
|
158
|
-
|
|
159
|
-
```typescript
|
|
160
|
-
import { GoogleTbm } from "thordata-js-sdk";
|
|
161
|
-
|
|
162
|
-
const images = await client.serpSearch({
|
|
163
|
-
query: "cats",
|
|
164
|
-
engine: Engine.GOOGLE,
|
|
165
|
-
searchType: GoogleTbm.IMAGES, // or just "isch"
|
|
166
|
-
});
|
|
167
|
-
```
|
|
168
|
-
|
|
169
|
-
> **Note:** Using dedicated engines (e.g., `Engine.GOOGLE_NEWS`) is recommended over `Engine.GOOGLE + searchType` for clearer parameter contracts.
|
|
170
|
-
|
|
171
52
|
---
|
|
172
53
|
|
|
173
|
-
##
|
|
174
|
-
|
|
175
|
-
### Supported Output Formats
|
|
176
|
-
|
|
177
|
-
| Format | Description |
|
|
178
|
-
| ------ | ---------------------- |
|
|
179
|
-
| html | HTML content (default) |
|
|
180
|
-
| png | PNG screenshot |
|
|
181
|
-
|
|
182
|
-
### Basic HTML Scraping
|
|
183
|
-
|
|
184
|
-
```typescript
|
|
185
|
-
const html = await client.universalScrape({
|
|
186
|
-
url: "https://httpbin.org/html",
|
|
187
|
-
jsRender: false,
|
|
188
|
-
outputFormat: "html",
|
|
189
|
-
});
|
|
54
|
+
## 📖 Features
|
|
190
55
|
|
|
191
|
-
|
|
192
|
-
```
|
|
193
|
-
|
|
194
|
-
### JS Rendering + Wait for Selector
|
|
56
|
+
### SERP API
|
|
195
57
|
|
|
196
58
|
```typescript
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
waitFor: ".main-content", // Wait for element to appear
|
|
59
|
+
const news = await client.serpSearch({
|
|
60
|
+
query: "AI News",
|
|
61
|
+
engine: "google_news",
|
|
62
|
+
num: 10,
|
|
202
63
|
});
|
|
203
64
|
```
|
|
204
65
|
|
|
205
|
-
###
|
|
66
|
+
### Web Unlocker (Universal)
|
|
206
67
|
|
|
207
68
|
```typescript
|
|
208
69
|
const html = await client.universalScrape({
|
|
209
70
|
url: "https://example.com",
|
|
210
71
|
jsRender: true,
|
|
211
|
-
|
|
212
|
-
cleanContent: "js,css", // Remove JS and CSS from output
|
|
72
|
+
waitFor: ".content",
|
|
213
73
|
});
|
|
214
74
|
```
|
|
215
75
|
|
|
216
|
-
###
|
|
217
|
-
|
|
218
|
-
```typescript
|
|
219
|
-
import { writeFileSync } from "node:fs";
|
|
220
|
-
|
|
221
|
-
const pngBytes = await client.universalScrape({
|
|
222
|
-
url: "https://example.com",
|
|
223
|
-
jsRender: true,
|
|
224
|
-
outputFormat: "png",
|
|
225
|
-
});
|
|
226
|
-
|
|
227
|
-
writeFileSync("screenshot.png", pngBytes as Buffer);
|
|
228
|
-
```
|
|
229
|
-
|
|
230
|
-
### Custom Headers and Cookies
|
|
231
|
-
|
|
232
|
-
```typescript
|
|
233
|
-
const html = await client.universalScrape({
|
|
234
|
-
url: "https://example.com/account",
|
|
235
|
-
jsRender: true,
|
|
236
|
-
headers: [
|
|
237
|
-
{ name: "User-Agent", value: "Mozilla/5.0 (ThordataBot)" },
|
|
238
|
-
{ name: "Accept-Language", value: "en-US,en;q=0.9" },
|
|
239
|
-
],
|
|
240
|
-
cookies: [{ name: "sessionid", value: "abc123" }],
|
|
241
|
-
});
|
|
242
|
-
```
|
|
243
|
-
|
|
244
|
-
---
|
|
245
|
-
|
|
246
|
-
## 🌍 Proxy Network
|
|
247
|
-
|
|
248
|
-
Each proxy product requires separate credentials from Thordata Dashboard.
|
|
249
|
-
|
|
250
|
-
### Environment Variables
|
|
251
|
-
|
|
252
|
-
```env
|
|
253
|
-
# Residential Proxy (port 9999)
|
|
254
|
-
THORDATA_RESIDENTIAL_USERNAME=your_residential_username
|
|
255
|
-
THORDATA_RESIDENTIAL_PASSWORD=your_residential_password
|
|
256
|
-
|
|
257
|
-
# Datacenter Proxy (port 7777)
|
|
258
|
-
THORDATA_DATACENTER_USERNAME=your_datacenter_username
|
|
259
|
-
THORDATA_DATACENTER_PASSWORD=your_datacenter_password
|
|
260
|
-
|
|
261
|
-
# Mobile Proxy (port 5555)
|
|
262
|
-
THORDATA_MOBILE_USERNAME=your_mobile_username
|
|
263
|
-
THORDATA_MOBILE_PASSWORD=your_mobile_password
|
|
264
|
-
|
|
265
|
-
# Static ISP Proxy (port 6666)
|
|
266
|
-
THORDATA_ISP_HOST=your_static_ip_address
|
|
267
|
-
THORDATA_ISP_USERNAME=your_isp_username
|
|
268
|
-
THORDATA_ISP_PASSWORD=your_isp_password
|
|
269
|
-
```
|
|
270
|
-
|
|
271
|
-
### Residential Proxy
|
|
76
|
+
### Web Scraper API (Async)
|
|
272
77
|
|
|
273
78
|
```typescript
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
const client = new Thordata();
|
|
277
|
-
|
|
278
|
-
// Basic US residential
|
|
279
|
-
const proxy = Thordata.Proxy.residentialFromEnv().country("us");
|
|
280
|
-
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
281
|
-
console.log("IP:", result.origin);
|
|
282
|
-
|
|
283
|
-
// Sticky session (same IP for 30 minutes)
|
|
284
|
-
const stickyProxy = Thordata.Proxy.residentialFromEnv()
|
|
285
|
-
.country("jp")
|
|
286
|
-
.city("tokyo")
|
|
287
|
-
.session("my_session")
|
|
288
|
-
.sticky(30);
|
|
289
|
-
```
|
|
290
|
-
|
|
291
|
-
### Datacenter Proxy
|
|
292
|
-
|
|
293
|
-
```typescript
|
|
294
|
-
const proxy = Thordata.Proxy.datacenterFromEnv();
|
|
295
|
-
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
296
|
-
```
|
|
297
|
-
|
|
298
|
-
### Mobile Proxy
|
|
299
|
-
|
|
300
|
-
```typescript
|
|
301
|
-
const proxy = Thordata.Proxy.mobileFromEnv().country("gb");
|
|
302
|
-
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
303
|
-
```
|
|
304
|
-
|
|
305
|
-
### Static ISP Proxy
|
|
306
|
-
|
|
307
|
-
```typescript
|
|
308
|
-
const proxy = Thordata.Proxy.ispFromEnv();
|
|
309
|
-
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
310
|
-
```
|
|
311
|
-
|
|
312
|
-
---
|
|
313
|
-
|
|
314
|
-
## 📍 Location API
|
|
315
|
-
|
|
316
|
-
Query available geo-targeting options. Requires `THORDATA_PUBLIC_TOKEN` and `THORDATA_PUBLIC_KEY`.
|
|
317
|
-
|
|
318
|
-
### List Countries
|
|
319
|
-
|
|
320
|
-
```typescript
|
|
321
|
-
// Using string parameter (recommended)
|
|
322
|
-
const countries = await client.listCountries("residential");
|
|
323
|
-
|
|
324
|
-
// Or numeric parameter (1 = residential, 2 = unlimited)
|
|
325
|
-
const countries2 = await client.listCountries(1);
|
|
326
|
-
|
|
327
|
-
console.log(`Found ${countries.length} countries`);
|
|
328
|
-
// [{ country_code: "US", country_name: "United States" }, ...]
|
|
329
|
-
```
|
|
330
|
-
|
|
331
|
-
### List States
|
|
332
|
-
|
|
333
|
-
```typescript
|
|
334
|
-
const states = await client.listStates("US", "residential");
|
|
335
|
-
console.log(`Found ${states.length} states`);
|
|
336
|
-
// [{ state_code: "california", state_name: "California" }, ...]
|
|
337
|
-
```
|
|
338
|
-
|
|
339
|
-
### List Cities
|
|
340
|
-
|
|
341
|
-
```typescript
|
|
342
|
-
const cities = await client.listCities("US", "california", "residential");
|
|
343
|
-
console.log(`Found ${cities.length} cities`);
|
|
344
|
-
// [{ city_name: "Los Angeles" }, { city_name: "San Francisco" }, ...]
|
|
345
|
-
```
|
|
346
|
-
|
|
347
|
-
### List ASNs
|
|
348
|
-
|
|
349
|
-
```typescript
|
|
350
|
-
const asns = await client.listAsns("US", "residential");
|
|
351
|
-
console.log(`Found ${asns.length} ASNs`);
|
|
352
|
-
// [{ asn_code: "AS7922", asn_name: "Comcast" }, ...]
|
|
353
|
-
```
|
|
354
|
-
|
|
355
|
-
---
|
|
356
|
-
|
|
357
|
-
## 🕷️ Web Scraper API (Task-based)
|
|
358
|
-
|
|
359
|
-
Requires `THORDATA_PUBLIC_TOKEN` and `THORDATA_PUBLIC_KEY`.
|
|
360
|
-
|
|
361
|
-
```typescript
|
|
362
|
-
const client = new ThordataClient({
|
|
363
|
-
scraperToken: process.env.THORDATA_TOKEN!,
|
|
364
|
-
publicToken: process.env.THORDATA_PUBLIC_TOKEN,
|
|
365
|
-
publicKey: process.env.THORDATA_PUBLIC_KEY,
|
|
366
|
-
});
|
|
367
|
-
|
|
368
|
-
// Create task
|
|
79
|
+
// Create Task
|
|
369
80
|
const taskId = await client.createScraperTask({
|
|
370
|
-
fileName: "
|
|
371
|
-
spiderId: "
|
|
372
|
-
spiderName: "
|
|
81
|
+
fileName: "task1",
|
|
82
|
+
spiderId: "universal",
|
|
83
|
+
spiderName: "universal",
|
|
373
84
|
parameters: { url: "https://example.com" },
|
|
374
85
|
});
|
|
375
86
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
87
|
+
// Video Task (New)
|
|
88
|
+
const vidId = await client.createVideoTask({
|
|
89
|
+
fileName: "video1",
|
|
90
|
+
spiderId: "youtube_video_by-url",
|
|
91
|
+
spiderName: "youtube.com",
|
|
92
|
+
parameters: { url: "..." },
|
|
93
|
+
commonSettings: { resolution: "1080p" },
|
|
382
94
|
});
|
|
383
95
|
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
console.log("Download URL:", downloadUrl);
|
|
390
|
-
}
|
|
391
|
-
```
|
|
392
|
-
|
|
393
|
-
---
|
|
394
|
-
|
|
395
|
-
## 🔧 Error Handling
|
|
396
|
-
|
|
397
|
-
The SDK throws typed errors based on API response codes:
|
|
398
|
-
|
|
399
|
-
| Code | Meaning | Error Class |
|
|
400
|
-
| ------- | ------------- | ------------------------- |
|
|
401
|
-
| 200 | Success | - |
|
|
402
|
-
| 300 | Not collected | ThordataNotCollectedError |
|
|
403
|
-
| 400 | Bad request | ThordataValidationError |
|
|
404
|
-
| 401/403 | Auth error | ThordataAuthError |
|
|
405
|
-
| 402/429 | Rate limit | ThordataRateLimitError |
|
|
406
|
-
| 5xx | Server error | ThordataServerError |
|
|
407
|
-
|
|
408
|
-
> **Billing Note:** Only requests with code == 200 are billed.
|
|
409
|
-
|
|
410
|
-
### Example
|
|
411
|
-
|
|
412
|
-
```typescript
|
|
413
|
-
import {
|
|
414
|
-
ThordataAuthError,
|
|
415
|
-
ThordataRateLimitError,
|
|
416
|
-
ThordataTimeoutError,
|
|
417
|
-
ThordataNotCollectedError,
|
|
418
|
-
} from "thordata-js-sdk";
|
|
419
|
-
|
|
420
|
-
try {
|
|
421
|
-
const data = await client.serpSearch({ query: "test", engine: "google" });
|
|
422
|
-
console.log(data);
|
|
423
|
-
} catch (e) {
|
|
424
|
-
if (e instanceof ThordataAuthError) {
|
|
425
|
-
console.error("Auth error: check your token.");
|
|
426
|
-
} else if (e instanceof ThordataRateLimitError) {
|
|
427
|
-
console.error(`Rate limited. Retry after: ${e.retryAfter ?? "N/A"} seconds.`);
|
|
428
|
-
} else if (e instanceof ThordataNotCollectedError) {
|
|
429
|
-
console.error("Not collected (code=300). Consider retrying.");
|
|
430
|
-
} else if (e instanceof ThordataTimeoutError) {
|
|
431
|
-
console.error("Request timed out.");
|
|
432
|
-
} else {
|
|
433
|
-
console.error("Unexpected error:", e);
|
|
434
|
-
}
|
|
96
|
+
// Wait & Result
|
|
97
|
+
const status = await client.waitForTask(taskId);
|
|
98
|
+
if (status === "ready") {
|
|
99
|
+
const url = await client.getTaskResult(taskId);
|
|
100
|
+
console.log(url);
|
|
435
101
|
}
|
|
436
102
|
```
|
|
437
103
|
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
## ⚙️ Advanced Configuration
|
|
441
|
-
|
|
442
|
-
### SSL Verification
|
|
443
|
-
|
|
444
|
-
```typescript
|
|
445
|
-
const client = new ThordataClient({
|
|
446
|
-
scraperToken: process.env.THORDATA_TOKEN!,
|
|
447
|
-
verifySsl: false, // Only for testing with self-signed certs
|
|
448
|
-
});
|
|
449
|
-
```
|
|
450
|
-
|
|
451
|
-
### Retry Configuration
|
|
104
|
+
### Account Management
|
|
452
105
|
|
|
453
106
|
```typescript
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
timeoutMs: 60000, // 60 second timeout
|
|
458
|
-
});
|
|
459
|
-
```
|
|
460
|
-
|
|
461
|
-
### Base URL Overrides
|
|
107
|
+
// Usage Stats
|
|
108
|
+
const stats = await client.getUsageStatistics("2024-01-01", "2024-01-31");
|
|
109
|
+
console.log("Balance:", stats.traffic_balance);
|
|
462
110
|
|
|
463
|
-
|
|
111
|
+
// Proxy Users
|
|
112
|
+
const users = await client.listProxyUsers("residential");
|
|
464
113
|
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
export THORDATA_UNIVERSALAPI_BASE_URL=http://127.0.0.1:12345
|
|
114
|
+
// Whitelist
|
|
115
|
+
await client.addWhitelistIp("1.2.3.4");
|
|
468
116
|
```
|
|
469
117
|
|
|
470
|
-
|
|
118
|
+
### Proxy Configuration
|
|
471
119
|
|
|
472
120
|
```typescript
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
});
|
|
477
|
-
```
|
|
478
|
-
|
|
479
|
-
---
|
|
480
|
-
|
|
481
|
-
## 🧪 Development
|
|
482
|
-
|
|
483
|
-
```bash
|
|
484
|
-
npm install
|
|
485
|
-
npm run build
|
|
486
|
-
npm test
|
|
487
|
-
npm run lint
|
|
488
|
-
npm run format
|
|
489
|
-
```
|
|
490
|
-
|
|
491
|
-
### Run Examples
|
|
492
|
-
|
|
493
|
-
```bash
|
|
494
|
-
node dist/examples/basic_serp.js
|
|
495
|
-
node dist/examples/basic_universal.js
|
|
496
|
-
```
|
|
497
|
-
|
|
498
|
-
---
|
|
499
|
-
|
|
500
|
-
## 📁 Project Structure
|
|
501
|
-
|
|
502
|
-
```
|
|
503
|
-
thordata-js-sdk/
|
|
504
|
-
├── src/
|
|
505
|
-
│ ├── index.ts # Main exports
|
|
506
|
-
│ ├── client.ts # ThordataClient class
|
|
507
|
-
│ ├── thordata.ts # Convenience wrapper
|
|
508
|
-
│ ├── proxy.ts # Proxy configuration
|
|
509
|
-
│ ├── models.ts # TypeScript interfaces
|
|
510
|
-
│ ├── enums.ts # Enumerations
|
|
511
|
-
│ ├── errors.ts # Error classes
|
|
512
|
-
│ ├── utils.ts # Helper functions
|
|
513
|
-
│ └── endpoints.ts # API endpoints
|
|
514
|
-
├── examples/
|
|
515
|
-
├── tests/
|
|
516
|
-
├── package.json
|
|
517
|
-
└── README.md
|
|
121
|
+
// Residential Proxy
|
|
122
|
+
const proxy = Thordata.Proxy.residentialFromEnv().country("us");
|
|
123
|
+
await client.request("https://httpbin.org/ip", { proxy });
|
|
518
124
|
```
|
|
519
125
|
|
|
520
126
|
---
|
|
521
127
|
|
|
522
|
-
## 📚 API Reference
|
|
523
|
-
|
|
524
|
-
For detailed API documentation, visit:
|
|
525
|
-
|
|
526
|
-
- [Google Search Parameters](#)
|
|
527
|
-
- [Google News Parameters](#)
|
|
528
|
-
- [Google Shopping Parameters](#)
|
|
529
|
-
- [Yandex Parameters](#)
|
|
530
|
-
- [Universal API Parameters](#)
|
|
531
|
-
|
|
532
|
-
---
|
|
533
|
-
|
|
534
128
|
## 📄 License
|
|
535
129
|
|
|
536
|
-
MIT License
|
|
130
|
+
MIT License
|
|
@@ -5,42 +5,67 @@ async function main() {
|
|
|
5
5
|
const scraperToken = process.env.THORDATA_SCRAPER_TOKEN;
|
|
6
6
|
const publicToken = process.env.THORDATA_PUBLIC_TOKEN;
|
|
7
7
|
const publicKey = process.env.THORDATA_PUBLIC_KEY;
|
|
8
|
+
// These are NOT universal constants; users should copy them from Dashboard -> Web Scraper Store -> API Builder.
|
|
9
|
+
const spiderName = process.env.THORDATA_TASK_SPIDER_NAME;
|
|
10
|
+
const spiderId = process.env.THORDATA_TASK_SPIDER_ID;
|
|
11
|
+
const fileName = process.env.THORDATA_TASK_FILE_NAME || "{{TasksID}}";
|
|
12
|
+
const parametersJson = process.env.THORDATA_TASK_PARAMETERS_JSON || "{}";
|
|
8
13
|
if (!scraperToken || !publicToken || !publicKey) {
|
|
9
14
|
console.error("Please set THORDATA_SCRAPER_TOKEN, THORDATA_PUBLIC_TOKEN, THORDATA_PUBLIC_KEY in .env");
|
|
10
15
|
process.exit(1);
|
|
11
16
|
}
|
|
17
|
+
// If task-specific env vars are missing, skip the example to keep CI/offline e2e stable.
|
|
18
|
+
if (!spiderName || !spiderId) {
|
|
19
|
+
console.log("Skipping tasks example. Set THORDATA_TASK_SPIDER_NAME and THORDATA_TASK_SPIDER_ID to run it (copy from Dashboard -> Web Scraper Store -> API Builder).");
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
let parameters;
|
|
23
|
+
try {
|
|
24
|
+
const raw = JSON.parse(parametersJson);
|
|
25
|
+
// Dashboard curl uses spider_parameters as an ARRAY string: [{...}]
|
|
26
|
+
// Accept both "{...}" and "[{...}]". SDK createScraperTask expects ONE parameter object.
|
|
27
|
+
if (Array.isArray(raw)) {
|
|
28
|
+
if (raw.length === 0) {
|
|
29
|
+
console.error("THORDATA_TASK_PARAMETERS_JSON must not be an empty array");
|
|
30
|
+
process.exit(1);
|
|
31
|
+
}
|
|
32
|
+
parameters = raw[0];
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
parameters = raw;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
console.error("THORDATA_TASK_PARAMETERS_JSON must be valid JSON");
|
|
40
|
+
process.exit(1);
|
|
41
|
+
}
|
|
12
42
|
const client = new ThordataClient({
|
|
13
43
|
scraperToken,
|
|
14
44
|
publicToken,
|
|
15
45
|
publicKey,
|
|
16
46
|
});
|
|
17
|
-
console.log("🕷️ Creating Web Scraper task (example
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
const downloadUrl = await client.getTaskResult(taskId, "json");
|
|
36
|
-
console.log("Download URL:", downloadUrl);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
catch (err) {
|
|
40
|
-
console.error("Error:", err);
|
|
47
|
+
console.log("🕷️ Creating Web Scraper task (live example)...");
|
|
48
|
+
const taskId = await client.createScraperTask({
|
|
49
|
+
// Recommended by docs: let the server substitute the actual task id.
|
|
50
|
+
fileName,
|
|
51
|
+
spiderId,
|
|
52
|
+
spiderName,
|
|
53
|
+
parameters,
|
|
54
|
+
});
|
|
55
|
+
console.log("Task created:", taskId);
|
|
56
|
+
console.log("⏱️ Waiting for task completion...");
|
|
57
|
+
const status = await client.waitForTask(taskId, {
|
|
58
|
+
pollIntervalMs: 5000,
|
|
59
|
+
maxWaitMs: 120_000,
|
|
60
|
+
});
|
|
61
|
+
console.log("Final status:", status);
|
|
62
|
+
if (status.toLowerCase() === "ready" || status.toLowerCase() === "success") {
|
|
63
|
+
const downloadUrl = await client.getTaskResult(taskId, "json");
|
|
64
|
+
console.log("Download URL:", downloadUrl);
|
|
41
65
|
}
|
|
42
66
|
}
|
|
43
67
|
main().catch((err) => {
|
|
44
68
|
console.error("Fatal error:", err);
|
|
69
|
+
process.exit(1);
|
|
45
70
|
});
|
|
46
71
|
//# sourceMappingURL=basic_scraper_task.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"basic_scraper_task.js","sourceRoot":"","sources":["../../examples/basic_scraper_task.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAEjC,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,KAAK,UAAU,IAAI;IACjB,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAElD,IAAI,CAAC,YAAY,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;QAChD,OAAO,CAAC,KAAK,CACX,uFAAuF,CACxF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC;QAChC,YAAY;QACZ,WAAW;QACX,SAAS;KACV,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;IAChE,
|
|
1
|
+
{"version":3,"file":"basic_scraper_task.js","sourceRoot":"","sources":["../../examples/basic_scraper_task.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAEjC,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,KAAK,UAAU,IAAI;IACjB,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAElD,gHAAgH;IAChH,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,aAAa,CAAC;IACtE,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,IAAI,CAAC;IAEzE,IAAI,CAAC,YAAY,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;QAChD,OAAO,CAAC,KAAK,CACX,uFAAuF,CACxF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yFAAyF;IACzF,IAAI,CAAC,UAAU,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CACT,wJAAwJ,CACzJ,CAAC;QACF,OAAO;IACT,CAAC;IAED,IAAI,UAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAEvC,oEAAoE;QACpE,yFAAyF;QACzF,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,KAAK,CAAC,0DAA0D,CAAC,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,UAAU,GAAG,GAAG,CAAC;QACnB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC;QAChC,YAAY;QACZ,WAAW;QACX,SAAS;KACV,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC;QAC5C,qEAAqE;QACrE,QAAQ;QACR,QAAQ;QACR,UAAU;QACV,UAAU;KACX,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAErC,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE;QAC9C,cAAc,EAAE,IAAI;QACpB,SAAS,EAAE,OAAO;KACnB,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAErC,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE,CAAC;QAC3E,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export declare function loadEnv(): void;
|
|
2
|
+
export declare function env(name: string): string;
|
|
3
|
+
export declare function skipIfMissing(...required: string[]): boolean;
|
|
4
|
+
export declare function printJSON(v: unknown): void;
|
|
5
|
+
export declare function truncate(s: string, n: number): string;
|