thordata-js-sdk 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -20
- package/README.md +55 -349
- package/dist/examples/basic_scraper_task.js +37 -24
- package/dist/examples/basic_scraper_task.js.map +1 -1
- package/dist/examples/basic_serp.js +3 -3
- package/dist/examples/basic_serp.js.map +1 -1
- package/dist/examples/location_api.js +47 -0
- package/dist/examples/location_api.js.map +1 -0
- package/dist/examples/proxy_datacenter.js +1 -1
- package/dist/examples/proxy_datacenter.js.map +1 -1
- package/dist/examples/proxy_isp.js +1 -1
- package/dist/examples/proxy_isp.js.map +1 -1
- package/dist/examples/proxy_mobile.js +1 -1
- package/dist/examples/proxy_mobile.js.map +1 -1
- package/dist/examples/proxy_residential.js +2 -2
- package/dist/examples/proxy_residential.js.map +1 -1
- package/dist/examples/serp_google_news.d.ts +0 -6
- package/dist/examples/serp_google_news.js +7 -10
- package/dist/examples/serp_google_news.js.map +1 -1
- package/dist/examples/verify_new_features.js +135 -0
- package/dist/examples/verify_new_features.js.map +1 -0
- package/dist/src/auth.d.ts +22 -0
- package/dist/src/auth.js +39 -0
- package/dist/src/auth.js.map +1 -0
- package/dist/src/client.d.ts +144 -6
- package/dist/src/client.js +396 -43
- package/dist/src/client.js.map +1 -1
- package/dist/src/endpoints.js +2 -2
- package/dist/src/endpoints.js.map +1 -1
- package/dist/src/enums.d.ts +219 -3
- package/dist/src/enums.js +289 -2
- package/dist/src/enums.js.map +1 -1
- package/dist/src/errors.d.ts +37 -5
- package/dist/src/errors.js +61 -17
- package/dist/src/errors.js.map +1 -1
- package/dist/src/index.d.ts +7 -5
- package/dist/src/index.js +21 -5
- package/dist/src/index.js.map +1 -1
- package/dist/src/models.d.ts +196 -12
- package/dist/src/models.js +1 -0
- package/dist/src/models.js.map +1 -1
- package/dist/src/proxy.d.ts +59 -0
- package/dist/src/proxy.js +83 -11
- package/dist/src/proxy.js.map +1 -1
- package/dist/src/retry.d.ts +25 -3
- package/dist/src/retry.js +16 -0
- package/dist/src/retry.js.map +1 -1
- package/dist/src/thordata.d.ts +5 -2
- package/dist/src/thordata.js +36 -6
- package/dist/src/thordata.js.map +1 -1
- package/dist/src/utils.d.ts +18 -13
- package/dist/src/utils.js +94 -43
- package/dist/src/utils.js.map +1 -1
- package/package.json +6 -6
- package/dist/examples/proxy_demo.js +0 -69
- package/dist/examples/proxy_demo.js.map +0 -1
- package/dist/examples/proxy_ip_check.js +0 -37
- package/dist/examples/proxy_ip_check.js.map +0 -1
- /package/dist/examples/{proxy_demo.d.ts → location_api.d.ts} +0 -0
- /package/dist/examples/{proxy_ip_check.d.ts → verify_new_features.d.ts} +0 -0
package/LICENSE
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2025 Thordata
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Thordata
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
21
|
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,18 +1,17 @@
|
|
|
1
1
|
# Thordata JS SDK (Node.js / TypeScript)
|
|
2
2
|
|
|
3
|
-
Official JavaScript/TypeScript SDK for
|
|
3
|
+
Official JavaScript/TypeScript SDK for Thordata APIs.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
[](https://www.npmjs.com/package/thordata-js-sdk)
|
|
6
|
+
[](LICENSE)
|
|
6
7
|
|
|
7
|
-
|
|
8
|
-
- **Web Unlocker / Universal API**
|
|
9
|
-
- **Web Scraper API** (task-based scraping)
|
|
8
|
+
Supports:
|
|
10
9
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
- **
|
|
14
|
-
- **
|
|
15
|
-
- **
|
|
10
|
+
- **SERP API** (Google / Bing / Yandex)
|
|
11
|
+
- **Web Unlocker** (Universal API)
|
|
12
|
+
- **Web Scraper API** (Text & Video Tasks)
|
|
13
|
+
- **Proxy Network** (Residential / Datacenter / Mobile / ISP)
|
|
14
|
+
- **Account Management** (Usage, Users, Whitelist)
|
|
16
15
|
|
|
17
16
|
---
|
|
18
17
|
|
|
@@ -22,403 +21,110 @@ It is designed to be:
|
|
|
22
21
|
npm install thordata-js-sdk
|
|
23
22
|
```
|
|
24
23
|
|
|
25
|
-
If you are developing locally:
|
|
26
|
-
|
|
27
|
-
```bash
|
|
28
|
-
git clone https://github.com/Thordata/thordata-js-sdk.git
|
|
29
|
-
cd thordata-js-sdk
|
|
30
|
-
npm install
|
|
31
|
-
npm run build
|
|
32
|
-
```
|
|
33
|
-
|
|
34
24
|
---
|
|
35
25
|
|
|
36
26
|
## 🔐 Configuration
|
|
37
27
|
|
|
38
|
-
Set environment variables:
|
|
39
|
-
|
|
40
28
|
```bash
|
|
41
|
-
export THORDATA_SCRAPER_TOKEN=
|
|
29
|
+
export THORDATA_SCRAPER_TOKEN=your_token
|
|
42
30
|
export THORDATA_PUBLIC_TOKEN=your_public_token
|
|
43
31
|
export THORDATA_PUBLIC_KEY=your_public_key
|
|
44
32
|
```
|
|
45
33
|
|
|
46
|
-
Or create a `.env` file (examples may use dotenv):
|
|
47
|
-
|
|
48
|
-
```env
|
|
49
|
-
THORDATA_SCRAPER_TOKEN=your_scraper_token
|
|
50
|
-
THORDATA_PUBLIC_TOKEN=your_public_token
|
|
51
|
-
THORDATA_PUBLIC_KEY=your_public_key
|
|
52
|
-
```
|
|
53
|
-
|
|
54
34
|
---
|
|
55
35
|
|
|
56
36
|
## 🚀 Quick Start
|
|
57
37
|
|
|
58
|
-
### Create a client
|
|
59
|
-
|
|
60
|
-
```typescript
|
|
61
|
-
import { ThordataClient } from "thordata-js-sdk";
|
|
62
|
-
|
|
63
|
-
const client = new ThordataClient({
|
|
64
|
-
scraperToken: process.env.THORDATA_SCRAPER_TOKEN!,
|
|
65
|
-
publicToken: process.env.THORDATA_PUBLIC_TOKEN,
|
|
66
|
-
publicKey: process.env.THORDATA_PUBLIC_KEY,
|
|
67
|
-
});
|
|
68
|
-
```
|
|
69
|
-
|
|
70
|
-
---
|
|
71
|
-
|
|
72
|
-
## 🔍 SERP API
|
|
73
|
-
|
|
74
|
-
### Basic Google Search
|
|
75
|
-
|
|
76
|
-
```typescript
|
|
77
|
-
import { ThordataClient, Engine } from "thordata-js-sdk";
|
|
78
|
-
|
|
79
|
-
const client = new ThordataClient({ scraperToken: process.env.THORDATA_SCRAPER_TOKEN! });
|
|
80
|
-
|
|
81
|
-
const data = await client.serpSearch({
|
|
82
|
-
query: "Thordata proxy network",
|
|
83
|
-
engine: Engine.GOOGLE,
|
|
84
|
-
num: 5,
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
const organic = data?.organic ?? [];
|
|
88
|
-
console.log(`Found ${organic.length} organic results`);
|
|
89
|
-
|
|
90
|
-
for (const item of organic.slice(0, 3)) {
|
|
91
|
-
console.log("-", item.title, "->", item.link);
|
|
92
|
-
}
|
|
93
|
-
```
|
|
94
|
-
|
|
95
|
-
### Recommended engines for Google verticals (News / Shopping)
|
|
96
|
-
|
|
97
|
-
Thordata supports both:
|
|
98
|
-
|
|
99
|
-
- Dedicated engines (recommended): `google_news`, `google_shopping`
|
|
100
|
-
- Generic Google + tbm via `searchType` (alternative)
|
|
101
|
-
|
|
102
|
-
#### Google News (recommended):
|
|
103
|
-
|
|
104
|
-
```typescript
|
|
105
|
-
const data = await client.serpSearch({
|
|
106
|
-
query: "AI regulation",
|
|
107
|
-
engine: "google_news",
|
|
108
|
-
country: "us",
|
|
109
|
-
language: "en",
|
|
110
|
-
num: 10,
|
|
111
|
-
so: 1, // 0=relevance, 1=date (Google News)
|
|
112
|
-
});
|
|
113
|
-
```
|
|
114
|
-
|
|
115
|
-
#### Google Shopping (recommended):
|
|
116
|
-
|
|
117
38
|
```typescript
|
|
118
|
-
|
|
119
|
-
query: "iPhone 15",
|
|
120
|
-
engine: "google_shopping",
|
|
121
|
-
country: "us",
|
|
122
|
-
language: "en",
|
|
123
|
-
num: 10,
|
|
124
|
-
min_price: 500,
|
|
125
|
-
max_price: 1500,
|
|
126
|
-
});
|
|
127
|
-
```
|
|
39
|
+
import { Thordata } from "thordata-js-sdk";
|
|
128
40
|
|
|
129
|
-
|
|
41
|
+
const client = new Thordata(); // Reads from env vars
|
|
130
42
|
|
|
131
|
-
|
|
132
|
-
const
|
|
133
|
-
query: "
|
|
43
|
+
// SERP Search
|
|
44
|
+
const results = await client.serpSearch({
|
|
45
|
+
query: "Thordata SDK",
|
|
134
46
|
engine: "google",
|
|
135
|
-
searchType: "shopping", // maps to tbm=shop
|
|
136
47
|
country: "us",
|
|
137
|
-
language: "en",
|
|
138
|
-
num: 10,
|
|
139
48
|
});
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
Official and up-to-date parameters are documented at: https://doc.thordata.com
|
|
143
|
-
|
|
144
|
-
---
|
|
145
|
-
|
|
146
|
-
## 🌐 Proxy Network
|
|
147
|
-
|
|
148
|
-
Each proxy product requires separate credentials from Thordata Dashboard.
|
|
149
|
-
|
|
150
|
-
### Environment Variables
|
|
151
|
-
|
|
152
|
-
```env
|
|
153
|
-
# Residential Proxy (port 9999)
|
|
154
|
-
THORDATA_RESIDENTIAL_USERNAME=your_residential_username
|
|
155
|
-
THORDATA_RESIDENTIAL_PASSWORD=your_residential_password
|
|
156
|
-
|
|
157
|
-
# Datacenter Proxy (port 7777)
|
|
158
|
-
THORDATA_DATACENTER_USERNAME=your_datacenter_username
|
|
159
|
-
THORDATA_DATACENTER_PASSWORD=your_datacenter_password
|
|
160
|
-
|
|
161
|
-
# Mobile Proxy (port 5555)
|
|
162
|
-
THORDATA_MOBILE_USERNAME=your_mobile_username
|
|
163
|
-
THORDATA_MOBILE_PASSWORD=your_mobile_password
|
|
164
|
-
|
|
165
|
-
# Static ISP Proxy (port 6666, direct IP connection)
|
|
166
|
-
THORDATA_ISP_HOST=your_static_ip_address
|
|
167
|
-
THORDATA_ISP_USERNAME=your_isp_username
|
|
168
|
-
THORDATA_ISP_PASSWORD=your_isp_password
|
|
169
|
-
```
|
|
170
|
-
|
|
171
|
-
### Residential Proxy
|
|
172
|
-
|
|
173
|
-
```typescript
|
|
174
|
-
import { Thordata } from "thordata-js-sdk";
|
|
175
|
-
|
|
176
|
-
const client = new Thordata();
|
|
177
|
-
|
|
178
|
-
// Basic US residential
|
|
179
|
-
const proxy = Thordata.Proxy.residentialFromEnv().country("us");
|
|
180
|
-
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
181
|
-
console.log("IP:", result.origin);
|
|
182
|
-
|
|
183
|
-
// Sticky session (same IP for 30 minutes)
|
|
184
|
-
const stickyProxy = Thordata.Proxy.residentialFromEnv()
|
|
185
|
-
.country("jp")
|
|
186
|
-
.city("tokyo")
|
|
187
|
-
.session("my_session")
|
|
188
|
-
.sticky(30);
|
|
189
|
-
const result2 = await client.request("http://httpbin.org/ip", { proxy: stickyProxy });
|
|
190
|
-
```
|
|
191
|
-
|
|
192
|
-
### Datacenter Proxy
|
|
193
|
-
|
|
194
|
-
```typescript
|
|
195
|
-
const proxy = Thordata.Proxy.datacenterFromEnv();
|
|
196
|
-
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
197
|
-
console.log("Datacenter IP:", result.origin);
|
|
198
|
-
```
|
|
199
|
-
|
|
200
|
-
### Mobile Proxy
|
|
201
|
-
|
|
202
|
-
```typescript
|
|
203
|
-
const proxy = Thordata.Proxy.mobileFromEnv().country("gb");
|
|
204
|
-
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
205
|
-
console.log("UK Mobile IP:", result.origin);
|
|
206
|
-
```
|
|
207
|
-
|
|
208
|
-
### Static ISP Proxy
|
|
209
|
-
|
|
210
|
-
```typescript
|
|
211
|
-
const proxy = Thordata.Proxy.ispFromEnv();
|
|
212
|
-
const result = await client.request("http://httpbin.org/ip", { proxy });
|
|
213
|
-
console.log("Static ISP IP:", result.origin);
|
|
214
|
-
// Returns your purchased static IP address
|
|
215
|
-
```
|
|
216
|
-
|
|
217
|
-
### Proxy Examples
|
|
218
|
-
|
|
219
|
-
```bash
|
|
220
|
-
node dist/examples/proxy_residential.js
|
|
221
|
-
node dist/examples/proxy_datacenter.js
|
|
222
|
-
node dist/examples/proxy_mobile.js
|
|
223
|
-
node dist/examples/proxy_isp.js
|
|
49
|
+
console.log(results.organic?.[0]?.link);
|
|
224
50
|
```
|
|
225
51
|
|
|
226
52
|
---
|
|
227
53
|
|
|
228
|
-
##
|
|
54
|
+
## 📖 Features
|
|
229
55
|
|
|
230
|
-
###
|
|
56
|
+
### SERP API
|
|
231
57
|
|
|
232
58
|
```typescript
|
|
233
|
-
const
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
59
|
+
const news = await client.serpSearch({
|
|
60
|
+
query: "AI News",
|
|
61
|
+
engine: "google_news",
|
|
62
|
+
num: 10,
|
|
237
63
|
});
|
|
238
|
-
|
|
239
|
-
console.log(String(html).slice(0, 300));
|
|
240
64
|
```
|
|
241
65
|
|
|
242
|
-
###
|
|
66
|
+
### Web Unlocker (Universal)
|
|
243
67
|
|
|
244
68
|
```typescript
|
|
245
69
|
const html = await client.universalScrape({
|
|
246
|
-
url: "https://example.com/spa",
|
|
247
|
-
jsRender: true,
|
|
248
|
-
outputFormat: "html",
|
|
249
|
-
waitFor: ".main-content",
|
|
250
|
-
});
|
|
251
|
-
```
|
|
252
|
-
|
|
253
|
-
### Screenshot (PNG)
|
|
254
|
-
|
|
255
|
-
```typescript
|
|
256
|
-
import { writeFileSync } from "node:fs";
|
|
257
|
-
|
|
258
|
-
const pngBytes = await client.universalScrape({
|
|
259
70
|
url: "https://example.com",
|
|
260
71
|
jsRender: true,
|
|
261
|
-
|
|
72
|
+
waitFor: ".content",
|
|
262
73
|
});
|
|
263
|
-
|
|
264
|
-
writeFileSync("screenshot.png", pngBytes as Buffer);
|
|
265
74
|
```
|
|
266
75
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
## 🕷️ Web Scraper API (Task-based)
|
|
270
|
-
|
|
271
|
-
Requires `THORDATA_PUBLIC_TOKEN` and `THORDATA_PUBLIC_KEY`.
|
|
76
|
+
### Web Scraper API (Async)
|
|
272
77
|
|
|
273
78
|
```typescript
|
|
274
|
-
|
|
275
|
-
scraperToken: process.env.THORDATA_SCRAPER_TOKEN!,
|
|
276
|
-
publicToken: process.env.THORDATA_PUBLIC_TOKEN,
|
|
277
|
-
publicKey: process.env.THORDATA_PUBLIC_KEY,
|
|
278
|
-
});
|
|
279
|
-
|
|
79
|
+
// Create Task
|
|
280
80
|
const taskId = await client.createScraperTask({
|
|
281
|
-
fileName: "
|
|
282
|
-
spiderId: "
|
|
283
|
-
spiderName: "
|
|
81
|
+
fileName: "task1",
|
|
82
|
+
spiderId: "universal",
|
|
83
|
+
spiderName: "universal",
|
|
284
84
|
parameters: { url: "https://example.com" },
|
|
285
85
|
});
|
|
286
86
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
87
|
+
// Video Task (New)
|
|
88
|
+
const vidId = await client.createVideoTask({
|
|
89
|
+
fileName: "video1",
|
|
90
|
+
spiderId: "youtube_video_by-url",
|
|
91
|
+
spiderName: "youtube.com",
|
|
92
|
+
parameters: { url: "..." },
|
|
93
|
+
commonSettings: { resolution: "1080p" },
|
|
292
94
|
});
|
|
293
95
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
if (
|
|
297
|
-
const
|
|
298
|
-
console.log(
|
|
96
|
+
// Wait & Result
|
|
97
|
+
const status = await client.waitForTask(taskId);
|
|
98
|
+
if (status === "ready") {
|
|
99
|
+
const url = await client.getTaskResult(taskId);
|
|
100
|
+
console.log(url);
|
|
299
101
|
}
|
|
300
102
|
```
|
|
301
103
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
## 🔧 Errors & Response Codes
|
|
305
|
-
|
|
306
|
-
The SDK throws typed errors when the API returns a non-success code (or non-2xx HTTP status).
|
|
307
|
-
|
|
308
|
-
| Code | Typical Meaning | Error class |
|
|
309
|
-
| ------- | --------------------- | ---------------------------------------------- |
|
|
310
|
-
| 200 | Success | - |
|
|
311
|
-
| 300 | Not collected | `ThordataNotCollectedError` |
|
|
312
|
-
| 400 | Bad request | `ThordataValidationError` |
|
|
313
|
-
| 401/403 | Auth/Forbidden | `ThordataAuthError` |
|
|
314
|
-
| 402/429 | Quota/Rate limit | `ThordataRateLimitError` |
|
|
315
|
-
| 5xx | Server/timeout issues | `ThordataServerError` / `ThordataTimeoutError` |
|
|
316
|
-
|
|
317
|
-
### Example error handling:
|
|
104
|
+
### Account Management
|
|
318
105
|
|
|
319
106
|
```typescript
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
ThordataTimeoutError,
|
|
324
|
-
ThordataNotCollectedError,
|
|
325
|
-
} from "thordata-js-sdk";
|
|
326
|
-
|
|
327
|
-
try {
|
|
328
|
-
const data = await client.serpSearch({ query: "test", engine: "google" });
|
|
329
|
-
console.log(data);
|
|
330
|
-
} catch (e) {
|
|
331
|
-
if (e instanceof ThordataAuthError) {
|
|
332
|
-
console.error("Auth error: check your token.");
|
|
333
|
-
} else if (e instanceof ThordataRateLimitError) {
|
|
334
|
-
console.error(`Rate limited. Retry after: ${e.retryAfter ?? "N/A"} seconds.`);
|
|
335
|
-
} else if (e instanceof ThordataNotCollectedError) {
|
|
336
|
-
console.error("Not collected (code=300). Consider retrying.");
|
|
337
|
-
} else if (e instanceof ThordataTimeoutError) {
|
|
338
|
-
console.error("Request timed out.");
|
|
339
|
-
} else {
|
|
340
|
-
console.error("Unexpected error:", e);
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
```
|
|
107
|
+
// Usage Stats
|
|
108
|
+
const stats = await client.getUsageStatistics("2024-01-01", "2024-01-31");
|
|
109
|
+
console.log("Balance:", stats.traffic_balance);
|
|
344
110
|
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
## 🌍 Base URL Overrides (for offline tests / custom routing)
|
|
111
|
+
// Proxy Users
|
|
112
|
+
const users = await client.listProxyUsers("residential");
|
|
348
113
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
```bash
|
|
352
|
-
export THORDATA_SCRAPERAPI_BASE_URL=http://127.0.0.1:12345
|
|
353
|
-
export THORDATA_UNIVERSALAPI_BASE_URL=http://127.0.0.1:12345
|
|
354
|
-
export THORDATA_WEB_SCRAPER_API_BASE_URL=http://127.0.0.1:12345
|
|
355
|
-
export THORDATA_LOCATIONS_BASE_URL=http://127.0.0.1:12345
|
|
114
|
+
// Whitelist
|
|
115
|
+
await client.addWhitelistIp("1.2.3.4");
|
|
356
116
|
```
|
|
357
117
|
|
|
358
|
-
|
|
118
|
+
### Proxy Configuration
|
|
359
119
|
|
|
360
120
|
```typescript
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
});
|
|
365
|
-
```
|
|
366
|
-
|
|
367
|
-
---
|
|
368
|
-
|
|
369
|
-
## 🧪 Development
|
|
370
|
-
|
|
371
|
-
```bash
|
|
372
|
-
npm install
|
|
373
|
-
npm run build
|
|
374
|
-
npm test
|
|
375
|
-
```
|
|
376
|
-
|
|
377
|
-
### Run examples (compiled):
|
|
378
|
-
|
|
379
|
-
```bash
|
|
380
|
-
node dist/examples/basic_serp.js
|
|
381
|
-
node dist/examples/basic_universal.js
|
|
382
|
-
```
|
|
383
|
-
|
|
384
|
-
---
|
|
385
|
-
|
|
386
|
-
## 📁 Project Structure
|
|
387
|
-
|
|
388
|
-
```
|
|
389
|
-
thordata-js-sdk/
|
|
390
|
-
├── src/
|
|
391
|
-
│ ├── index.ts
|
|
392
|
-
│ ├── client.ts
|
|
393
|
-
│ ├── models.ts
|
|
394
|
-
│ ├── enums.ts
|
|
395
|
-
│ ├── errors.ts
|
|
396
|
-
│ ├── retry.ts
|
|
397
|
-
│ ├── endpoints.ts
|
|
398
|
-
│ └── utils.ts
|
|
399
|
-
├── examples/
|
|
400
|
-
│ ├── basic_serp.ts
|
|
401
|
-
│ ├── basic_universal.ts
|
|
402
|
-
│ ├── basic_scraper_task.ts
|
|
403
|
-
│ └── serp_google_news.ts
|
|
404
|
-
├── tests/
|
|
405
|
-
│ ├── serp.offline.test.ts
|
|
406
|
-
│ ├── mockServer.ts
|
|
407
|
-
│ └── examples.e2e.test.ts
|
|
408
|
-
├── .github/workflows/ci.yml
|
|
409
|
-
├── package.json
|
|
410
|
-
├── tsconfig.json
|
|
411
|
-
├── tsconfig.build.json
|
|
412
|
-
└── README.md
|
|
121
|
+
// Residential Proxy
|
|
122
|
+
const proxy = Thordata.Proxy.residentialFromEnv().country("us");
|
|
123
|
+
await client.request("https://httpbin.org/ip", { proxy });
|
|
413
124
|
```
|
|
414
125
|
|
|
415
126
|
---
|
|
416
127
|
|
|
417
|
-
##
|
|
128
|
+
## 📄 License
|
|
418
129
|
|
|
419
|
-
|
|
420
|
-
- Add async streaming / higher-level helpers for AI agents
|
|
421
|
-
- Expand coverage for more engines/verticals (Flights/Maps/Scholar/Jobs, etc.)
|
|
422
|
-
- Add integration tests (optional scheduled job with real tokens)
|
|
423
|
-
|
|
424
|
-
---
|
|
130
|
+
MIT License
|
|
@@ -5,42 +5,55 @@ async function main() {
|
|
|
5
5
|
const scraperToken = process.env.THORDATA_SCRAPER_TOKEN;
|
|
6
6
|
const publicToken = process.env.THORDATA_PUBLIC_TOKEN;
|
|
7
7
|
const publicKey = process.env.THORDATA_PUBLIC_KEY;
|
|
8
|
+
// These are NOT universal constants; users should copy them from Dashboard -> Web Scraper Store -> API Builder.
|
|
9
|
+
const spiderName = process.env.THORDATA_TASK_SPIDER_NAME;
|
|
10
|
+
const spiderId = process.env.THORDATA_TASK_SPIDER_ID;
|
|
11
|
+
const fileName = process.env.THORDATA_TASK_FILE_NAME || "{{TasksID}}";
|
|
12
|
+
const parametersJson = process.env.THORDATA_TASK_PARAMETERS_JSON || "{}";
|
|
8
13
|
if (!scraperToken || !publicToken || !publicKey) {
|
|
9
14
|
console.error("Please set THORDATA_SCRAPER_TOKEN, THORDATA_PUBLIC_TOKEN, THORDATA_PUBLIC_KEY in .env");
|
|
10
15
|
process.exit(1);
|
|
11
16
|
}
|
|
17
|
+
// If task-specific env vars are missing, skip the example to keep CI/offline e2e stable.
|
|
18
|
+
if (!spiderName || !spiderId) {
|
|
19
|
+
console.log("Skipping tasks example. Set THORDATA_TASK_SPIDER_NAME and THORDATA_TASK_SPIDER_ID to run it (copy from Dashboard -> Web Scraper Store -> API Builder).");
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
let parameters;
|
|
23
|
+
try {
|
|
24
|
+
parameters = JSON.parse(parametersJson);
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
console.error("THORDATA_TASK_PARAMETERS_JSON must be valid JSON");
|
|
28
|
+
process.exit(1);
|
|
29
|
+
}
|
|
12
30
|
const client = new ThordataClient({
|
|
13
31
|
scraperToken,
|
|
14
32
|
publicToken,
|
|
15
33
|
publicKey,
|
|
16
34
|
});
|
|
17
|
-
console.log("🕷️ Creating Web Scraper task (example
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
const downloadUrl = await client.getTaskResult(taskId, "json");
|
|
36
|
-
console.log("Download URL:", downloadUrl);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
catch (err) {
|
|
40
|
-
console.error("Error:", err);
|
|
35
|
+
console.log("🕷️ Creating Web Scraper task (live example)...");
|
|
36
|
+
const taskId = await client.createScraperTask({
|
|
37
|
+
// Recommended by docs: let the server substitute the actual task id.
|
|
38
|
+
fileName,
|
|
39
|
+
spiderId,
|
|
40
|
+
spiderName,
|
|
41
|
+
parameters,
|
|
42
|
+
});
|
|
43
|
+
console.log("Task created:", taskId);
|
|
44
|
+
console.log("⏱️ Waiting for task completion...");
|
|
45
|
+
const status = await client.waitForTask(taskId, {
|
|
46
|
+
pollIntervalMs: 5000,
|
|
47
|
+
maxWaitMs: 120_000,
|
|
48
|
+
});
|
|
49
|
+
console.log("Final status:", status);
|
|
50
|
+
if (status.toLowerCase() === "ready" || status.toLowerCase() === "success") {
|
|
51
|
+
const downloadUrl = await client.getTaskResult(taskId, "json");
|
|
52
|
+
console.log("Download URL:", downloadUrl);
|
|
41
53
|
}
|
|
42
54
|
}
|
|
43
55
|
main().catch((err) => {
|
|
44
56
|
console.error("Fatal error:", err);
|
|
57
|
+
process.exit(1);
|
|
45
58
|
});
|
|
46
59
|
//# sourceMappingURL=basic_scraper_task.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"basic_scraper_task.js","sourceRoot":"","sources":["../../examples/basic_scraper_task.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAEjC,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,KAAK,UAAU,IAAI;IACjB,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAElD,IAAI,CAAC,YAAY,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;QAChD,OAAO,CAAC,KAAK,CACX,uFAAuF,CACxF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC;QAChC,YAAY;QACZ,WAAW;QACX,SAAS;KACV,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;IAChE,
|
|
1
|
+
{"version":3,"file":"basic_scraper_task.js","sourceRoot":"","sources":["../../examples/basic_scraper_task.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAEjC,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,KAAK,UAAU,IAAI;IACjB,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAElD,gHAAgH;IAChH,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,aAAa,CAAC;IACtE,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,IAAI,CAAC;IAEzE,IAAI,CAAC,YAAY,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;QAChD,OAAO,CAAC,KAAK,CACX,uFAAuF,CACxF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yFAAyF;IACzF,IAAI,CAAC,UAAU,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CACT,wJAAwJ,CACzJ,CAAC;QACF,OAAO;IACT,CAAC;IAED,IAAI,UAAe,CAAC;IACpB,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC;QAChC,YAAY;QACZ,WAAW;QACX,SAAS;KACV,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC;QAC5C,qEAAqE;QACrE,QAAQ;QACR,QAAQ;QACR,UAAU;QACV,UAAU;KACX,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAErC,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE;QAC9C,cAAc,EAAE,IAAI;QACpB,SAAS,EAAE,OAAO;KACnB,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAErC,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE,CAAC;QAC3E,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -14,10 +14,10 @@ async function main() {
|
|
|
14
14
|
engine: Engine.GOOGLE,
|
|
15
15
|
num: 5,
|
|
16
16
|
});
|
|
17
|
-
//
|
|
17
|
+
// Debug: show actual response structure
|
|
18
18
|
console.dir(results, { depth: 4 });
|
|
19
|
-
//
|
|
20
|
-
const organic = results?.organic ?? results?.organic_results ?? [];
|
|
19
|
+
// Type-safe access with fallback
|
|
20
|
+
const organic = (results?.organic ?? results?.organic_results ?? []);
|
|
21
21
|
console.log(`Found ${organic.length} organic results`);
|
|
22
22
|
for (const item of organic.slice(0, 3)) {
|
|
23
23
|
console.log("-", item.title, "->", item.link);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"basic_serp.js","sourceRoot":"","sources":["../../examples/basic_serp.ts"],"names":[],"mappings":"AAAA,yBAAyB;AAEzB,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzD,KAAK,UAAU,IAAI;IACjB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACjD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC;QACtC,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,CAAC;KACP,CAAC,CAAC;IAEH,
|
|
1
|
+
{"version":3,"file":"basic_serp.js","sourceRoot":"","sources":["../../examples/basic_serp.ts"],"names":[],"mappings":"AAAA,yBAAyB;AAEzB,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzD,KAAK,UAAU,IAAI;IACjB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACjD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC;QACtC,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,CAAC;KACP,CAAC,CAAC;IAEH,wCAAwC;IACxC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEnC,iCAAiC;IACjC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,eAAe,IAAI,EAAE,CAGjE,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;IACvD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC/B,CAAC,CAAC,CAAC"}
|