thordata-js-sdk 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/LICENSE +20 -20
  2. package/README.md +55 -349
  3. package/dist/examples/basic_scraper_task.js +37 -24
  4. package/dist/examples/basic_scraper_task.js.map +1 -1
  5. package/dist/examples/basic_serp.js +3 -3
  6. package/dist/examples/basic_serp.js.map +1 -1
  7. package/dist/examples/location_api.js +47 -0
  8. package/dist/examples/location_api.js.map +1 -0
  9. package/dist/examples/proxy_datacenter.js +1 -1
  10. package/dist/examples/proxy_datacenter.js.map +1 -1
  11. package/dist/examples/proxy_isp.js +1 -1
  12. package/dist/examples/proxy_isp.js.map +1 -1
  13. package/dist/examples/proxy_mobile.js +1 -1
  14. package/dist/examples/proxy_mobile.js.map +1 -1
  15. package/dist/examples/proxy_residential.js +2 -2
  16. package/dist/examples/proxy_residential.js.map +1 -1
  17. package/dist/examples/serp_google_news.d.ts +0 -6
  18. package/dist/examples/serp_google_news.js +7 -10
  19. package/dist/examples/serp_google_news.js.map +1 -1
  20. package/dist/examples/verify_new_features.js +135 -0
  21. package/dist/examples/verify_new_features.js.map +1 -0
  22. package/dist/src/auth.d.ts +22 -0
  23. package/dist/src/auth.js +39 -0
  24. package/dist/src/auth.js.map +1 -0
  25. package/dist/src/client.d.ts +144 -6
  26. package/dist/src/client.js +396 -43
  27. package/dist/src/client.js.map +1 -1
  28. package/dist/src/endpoints.js +2 -2
  29. package/dist/src/endpoints.js.map +1 -1
  30. package/dist/src/enums.d.ts +219 -3
  31. package/dist/src/enums.js +289 -2
  32. package/dist/src/enums.js.map +1 -1
  33. package/dist/src/errors.d.ts +37 -5
  34. package/dist/src/errors.js +61 -17
  35. package/dist/src/errors.js.map +1 -1
  36. package/dist/src/index.d.ts +7 -5
  37. package/dist/src/index.js +21 -5
  38. package/dist/src/index.js.map +1 -1
  39. package/dist/src/models.d.ts +196 -12
  40. package/dist/src/models.js +1 -0
  41. package/dist/src/models.js.map +1 -1
  42. package/dist/src/proxy.d.ts +59 -0
  43. package/dist/src/proxy.js +83 -11
  44. package/dist/src/proxy.js.map +1 -1
  45. package/dist/src/retry.d.ts +25 -3
  46. package/dist/src/retry.js +16 -0
  47. package/dist/src/retry.js.map +1 -1
  48. package/dist/src/thordata.d.ts +5 -2
  49. package/dist/src/thordata.js +36 -6
  50. package/dist/src/thordata.js.map +1 -1
  51. package/dist/src/utils.d.ts +18 -13
  52. package/dist/src/utils.js +94 -43
  53. package/dist/src/utils.js.map +1 -1
  54. package/package.json +6 -6
  55. package/dist/examples/proxy_demo.js +0 -69
  56. package/dist/examples/proxy_demo.js.map +0 -1
  57. package/dist/examples/proxy_ip_check.js +0 -37
  58. package/dist/examples/proxy_ip_check.js.map +0 -1
  59. /package/dist/examples/{proxy_demo.d.ts → location_api.d.ts} +0 -0
  60. /package/dist/examples/{proxy_ip_check.d.ts → verify_new_features.d.ts} +0 -0
package/LICENSE CHANGED
@@ -1,21 +1,21 @@
1
- MIT License
2
-
3
- Copyright (c) 2025 Thordata
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy
6
- of this software and associated documentation files (the "Software"), to deal
7
- in the Software without restriction, including without limitation the rights
8
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
- copies of the Software, and to permit persons to whom the Software is
10
- furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in all
13
- copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Thordata
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
21
  SOFTWARE.
package/README.md CHANGED
@@ -1,18 +1,17 @@
1
1
  # Thordata JS SDK (Node.js / TypeScript)
2
2
 
3
- Official JavaScript/TypeScript SDK for <!--citation:1-->.
3
+ Official JavaScript/TypeScript SDK for Thordata APIs.
4
4
 
5
- This SDK supports:
5
+ [![npm version](https://img.shields.io/npm/v/thordata-js-sdk.svg)](https://www.npmjs.com/package/thordata-js-sdk)
6
+ [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
6
7
 
7
- - **SERP API** (Google / Bing / Yandex / DuckDuckGo / Baidu)
8
- - **Web Unlocker / Universal API**
9
- - **Web Scraper API** (task-based scraping)
8
+ Supports:
10
9
 
11
- It is designed to be:
12
-
13
- - **TypeScript-first**
14
- - **ESM-ready**
15
- - **Offline-test friendly** (base URLs can be overridden to run examples/tests against a mock server)
10
+ - **SERP API** (Google / Bing / Yandex)
11
+ - **Web Unlocker** (Universal API)
12
+ - **Web Scraper API** (Text & Video Tasks)
13
+ - **Proxy Network** (Residential / Datacenter / Mobile / ISP)
14
+ - **Account Management** (Usage, Users, Whitelist)
16
15
 
17
16
  ---
18
17
 
@@ -22,403 +21,110 @@ It is designed to be:
22
21
  npm install thordata-js-sdk
23
22
  ```
24
23
 
25
- If you are developing locally:
26
-
27
- ```bash
28
- git clone https://github.com/Thordata/thordata-js-sdk.git
29
- cd thordata-js-sdk
30
- npm install
31
- npm run build
32
- ```
33
-
34
24
  ---
35
25
 
36
26
  ## 🔐 Configuration
37
27
 
38
- Set environment variables:
39
-
40
28
  ```bash
41
- export THORDATA_SCRAPER_TOKEN=your_scraper_token
29
+ export THORDATA_SCRAPER_TOKEN=your_token
42
30
  export THORDATA_PUBLIC_TOKEN=your_public_token
43
31
  export THORDATA_PUBLIC_KEY=your_public_key
44
32
  ```
45
33
 
46
- Or create a `.env` file (examples may use dotenv):
47
-
48
- ```env
49
- THORDATA_SCRAPER_TOKEN=your_scraper_token
50
- THORDATA_PUBLIC_TOKEN=your_public_token
51
- THORDATA_PUBLIC_KEY=your_public_key
52
- ```
53
-
54
34
  ---
55
35
 
56
36
  ## 🚀 Quick Start
57
37
 
58
- ### Create a client
59
-
60
- ```typescript
61
- import { ThordataClient } from "thordata-js-sdk";
62
-
63
- const client = new ThordataClient({
64
- scraperToken: process.env.THORDATA_SCRAPER_TOKEN!,
65
- publicToken: process.env.THORDATA_PUBLIC_TOKEN,
66
- publicKey: process.env.THORDATA_PUBLIC_KEY,
67
- });
68
- ```
69
-
70
- ---
71
-
72
- ## 🔍 SERP API
73
-
74
- ### Basic Google Search
75
-
76
- ```typescript
77
- import { ThordataClient, Engine } from "thordata-js-sdk";
78
-
79
- const client = new ThordataClient({ scraperToken: process.env.THORDATA_SCRAPER_TOKEN! });
80
-
81
- const data = await client.serpSearch({
82
- query: "Thordata proxy network",
83
- engine: Engine.GOOGLE,
84
- num: 5,
85
- });
86
-
87
- const organic = data?.organic ?? [];
88
- console.log(`Found ${organic.length} organic results`);
89
-
90
- for (const item of organic.slice(0, 3)) {
91
- console.log("-", item.title, "->", item.link);
92
- }
93
- ```
94
-
95
- ### Recommended engines for Google verticals (News / Shopping)
96
-
97
- Thordata supports both:
98
-
99
- - Dedicated engines (recommended): `google_news`, `google_shopping`
100
- - Generic Google + tbm via `searchType` (alternative)
101
-
102
- #### Google News (recommended):
103
-
104
- ```typescript
105
- const data = await client.serpSearch({
106
- query: "AI regulation",
107
- engine: "google_news",
108
- country: "us",
109
- language: "en",
110
- num: 10,
111
- so: 1, // 0=relevance, 1=date (Google News)
112
- });
113
- ```
114
-
115
- #### Google Shopping (recommended):
116
-
117
38
  ```typescript
118
- const data = await client.serpSearch({
119
- query: "iPhone 15",
120
- engine: "google_shopping",
121
- country: "us",
122
- language: "en",
123
- num: 10,
124
- min_price: 500,
125
- max_price: 1500,
126
- });
127
- ```
39
+ import { Thordata } from "thordata-js-sdk";
128
40
 
129
- #### Alternative: Google generic engine + tbm (via searchType):
41
+ const client = new Thordata(); // Reads from env vars
130
42
 
131
- ```typescript
132
- const data = await client.serpSearch({
133
- query: "iPhone 15",
43
+ // SERP Search
44
+ const results = await client.serpSearch({
45
+ query: "Thordata SDK",
134
46
  engine: "google",
135
- searchType: "shopping", // maps to tbm=shop
136
47
  country: "us",
137
- language: "en",
138
- num: 10,
139
48
  });
140
- ```
141
-
142
- Official and up-to-date parameters are documented at: https://doc.thordata.com
143
-
144
- ---
145
-
146
- ## 🌐 Proxy Network
147
-
148
- Each proxy product requires separate credentials from Thordata Dashboard.
149
-
150
- ### Environment Variables
151
-
152
- ```env
153
- # Residential Proxy (port 9999)
154
- THORDATA_RESIDENTIAL_USERNAME=your_residential_username
155
- THORDATA_RESIDENTIAL_PASSWORD=your_residential_password
156
-
157
- # Datacenter Proxy (port 7777)
158
- THORDATA_DATACENTER_USERNAME=your_datacenter_username
159
- THORDATA_DATACENTER_PASSWORD=your_datacenter_password
160
-
161
- # Mobile Proxy (port 5555)
162
- THORDATA_MOBILE_USERNAME=your_mobile_username
163
- THORDATA_MOBILE_PASSWORD=your_mobile_password
164
-
165
- # Static ISP Proxy (port 6666, direct IP connection)
166
- THORDATA_ISP_HOST=your_static_ip_address
167
- THORDATA_ISP_USERNAME=your_isp_username
168
- THORDATA_ISP_PASSWORD=your_isp_password
169
- ```
170
-
171
- ### Residential Proxy
172
-
173
- ```typescript
174
- import { Thordata } from "thordata-js-sdk";
175
-
176
- const client = new Thordata();
177
-
178
- // Basic US residential
179
- const proxy = Thordata.Proxy.residentialFromEnv().country("us");
180
- const result = await client.request("http://httpbin.org/ip", { proxy });
181
- console.log("IP:", result.origin);
182
-
183
- // Sticky session (same IP for 30 minutes)
184
- const stickyProxy = Thordata.Proxy.residentialFromEnv()
185
- .country("jp")
186
- .city("tokyo")
187
- .session("my_session")
188
- .sticky(30);
189
- const result2 = await client.request("http://httpbin.org/ip", { proxy: stickyProxy });
190
- ```
191
-
192
- ### Datacenter Proxy
193
-
194
- ```typescript
195
- const proxy = Thordata.Proxy.datacenterFromEnv();
196
- const result = await client.request("http://httpbin.org/ip", { proxy });
197
- console.log("Datacenter IP:", result.origin);
198
- ```
199
-
200
- ### Mobile Proxy
201
-
202
- ```typescript
203
- const proxy = Thordata.Proxy.mobileFromEnv().country("gb");
204
- const result = await client.request("http://httpbin.org/ip", { proxy });
205
- console.log("UK Mobile IP:", result.origin);
206
- ```
207
-
208
- ### Static ISP Proxy
209
-
210
- ```typescript
211
- const proxy = Thordata.Proxy.ispFromEnv();
212
- const result = await client.request("http://httpbin.org/ip", { proxy });
213
- console.log("Static ISP IP:", result.origin);
214
- // Returns your purchased static IP address
215
- ```
216
-
217
- ### Proxy Examples
218
-
219
- ```bash
220
- node dist/examples/proxy_residential.js
221
- node dist/examples/proxy_datacenter.js
222
- node dist/examples/proxy_mobile.js
223
- node dist/examples/proxy_isp.js
49
+ console.log(results.organic?.[0]?.link);
224
50
  ```
225
51
 
226
52
  ---
227
53
 
228
- ## 🔓 Web Unlocker / Universal API
54
+ ## 📖 Features
229
55
 
230
- ### Basic HTML scraping
56
+ ### SERP API
231
57
 
232
58
  ```typescript
233
- const html = await client.universalScrape({
234
- url: "https://httpbin.org/html",
235
- jsRender: false,
236
- outputFormat: "html",
59
+ const news = await client.serpSearch({
60
+ query: "AI News",
61
+ engine: "google_news",
62
+ num: 10,
237
63
  });
238
-
239
- console.log(String(html).slice(0, 300));
240
64
  ```
241
65
 
242
- ### JS rendering + wait for selector
66
+ ### Web Unlocker (Universal)
243
67
 
244
68
  ```typescript
245
69
  const html = await client.universalScrape({
246
- url: "https://example.com/spa",
247
- jsRender: true,
248
- outputFormat: "html",
249
- waitFor: ".main-content",
250
- });
251
- ```
252
-
253
- ### Screenshot (PNG)
254
-
255
- ```typescript
256
- import { writeFileSync } from "node:fs";
257
-
258
- const pngBytes = await client.universalScrape({
259
70
  url: "https://example.com",
260
71
  jsRender: true,
261
- outputFormat: "png",
72
+ waitFor: ".content",
262
73
  });
263
-
264
- writeFileSync("screenshot.png", pngBytes as Buffer);
265
74
  ```
266
75
 
267
- ---
268
-
269
- ## 🕷️ Web Scraper API (Task-based)
270
-
271
- Requires `THORDATA_PUBLIC_TOKEN` and `THORDATA_PUBLIC_KEY`.
76
+ ### Web Scraper API (Async)
272
77
 
273
78
  ```typescript
274
- const client = new ThordataClient({
275
- scraperToken: process.env.THORDATA_SCRAPER_TOKEN!,
276
- publicToken: process.env.THORDATA_PUBLIC_TOKEN,
277
- publicKey: process.env.THORDATA_PUBLIC_KEY,
278
- });
279
-
79
+ // Create Task
280
80
  const taskId = await client.createScraperTask({
281
- fileName: "demo_task",
282
- spiderId: "example-spider-id",
283
- spiderName: "example.com",
81
+ fileName: "task1",
82
+ spiderId: "universal",
83
+ spiderName: "universal",
284
84
  parameters: { url: "https://example.com" },
285
85
  });
286
86
 
287
- console.log("Task created:", taskId);
288
-
289
- const status = await client.waitForTask(taskId, {
290
- pollIntervalMs: 5000,
291
- maxWaitMs: 60000,
87
+ // Video Task (New)
88
+ const vidId = await client.createVideoTask({
89
+ fileName: "video1",
90
+ spiderId: "youtube_video_by-url",
91
+ spiderName: "youtube.com",
92
+ parameters: { url: "..." },
93
+ commonSettings: { resolution: "1080p" },
292
94
  });
293
95
 
294
- console.log("Final status:", status);
295
-
296
- if (["ready", "success", "finished"].includes(status.toLowerCase())) {
297
- const downloadUrl = await client.getTaskResult(taskId, "json");
298
- console.log("Download URL:", downloadUrl);
96
+ // Wait & Result
97
+ const status = await client.waitForTask(taskId);
98
+ if (status === "ready") {
99
+ const url = await client.getTaskResult(taskId);
100
+ console.log(url);
299
101
  }
300
102
  ```
301
103
 
302
- ---
303
-
304
- ## 🔧 Errors & Response Codes
305
-
306
- The SDK throws typed errors when the API returns a non-success code (or non-2xx HTTP status).
307
-
308
- | Code | Typical Meaning | Error class |
309
- | ------- | --------------------- | ---------------------------------------------- |
310
- | 200 | Success | - |
311
- | 300 | Not collected | `ThordataNotCollectedError` |
312
- | 400 | Bad request | `ThordataValidationError` |
313
- | 401/403 | Auth/Forbidden | `ThordataAuthError` |
314
- | 402/429 | Quota/Rate limit | `ThordataRateLimitError` |
315
- | 5xx | Server/timeout issues | `ThordataServerError` / `ThordataTimeoutError` |
316
-
317
- ### Example error handling:
104
+ ### Account Management
318
105
 
319
106
  ```typescript
320
- import {
321
- ThordataAuthError,
322
- ThordataRateLimitError,
323
- ThordataTimeoutError,
324
- ThordataNotCollectedError,
325
- } from "thordata-js-sdk";
326
-
327
- try {
328
- const data = await client.serpSearch({ query: "test", engine: "google" });
329
- console.log(data);
330
- } catch (e) {
331
- if (e instanceof ThordataAuthError) {
332
- console.error("Auth error: check your token.");
333
- } else if (e instanceof ThordataRateLimitError) {
334
- console.error(`Rate limited. Retry after: ${e.retryAfter ?? "N/A"} seconds.`);
335
- } else if (e instanceof ThordataNotCollectedError) {
336
- console.error("Not collected (code=300). Consider retrying.");
337
- } else if (e instanceof ThordataTimeoutError) {
338
- console.error("Request timed out.");
339
- } else {
340
- console.error("Unexpected error:", e);
341
- }
342
- }
343
- ```
107
+ // Usage Stats
108
+ const stats = await client.getUsageStatistics("2024-01-01", "2024-01-31");
109
+ console.log("Balance:", stats.traffic_balance);
344
110
 
345
- ---
346
-
347
- ## 🌍 Base URL Overrides (for offline tests / custom routing)
111
+ // Proxy Users
112
+ const users = await client.listProxyUsers("residential");
348
113
 
349
- You can override API base URLs via environment variables:
350
-
351
- ```bash
352
- export THORDATA_SCRAPERAPI_BASE_URL=http://127.0.0.1:12345
353
- export THORDATA_UNIVERSALAPI_BASE_URL=http://127.0.0.1:12345
354
- export THORDATA_WEB_SCRAPER_API_BASE_URL=http://127.0.0.1:12345
355
- export THORDATA_LOCATIONS_BASE_URL=http://127.0.0.1:12345
114
+ // Whitelist
115
+ await client.addWhitelistIp("1.2.3.4");
356
116
  ```
357
117
 
358
- Or via client config:
118
+ ### Proxy Configuration
359
119
 
360
120
  ```typescript
361
- const client = new ThordataClient({
362
- scraperToken: "dummy",
363
- baseUrls: { scraperapiBaseUrl: "http://127.0.0.1:12345" },
364
- });
365
- ```
366
-
367
- ---
368
-
369
- ## 🧪 Development
370
-
371
- ```bash
372
- npm install
373
- npm run build
374
- npm test
375
- ```
376
-
377
- ### Run examples (compiled):
378
-
379
- ```bash
380
- node dist/examples/basic_serp.js
381
- node dist/examples/basic_universal.js
382
- ```
383
-
384
- ---
385
-
386
- ## 📁 Project Structure
387
-
388
- ```
389
- thordata-js-sdk/
390
- ├── src/
391
- │ ├── index.ts
392
- │ ├── client.ts
393
- │ ├── models.ts
394
- │ ├── enums.ts
395
- │ ├── errors.ts
396
- │ ├── retry.ts
397
- │ ├── endpoints.ts
398
- │ └── utils.ts
399
- ├── examples/
400
- │ ├── basic_serp.ts
401
- │ ├── basic_universal.ts
402
- │ ├── basic_scraper_task.ts
403
- │ └── serp_google_news.ts
404
- ├── tests/
405
- │ ├── serp.offline.test.ts
406
- │ ├── mockServer.ts
407
- │ └── examples.e2e.test.ts
408
- ├── .github/workflows/ci.yml
409
- ├── package.json
410
- ├── tsconfig.json
411
- ├── tsconfig.build.json
412
- └── README.md
121
+ // Residential Proxy
122
+ const proxy = Thordata.Proxy.residentialFromEnv().country("us");
123
+ await client.request("https://httpbin.org/ip", { proxy });
413
124
  ```
414
125
 
415
126
  ---
416
127
 
417
- ## 🔮 Roadmap
128
+ ## 📄 License
418
129
 
419
- - Publish stable releases to npm
420
- - Add async streaming / higher-level helpers for AI agents
421
- - Expand coverage for more engines/verticals (Flights/Maps/Scholar/Jobs, etc.)
422
- - Add integration tests (optional scheduled job with real tokens)
423
-
424
- ---
130
+ MIT License
@@ -5,42 +5,55 @@ async function main() {
5
5
  const scraperToken = process.env.THORDATA_SCRAPER_TOKEN;
6
6
  const publicToken = process.env.THORDATA_PUBLIC_TOKEN;
7
7
  const publicKey = process.env.THORDATA_PUBLIC_KEY;
8
+ // These are NOT universal constants; users should copy them from Dashboard -> Web Scraper Store -> API Builder.
9
+ const spiderName = process.env.THORDATA_TASK_SPIDER_NAME;
10
+ const spiderId = process.env.THORDATA_TASK_SPIDER_ID;
11
+ const fileName = process.env.THORDATA_TASK_FILE_NAME || "{{TasksID}}";
12
+ const parametersJson = process.env.THORDATA_TASK_PARAMETERS_JSON || "{}";
8
13
  if (!scraperToken || !publicToken || !publicKey) {
9
14
  console.error("Please set THORDATA_SCRAPER_TOKEN, THORDATA_PUBLIC_TOKEN, THORDATA_PUBLIC_KEY in .env");
10
15
  process.exit(1);
11
16
  }
17
+ // If task-specific env vars are missing, skip the example to keep CI/offline e2e stable.
18
+ if (!spiderName || !spiderId) {
19
+ console.log("Skipping tasks example. Set THORDATA_TASK_SPIDER_NAME and THORDATA_TASK_SPIDER_ID to run it (copy from Dashboard -> Web Scraper Store -> API Builder).");
20
+ return;
21
+ }
22
+ let parameters;
23
+ try {
24
+ parameters = JSON.parse(parametersJson);
25
+ }
26
+ catch {
27
+ console.error("THORDATA_TASK_PARAMETERS_JSON must be valid JSON");
28
+ process.exit(1);
29
+ }
12
30
  const client = new ThordataClient({
13
31
  scraperToken,
14
32
  publicToken,
15
33
  publicKey,
16
34
  });
17
- console.log("🕷️ Creating Web Scraper task (example only)...");
18
- try {
19
- const taskId = await client.createScraperTask({
20
- fileName: "demo_task",
21
- spiderId: "example-spider-id",
22
- spiderName: "example.com",
23
- parameters: {
24
- url: "https://example.com",
25
- },
26
- });
27
- console.log("Task created:", taskId);
28
- console.log("⏱️ Waiting for task completion...");
29
- const status = await client.waitForTask(taskId, {
30
- pollIntervalMs: 5000,
31
- maxWaitMs: 60_000,
32
- });
33
- console.log("Final status:", status);
34
- if (status.toLowerCase() === "ready" || status.toLowerCase() === "success") {
35
- const downloadUrl = await client.getTaskResult(taskId, "json");
36
- console.log("Download URL:", downloadUrl);
37
- }
38
- }
39
- catch (err) {
40
- console.error("Error:", err);
35
+ console.log("🕷️ Creating Web Scraper task (live example)...");
36
+ const taskId = await client.createScraperTask({
37
+ // Recommended by docs: let the server substitute the actual task id.
38
+ fileName,
39
+ spiderId,
40
+ spiderName,
41
+ parameters,
42
+ });
43
+ console.log("Task created:", taskId);
44
+ console.log("⏱️ Waiting for task completion...");
45
+ const status = await client.waitForTask(taskId, {
46
+ pollIntervalMs: 5000,
47
+ maxWaitMs: 120_000,
48
+ });
49
+ console.log("Final status:", status);
50
+ if (status.toLowerCase() === "ready" || status.toLowerCase() === "success") {
51
+ const downloadUrl = await client.getTaskResult(taskId, "json");
52
+ console.log("Download URL:", downloadUrl);
41
53
  }
42
54
  }
43
55
  main().catch((err) => {
44
56
  console.error("Fatal error:", err);
57
+ process.exit(1);
45
58
  });
46
59
  //# sourceMappingURL=basic_scraper_task.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"basic_scraper_task.js","sourceRoot":"","sources":["../../examples/basic_scraper_task.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAEjC,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,KAAK,UAAU,IAAI;IACjB,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAElD,IAAI,CAAC,YAAY,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;QAChD,OAAO,CAAC,KAAK,CACX,uFAAuF,CACxF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC;QAChC,YAAY;QACZ,WAAW;QACX,SAAS;KACV,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;IAChE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC;YAC5C,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,mBAAmB;YAC7B,UAAU,EAAE,aAAa;YACzB,UAAU,EAAE;gBACV,GAAG,EAAE,qBAAqB;aAC3B;SACF,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;QAErC,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE;YAC9C,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,MAAM;SAClB,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;QAErC,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE,CAAC;YAC3E,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC/D,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;AACrC,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"basic_scraper_task.js","sourceRoot":"","sources":["../../examples/basic_scraper_task.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAEjC,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,KAAK,UAAU,IAAI;IACjB,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAElD,gHAAgH;IAChH,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,aAAa,CAAC;IACtE,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,IAAI,CAAC;IAEzE,IAAI,CAAC,YAAY,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;QAChD,OAAO,CAAC,KAAK,CACX,uFAAuF,CACxF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yFAAyF;IACzF,IAAI,CAAC,UAAU,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CACT,wJAAwJ,CACzJ,CAAC;QACF,OAAO;IACT,CAAC;IAED,IAAI,UAAe,CAAC;IACpB,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC;QAChC,YAAY;QACZ,WAAW;QACX,SAAS;KACV,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC;QAC5C,qEAAqE;QACrE,QAAQ;QACR,QAAQ;QACR,UAAU;QACV,UAAU;KACX,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAErC,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE;QAC9C,cAAc,EAAE,IAAI;QACpB,SAAS,EAAE,OAAO;KACnB,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAErC,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE,CAAC;QAC3E,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
@@ -14,10 +14,10 @@ async function main() {
14
14
  engine: Engine.GOOGLE,
15
15
  num: 5,
16
16
  });
17
- // 调试用:看真实返回结构
17
+ // Debug: show actual response structure
18
18
  console.dir(results, { depth: 4 });
19
- // 优先找 'organic',其次才是 'organic_results'
20
- const organic = results?.organic ?? results?.organic_results ?? [];
19
+ // Type-safe access with fallback
20
+ const organic = (results?.organic ?? results?.organic_results ?? []);
21
21
  console.log(`Found ${organic.length} organic results`);
22
22
  for (const item of organic.slice(0, 3)) {
23
23
  console.log("-", item.title, "->", item.link);
@@ -1 +1 @@
1
- {"version":3,"file":"basic_serp.js","sourceRoot":"","sources":["../../examples/basic_serp.ts"],"names":[],"mappings":"AAAA,yBAAyB;AAEzB,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzD,KAAK,UAAU,IAAI;IACjB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACjD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC;QACtC,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,CAAC;KACP,CAAC,CAAC;IAEH,cAAc;IACd,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEnC,uCAAuC;IACvC,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,eAAe,IAAI,EAAE,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;IACvD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC/B,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"basic_serp.js","sourceRoot":"","sources":["../../examples/basic_serp.ts"],"names":[],"mappings":"AAAA,yBAAyB;AAEzB,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzD,KAAK,UAAU,IAAI;IACjB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACjD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC;QACtC,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,CAAC;KACP,CAAC,CAAC;IAEH,wCAAwC;IACxC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEnC,iCAAiC;IACjC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,eAAe,IAAI,EAAE,CAGjE,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;IACvD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC/B,CAAC,CAAC,CAAC"}