webcrawlerapi-js 2.0.5 → 2.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -4
- package/dist/@types/model.d.ts +5 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -27,7 +27,7 @@ In this mode, the promise will be resolved when all data is ready. Synchronous j
|
|
|
27
27
|
import webcrawlerapi from "webcrawlerapi-js"
|
|
28
28
|
|
|
29
29
|
async function main() {
|
|
30
|
-
const client = new WebcrawlerClient("YOUR API ACCESS KEY HERE")
|
|
30
|
+
const client = new webcrawlerapi.WebcrawlerClient("YOUR API ACCESS KEY HERE")
|
|
31
31
|
|
|
32
32
|
try {
|
|
33
33
|
const crawlJob = await client.crawl({
|
|
@@ -64,7 +64,7 @@ In this mode, you get a job ID immediately and can poll for results. Asynchronou
|
|
|
64
64
|
import webcrawlerapi from "webcrawlerapi-js"
|
|
65
65
|
|
|
66
66
|
async function main() {
|
|
67
|
-
const client = new WebcrawlerClient("YOUR API ACCESS KEY HERE")
|
|
67
|
+
const client = new webcrawlerapi.WebcrawlerClient("YOUR API ACCESS KEY HERE")
|
|
68
68
|
|
|
69
69
|
try {
|
|
70
70
|
try {
|
|
@@ -115,7 +115,7 @@ The SDK provides full TypeScript support with type safety and error handling. Ou
|
|
|
115
115
|
import {WebcrawlerClient, WebcrawlerApiError} from "webcrawlerapi-js";
|
|
116
116
|
|
|
117
117
|
async function main() {
|
|
118
|
-
const client = new WebcrawlerClient("YOUR_API_KEY")
|
|
118
|
+
const client = new webcrawlerapi.WebcrawlerClient("YOUR_API_KEY")
|
|
119
119
|
|
|
120
120
|
try {
|
|
121
121
|
// Basic crawl with markdown
|
|
@@ -152,7 +152,6 @@ Example of a job response. The response structure is carefully designed to provi
|
|
|
152
152
|
scrape_type: 'markdown',
|
|
153
153
|
whitelist_regexp: '',
|
|
154
154
|
blacklist_regexp: '',
|
|
155
|
-
allow_subdomains: false,
|
|
156
155
|
items_limit: 10,
|
|
157
156
|
created_at: '2024-12-28T21:36:04.417Z',
|
|
158
157
|
finished_at: null,
|
package/dist/@types/model.d.ts
CHANGED
|
@@ -5,8 +5,10 @@ export interface ScrapeRequest {
|
|
|
5
5
|
clean_selectors?: string;
|
|
6
6
|
actions?: Action[];
|
|
7
7
|
prompt?: string;
|
|
8
|
+
response_schema?: Record<string, any>;
|
|
8
9
|
respect_robots_txt?: boolean;
|
|
9
10
|
main_content_only?: boolean;
|
|
11
|
+
max_age?: number;
|
|
10
12
|
}
|
|
11
13
|
export interface ScrapeId {
|
|
12
14
|
id: string;
|
|
@@ -20,6 +22,7 @@ export interface ScrapeResponse {
|
|
|
20
22
|
page_status_code: number;
|
|
21
23
|
page_title?: string;
|
|
22
24
|
structured_data?: any;
|
|
25
|
+
links?: string[];
|
|
23
26
|
}
|
|
24
27
|
export interface ScrapeResponseError {
|
|
25
28
|
success: boolean;
|
|
@@ -37,12 +40,12 @@ export interface CrawlRequest {
|
|
|
37
40
|
scrape_type?: string;
|
|
38
41
|
whitelist_regexp?: string;
|
|
39
42
|
blacklist_regexp?: string;
|
|
40
|
-
allow_subdomains?: boolean;
|
|
41
43
|
webhook_url?: string;
|
|
42
44
|
actions?: Action[];
|
|
43
45
|
respect_robots_txt?: boolean;
|
|
44
46
|
main_content_only?: boolean;
|
|
45
47
|
max_depth?: number;
|
|
48
|
+
max_age?: number;
|
|
46
49
|
}
|
|
47
50
|
export interface Job {
|
|
48
51
|
id: string;
|
|
@@ -52,7 +55,6 @@ export interface Job {
|
|
|
52
55
|
scrape_type: string;
|
|
53
56
|
whitelist_regexp: string;
|
|
54
57
|
blacklist_regexp: string;
|
|
55
|
-
allow_subdomains: boolean;
|
|
56
58
|
items_limit: number;
|
|
57
59
|
max_depth?: number;
|
|
58
60
|
created_at: string;
|
|
@@ -78,6 +80,7 @@ export interface JobItem {
|
|
|
78
80
|
depth?: number;
|
|
79
81
|
raw_content_url?: string;
|
|
80
82
|
cleaned_content_url?: string;
|
|
83
|
+
error_code?: string;
|
|
81
84
|
getContent(): Promise<string | null>;
|
|
82
85
|
}
|
|
83
86
|
export interface Action {
|