xscrape 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -7,11 +7,13 @@ Effect Schema, allowing you to use your preferred validation tool.
|
|
|
7
7
|
|
|
8
8
|
## Features
|
|
9
9
|
|
|
10
|
-
- HTML Parsing
|
|
10
|
+
- **HTML Parsing**: Extract data from HTML using CSS selectors with the help of
|
|
11
11
|
[cheerio](https://github.com/cheeriojs/cheerio).
|
|
12
|
-
- Schema Validation
|
|
13
|
-
- Custom Transformations
|
|
14
|
-
- Default Values
|
|
12
|
+
- **Schema Validation**: Validate and transform extracted data with schema validation libraries like [Zod](https://github.com/colinhacks/zod).
|
|
13
|
+
- **Custom Transformations**: Provide custom transformations for extractedattributes.
|
|
14
|
+
- **Default Values**: Define default values for missing data fields.
|
|
15
|
+
- **Nested Field Support**: Define and extract nested data structures from
|
|
16
|
+
HTML elements.
|
|
15
17
|
|
|
16
18
|
### Schema Support
|
|
17
19
|
|
|
@@ -48,6 +50,14 @@ const schema = z.object({
|
|
|
48
50
|
description: z.string(),
|
|
49
51
|
keywords: z.array(z.string()),
|
|
50
52
|
views: z.number(),
|
|
53
|
+
image: z
|
|
54
|
+
.object({
|
|
55
|
+
url: z.string(),
|
|
56
|
+
width: z.number(),
|
|
57
|
+
height: z.number(),
|
|
58
|
+
})
|
|
59
|
+
.default({ url: '', width: 0, height: 0 })
|
|
60
|
+
.optional(),
|
|
51
61
|
});
|
|
52
62
|
```
|
|
53
63
|
|
|
@@ -78,6 +88,25 @@ const fields: FieldDefinitions = {
|
|
|
78
88
|
transform: (value) => parseInt(value, 10),
|
|
79
89
|
defaultValue: 0,
|
|
80
90
|
},
|
|
91
|
+
// Example of a nested field
|
|
92
|
+
image: {
|
|
93
|
+
fields: {
|
|
94
|
+
url: {
|
|
95
|
+
selector: 'meta[property="og:image"]',
|
|
96
|
+
attribute: 'content',
|
|
97
|
+
},
|
|
98
|
+
width: {
|
|
99
|
+
selector: 'meta[property="og:image:width"]',
|
|
100
|
+
attribute: 'content',
|
|
101
|
+
transform: (value) => parseInt(value, 10),
|
|
102
|
+
},
|
|
103
|
+
height: {
|
|
104
|
+
selector: 'meta[property="og:image:height"]',
|
|
105
|
+
attribute: 'content',
|
|
106
|
+
transform: (value) => parseInt(value, 10),
|
|
107
|
+
},
|
|
108
|
+
},
|
|
109
|
+
},
|
|
81
110
|
};
|
|
82
111
|
```
|
|
83
112
|
|
|
@@ -96,6 +125,9 @@ const html = `
|
|
|
96
125
|
<meta name="description" content="An example description.">
|
|
97
126
|
<meta name="keywords" content="typescript,html,parsing">
|
|
98
127
|
<meta name="views" content="1234">
|
|
128
|
+
<meta property="og:image" content="https://example.se/images/c12ffe73-3227-4a4a-b8ad-a3003cdf1d70?h=708&tight=false&w=1372">
|
|
129
|
+
<meta property="og:image:width" content="1372">
|
|
130
|
+
<meta property="og:image:height" content="708">
|
|
99
131
|
<title>Example Title</title>
|
|
100
132
|
</head>
|
|
101
133
|
<body></body>
|
|
@@ -111,6 +143,11 @@ console.log(data);
|
|
|
111
143
|
// description: 'An example description.',
|
|
112
144
|
// keywords: ['typescript', 'html', 'parsing'],
|
|
113
145
|
// views: 1234
|
|
146
|
+
// image: {
|
|
147
|
+
// url: 'https://example.se/images/c12ffe73-3227-4a4a-b8ad-a3003cdf1d70?h=708&tight=false&w=1372',
|
|
148
|
+
// width: 1372,
|
|
149
|
+
// height: 708
|
|
150
|
+
// }
|
|
114
151
|
// }
|
|
115
152
|
```
|
|
116
153
|
|
package/package.json
CHANGED