xscrape 1.0.0 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/README.md +41 -4
  2. package/dist/index.cjs +23 -11
  3. package/dist/index.d.cts +5 -1
  4. package/dist/index.d.ts +5 -1
  5. package/dist/index.js +23 -11
  6. package/package.json +1 -1
package/README.md CHANGED
@@ -7,11 +7,13 @@ Effect Schema, allowing you to use your preferred validation tool.
7
7
 
8
8
  ## Features
9
9
 
10
- - HTML Parsing: Extract data from HTML using CSS selectors with the help of
10
+ - **HTML Parsing**: Extract data from HTML using CSS selectors with the help of
11
11
  [cheerio](https://github.com/cheeriojs/cheerio).
12
- - Schema Validation: Validate and transform extracted data with schema validation libraries like [Zod](https://github.com/colinhacks/zod).
13
- - Custom Transformations: Provide custom transformations for extractedattributes.
14
- - Default Values: Define default values for missing data fields.
12
+ - **Schema Validation**: Validate and transform extracted data with schema validation libraries like [Zod](https://github.com/colinhacks/zod).
13
+ - **Custom Transformations**: Provide custom transformations for extractedattributes.
14
+ - **Default Values**: Define default values for missing data fields.
15
+ - **Nested Field Support**: Define and extract nested data structures from
16
+ HTML elements.
15
17
 
16
18
  ### Schema Support
17
19
 
@@ -48,6 +50,14 @@ const schema = z.object({
48
50
  description: z.string(),
49
51
  keywords: z.array(z.string()),
50
52
  views: z.number(),
53
+ image: z
54
+ .object({
55
+ url: z.string(),
56
+ width: z.number(),
57
+ height: z.number(),
58
+ })
59
+ .default({ url: '', width: 0, height: 0 })
60
+ .optional(),
51
61
  });
52
62
  ```
53
63
 
@@ -78,6 +88,25 @@ const fields: FieldDefinitions = {
78
88
  transform: (value) => parseInt(value, 10),
79
89
  defaultValue: 0,
80
90
  },
91
+ // Example of a nested field
92
+ image: {
93
+ fields: {
94
+ url: {
95
+ selector: 'meta[property="og:image"]',
96
+ attribute: 'content',
97
+ },
98
+ width: {
99
+ selector: 'meta[property="og:image:width"]',
100
+ attribute: 'content',
101
+ transform: (value) => parseInt(value, 10),
102
+ },
103
+ height: {
104
+ selector: 'meta[property="og:image:height"]',
105
+ attribute: 'content',
106
+ transform: (value) => parseInt(value, 10),
107
+ },
108
+ },
109
+ },
81
110
  };
82
111
  ```
83
112
 
@@ -96,6 +125,9 @@ const html = `
96
125
  <meta name="description" content="An example description.">
97
126
  <meta name="keywords" content="typescript,html,parsing">
98
127
  <meta name="views" content="1234">
128
+ <meta property="og:image" content="https://example.se/images/c12ffe73-3227-4a4a-b8ad-a3003cdf1d70?h=708&amp;tight=false&amp;w=1372">
129
+ <meta property="og:image:width" content="1372">
130
+ <meta property="og:image:height" content="708">
99
131
  <title>Example Title</title>
100
132
  </head>
101
133
  <body></body>
@@ -111,6 +143,11 @@ console.log(data);
111
143
  // description: 'An example description.',
112
144
  // keywords: ['typescript', 'html', 'parsing'],
113
145
  // views: 1234
146
+ // image: {
147
+ // url: 'https://example.se/images/c12ffe73-3227-4a4a-b8ad-a3003cdf1d70?h=708&amp;tight=false&amp;w=1372',
148
+ // width: 1372,
149
+ // height: 708
150
+ // }
114
151
  // }
115
152
  ```
116
153
 
package/dist/index.cjs CHANGED
@@ -37,19 +37,21 @@ module.exports = __toCommonJS(src_exports);
37
37
 
38
38
  // src/createScraper.ts
39
39
  var cheerio = __toESM(require("cheerio"), 1);
40
- var createScraper = ({
41
- fields,
42
- validator
43
- }) => {
44
- return (html) => {
45
- const $ = cheerio.load(html);
46
- const data = {};
47
- for (const key in fields) {
48
- const fieldDef = fields[key];
49
- const elements = $(fieldDef.selector);
40
+ var extractData = (fields, $context) => {
41
+ const data = {};
42
+ for (const key in fields) {
43
+ const fieldDef = fields[key];
44
+ if ("fields" in fieldDef) {
45
+ const nestedData = extractData(
46
+ fieldDef.fields,
47
+ $context
48
+ );
49
+ data[key] = nestedData;
50
+ } else {
51
+ const elements = $context(fieldDef.selector);
50
52
  let values = [];
51
53
  elements.each((_, element) => {
52
- const value = fieldDef.attribute ? $(element).attr(fieldDef.attribute) : $(element).text();
54
+ const value = fieldDef.attribute ? $context(element).attr(fieldDef.attribute) : $context(element).text().trim();
53
55
  if (value !== void 0) {
54
56
  values.push(value);
55
57
  }
@@ -65,6 +67,16 @@ var createScraper = ({
65
67
  data[key] = fieldDef.transform && value ? fieldDef.transform(value) : value;
66
68
  }
67
69
  }
70
+ }
71
+ return data;
72
+ };
73
+ var createScraper = ({
74
+ fields,
75
+ validator
76
+ }) => {
77
+ return (html) => {
78
+ const $ = typeof html === "string" ? cheerio.load(html) : html;
79
+ const data = extractData(fields, $);
68
80
  return validator.validate(data);
69
81
  };
70
82
  };
package/dist/index.d.cts CHANGED
@@ -1,3 +1,4 @@
1
+ import * as cheerio from 'cheerio';
1
2
  import { ZodSchema } from 'zod';
2
3
 
3
4
  type ScrapeConfig<T> = {
@@ -10,6 +11,9 @@ type FieldDefinition<T> = {
10
11
  transform?: (value: string) => T;
11
12
  defaultValue?: T;
12
13
  multiple?: boolean;
14
+ } | NestedFieldDefinition<T>;
15
+ type NestedFieldDefinition<T> = {
16
+ fields: SchemaFieldDefinitions<T>;
13
17
  };
14
18
  type SchemaFieldDefinitions<T> = {
15
19
  [K in keyof T]: FieldDefinition<T[K]>;
@@ -18,7 +22,7 @@ interface SchemaValidator<T> {
18
22
  validate(data: unknown): T;
19
23
  }
20
24
 
21
- declare const createScraper: <T>({ fields, validator, }: ScrapeConfig<T>) => ((html: string) => T);
25
+ declare const createScraper: <T>({ fields, validator, }: ScrapeConfig<T>) => ((html: cheerio.CheerioAPI | string) => T);
22
26
 
23
27
  declare class ZodValidator<T> implements SchemaValidator<T> {
24
28
  private schema;
package/dist/index.d.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import * as cheerio from 'cheerio';
1
2
  import { ZodSchema } from 'zod';
2
3
 
3
4
  type ScrapeConfig<T> = {
@@ -10,6 +11,9 @@ type FieldDefinition<T> = {
10
11
  transform?: (value: string) => T;
11
12
  defaultValue?: T;
12
13
  multiple?: boolean;
14
+ } | NestedFieldDefinition<T>;
15
+ type NestedFieldDefinition<T> = {
16
+ fields: SchemaFieldDefinitions<T>;
13
17
  };
14
18
  type SchemaFieldDefinitions<T> = {
15
19
  [K in keyof T]: FieldDefinition<T[K]>;
@@ -18,7 +22,7 @@ interface SchemaValidator<T> {
18
22
  validate(data: unknown): T;
19
23
  }
20
24
 
21
- declare const createScraper: <T>({ fields, validator, }: ScrapeConfig<T>) => ((html: string) => T);
25
+ declare const createScraper: <T>({ fields, validator, }: ScrapeConfig<T>) => ((html: cheerio.CheerioAPI | string) => T);
22
26
 
23
27
  declare class ZodValidator<T> implements SchemaValidator<T> {
24
28
  private schema;
package/dist/index.js CHANGED
@@ -1,18 +1,20 @@
1
1
  // src/createScraper.ts
2
2
  import * as cheerio from "cheerio";
3
- var createScraper = ({
4
- fields,
5
- validator
6
- }) => {
7
- return (html) => {
8
- const $ = cheerio.load(html);
9
- const data = {};
10
- for (const key in fields) {
11
- const fieldDef = fields[key];
12
- const elements = $(fieldDef.selector);
3
+ var extractData = (fields, $context) => {
4
+ const data = {};
5
+ for (const key in fields) {
6
+ const fieldDef = fields[key];
7
+ if ("fields" in fieldDef) {
8
+ const nestedData = extractData(
9
+ fieldDef.fields,
10
+ $context
11
+ );
12
+ data[key] = nestedData;
13
+ } else {
14
+ const elements = $context(fieldDef.selector);
13
15
  let values = [];
14
16
  elements.each((_, element) => {
15
- const value = fieldDef.attribute ? $(element).attr(fieldDef.attribute) : $(element).text();
17
+ const value = fieldDef.attribute ? $context(element).attr(fieldDef.attribute) : $context(element).text().trim();
16
18
  if (value !== void 0) {
17
19
  values.push(value);
18
20
  }
@@ -28,6 +30,16 @@ var createScraper = ({
28
30
  data[key] = fieldDef.transform && value ? fieldDef.transform(value) : value;
29
31
  }
30
32
  }
33
+ }
34
+ return data;
35
+ };
36
+ var createScraper = ({
37
+ fields,
38
+ validator
39
+ }) => {
40
+ return (html) => {
41
+ const $ = typeof html === "string" ? cheerio.load(html) : html;
42
+ const data = extractData(fields, $);
31
43
  return validator.validate(data);
32
44
  };
33
45
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "xscrape",
3
- "version": "1.0.0",
3
+ "version": "1.1.1",
4
4
  "description": "A flexible and powerful library designed to extract and transform data from HTML documents using user-defined schemas",
5
5
  "main": "dist/index.js",
6
6
  "exports": {