xscrape 1.3.1 → 2.0.0

package/README.md

@@ -1,9 +1,6 @@
 # xscrape
 
-`xscrape` is a powerful and flexible library designed for extracting and
-transforming data from HTML documents using user-defined schemas. It integrates
-seamlessly with various schema validation libraries such as Zod, Yup, Joi, and
-Effect Schema, allowing you to use your preferred validation tool.
+`xscrape` is a powerful and flexible library designed for extracting and transforming data from HTML documents using user-defined schemas. 
 
 ## Features
 
@@ -17,13 +14,13 @@ Effect Schema, allowing you to use your preferred validation tool.
 
 ### Schema Support
 
-| Schema Library                                       | Status              | Notes                                                              |
-| ---------------------------------------------------- | ------------------- | ------------------------------------------------------------------ |
-| [Zod](https://github.com/colinhacks/zod)             | ✅ Supported        | Default schema tool for `xscrape`                                  |
-| [Effect/Schema](https://github.com/Effect-TS/effect) | ✅ Supported        | Support for Effect/Schema for additional flexibility               |
-| [Joi](https://github.com/sideway/joi)                | ✅ Supported        | Support for Joi for those familiar with server-side validation     |
-| [Yup](https://github.com/jquense/yup)                | 🚧 Planned          | Adding Yup support for schema validation in front-end applications |
-| Others...                                            | 🔄 In Consideration | Potential support for other schema tools as per user feedback      |
+| Schema Library                                       | Status              | Notes                                                         |
+| ---------------------------------------------------- | ------------------- | ------------------------------------------------------------- |
+| [Zod](https://github.com/colinhacks/zod)             | ✅ Supported        | Default schema tool for `xscrape`                             |
+| [Effect/Schema](https://github.com/Effect-TS/effect) | 🔄 In Consideration        | Support for Effect/Schema for additional flexibility          |
+| [Joi](https://github.com/sideway/joi)                | 🔄 In Consideration        | Support for Joi for validation                                |
+| [Yup](https://github.com/jquense/yup)                | 🔄 In Consideration        | Support for Yup for validation                                |
+| Others...                                            | 🔄 In Consideration | Potential support for other schema tools as per user feedback |
 
 ## Installation
 

package/dist/index.cjs

@@ -30,108 +30,87 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
-  EffectValidator: () => EffectValidator,
-  JoiValidator: () => JoiValidator,
-  ZodValidator: () => ZodValidator,
-  createScraper: () => createScraper
+  defineScraper: () => defineScraper
 });
 module.exports = __toCommonJS(src_exports);
 
-// src/createScraper.ts
+// src/defineScraper.ts
 var cheerio = __toESM(require("cheerio"), 1);
-var extractData = (fields, $context) => {
-  const data = {};
-  for (const key in fields) {
-    const fieldDef = fields[key];
-    if ("fields" in fieldDef) {
-      const nestedData = extractData(
-        fieldDef.fields,
-        $context
-      );
-      data[key] = nestedData;
-    } else {
-      const elements = $context(fieldDef.selector);
-      let values = [];
-      elements.each((_, element) => {
-        const value = fieldDef.attribute ? $context(element).attr(fieldDef.attribute) : $context(element).text().trim();
-        if (value !== void 0) {
-          values.push(value);
-        }
-      });
-      if (values.length === 0 && fieldDef.defaultValue !== void 0) {
-        data[key] = fieldDef.defaultValue;
-      } else if (fieldDef.multiple) {
-        data[key] = values.map(
-          (value) => fieldDef.transform ? fieldDef.transform(value) : value
-        );
-      } else {
-        const value = values[0];
-        data[key] = fieldDef.transform && value ? fieldDef.transform(value) : value;
-      }
-    }
-  }
-  return data;
-};
-var createScraper = ({
-  fields,
-  validator
-}) => {
-  return (html) => {
-    const $ = typeof html === "string" ? cheerio.load(html) : html;
-    const data = extractData(fields, $);
-    return validator.validate(data);
-  };
-};
 
-// src/validators/effect.ts
-var Schema = __toESM(require("effect/Schema"), 1);
-var import_effect = require("effect");
-var EffectValidator = class {
-  constructor(schema) {
+// src/validators.ts
+var import_zod = require("zod");
+var Validator = class {
+  constructor(schema, validateFunction) {
     this.schema = schema;
+    this.validateFunction = validateFunction;
   }
   validate(data) {
-    const result = Schema.decodeUnknown(this.schema)(data);
-    return import_effect.Effect.runSync(result);
+    try {
+      const result = this.validateFunction(this.schema, data);
+      return { success: true, data: result };
+    } catch (error) {
+      return { success: false, error };
+    }
   }
 };
-
-// src/validators/zod.ts
-var import_zod = require("zod");
-var ZodValidator = class {
-  constructor(schema) {
-    this.schema = schema;
+function getSchemaBuilder(type) {
+  switch (type) {
+    case "zod":
+      return import_zod.z;
+    default:
+      throw new Error(`Unsupported validator type: ${type}`);
   }
-  validate(data) {
-    return this.schema.parse(data);
+}
+function createValidator(type, schemaFn) {
+  const builder = getSchemaBuilder(type);
+  const schema = schemaFn(builder);
+  switch (type) {
+    case "zod":
+      return new Validator(
+        schema,
+        (schema2, data) => schema2.parse(data)
+      );
+    default:
+      throw new Error(`Unsupported validator type: ${type}`);
   }
-};
+}
 
-// src/validators/joi.ts
-var JoiValidator = class {
-  constructor(schema) {
-    this.schema = schema;
-  }
-  validate(data) {
-    const { error, value } = this.schema.validate(data, {
-      convert: true,
-      stripUnknown: true,
-      presence: "optional",
-      abortEarly: false
-    });
-    if (error) {
-      throw new Error(this.formatError(error));
+// src/defineScraper.ts
+function defineScraper(config) {
+  const validator = createValidator(config.validator, config.schema);
+  return async (html) => {
+    try {
+      const $ = cheerio.load(html);
+      const extractedData = $.extract(config.extract);
+      const validationResult = validator.validate(extractedData);
+      if (!validationResult.success) {
+        return { error: validationResult.error };
+      }
+      if (!validationResult.data) {
+        return {
+          error: new Error("Validation succeeded but no data was returned")
+        };
+      }
+      if (config.transform) {
+        try {
+          const transformed = await Promise.resolve(
+            config.transform(validationResult.data)
+          );
+          return { data: transformed };
+        } catch (error) {
+          return { error };
+        }
+      }
+      return { data: validationResult.data };
+    } catch (error) {
+      return { error };
     }
-    return value;
-  }
-  formatError(error) {
-    return error.details.map((detail) => detail.message).join("\n");
-  }
-};
+  };
+}
+
+// src/types/main.ts
+var import_zod2 = require("zod");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  EffectValidator,
-  JoiValidator,
-  ZodValidator,
-  createScraper
+  defineScraper
 });

package/dist/index.d.cts

@@ -1,48 +1,66 @@
-import * as cheerio from 'cheerio';
-import * as Schema from 'effect/Schema';
-import { ZodSchema } from 'zod';
-import { Schema as Schema$1 } from 'joi';
+import { z } from 'zod';
+import { Element } from 'domhandler';
 
-type ScrapeConfig<T> = {
-    fields: SchemaFieldDefinitions<T>;
-    validator: SchemaValidator<T>;
-};
-type FieldDefinition<T> = {
+type ExtractDescriptorFn = (el: Element, key: string, obj: Record<string, unknown>) => unknown;
+interface ExtractDescriptor {
     selector: string;
-    attribute?: string;
-    transform?: (value: string) => T;
-    defaultValue?: T;
-    multiple?: boolean;
-} | NestedFieldDefinition<T>;
-type NestedFieldDefinition<T> = {
-    fields: SchemaFieldDefinitions<T>;
-};
-type SchemaFieldDefinitions<T> = {
-    [K in keyof T]: FieldDefinition<T[K]>;
-};
-interface SchemaValidator<T> {
-    validate(data: unknown): T;
+    value?: string | ExtractDescriptorFn | ExtractMap;
 }
-
-declare const createScraper: <T>({ fields, validator, }: ScrapeConfig<T>) => ((html: cheerio.CheerioAPI | string) => T);
-
-declare class EffectValidator<A, I = A> implements SchemaValidator<A> {
-    private schema;
-    constructor(schema: Schema.Schema<A, I>);
-    validate(data: unknown): A;
+type ExtractValue = string | ExtractDescriptor | [string | ExtractDescriptor];
+interface ExtractMap {
+    [key: string]: ExtractValue;
 }
 
-declare class ZodValidator<T> implements SchemaValidator<T> {
-    private schema;
-    constructor(schema: ZodSchema<T>);
-    validate(data: unknown): T;
-}
+type ValidatorType = 'zod';
+type ZodBuilder = typeof z;
+type SchemaBuilder<V extends ValidatorType> = V extends 'zod' ? ZodBuilder : never;
+type SchemaFunction<V extends ValidatorType, T> = (builder: SchemaBuilder<V>) => V extends 'zod' ? z.ZodSchema<T> : never;
+type ScraperConfig<T extends Record<string, unknown>, V extends ValidatorType, R extends T = T> = {
+    validator: V;
+    schema: SchemaFunction<V, T>;
+    extract: ExtractMap;
+    transform?: (data: T) => Promise<R> | R;
+};
+type BaseFieldOptions = {
+    attribute?: string;
+};
+type LeafFieldConfig = BaseFieldOptions & {
+    selector?: string;
+    selectorAll?: string;
+} & ({
+    selector: string;
+    selectorAll?: never;
+} | {
+    selector?: never;
+    selectorAll: string;
+});
+type FieldConfig<T> = T extends object ? T extends Array<infer U> ? LeafFieldConfig : {
+    fields: Fields<T>;
+} : LeafFieldConfig;
+type Fields<T> = {
+    [K in keyof T]: FieldConfig<T[K]>;
+};
+type ValidationResult<T> = {
+    success: boolean;
+    data?: T;
+    error?: unknown;
+};
+type ScraperResult<T> = {
+    data?: T;
+    error?: unknown;
+};
 
-declare class JoiValidator<T> implements SchemaValidator<T> {
-    private schema;
-    constructor(schema: Schema$1<T>);
-    validate(data: unknown): T;
-    private formatError;
-}
+/**
+ * Defines a scraper with the provided configuration.
+ *
+ * @template T - The shape of the extracted data.
+ * @template V - The type of the validator used for validation.
+ * @template R - The type of the result after optional transformation, defaults to T.
+ *
+ * @param config - The configuration object for the scraper.
+ * @returns A function that takes an HTML string and returns the scraping result, which could be
+ * a scraper result or a promise of a scraper result.
+ */
+declare function defineScraper<T extends Record<string, unknown>, V extends ValidatorType, R extends T = T>(config: ScraperConfig<T, V, R>): (html: string) => Promise<ScraperResult<R>>;
 
-export { EffectValidator, type FieldDefinition, JoiValidator, type SchemaFieldDefinitions, type SchemaValidator, type ScrapeConfig, ZodValidator, createScraper };
+export { type FieldConfig, type Fields, type LeafFieldConfig, type SchemaBuilder, type SchemaFunction, type ScraperConfig, type ScraperResult, type ValidationResult, type ValidatorType, defineScraper };

package/dist/index.d.ts

@@ -1,48 +1,66 @@
-import * as cheerio from 'cheerio';
-import * as Schema from 'effect/Schema';
-import { ZodSchema } from 'zod';
-import { Schema as Schema$1 } from 'joi';
+import { z } from 'zod';
+import { Element } from 'domhandler';
 
-type ScrapeConfig<T> = {
-    fields: SchemaFieldDefinitions<T>;
-    validator: SchemaValidator<T>;
-};
-type FieldDefinition<T> = {
+type ExtractDescriptorFn = (el: Element, key: string, obj: Record<string, unknown>) => unknown;
+interface ExtractDescriptor {
     selector: string;
-    attribute?: string;
-    transform?: (value: string) => T;
-    defaultValue?: T;
-    multiple?: boolean;
-} | NestedFieldDefinition<T>;
-type NestedFieldDefinition<T> = {
-    fields: SchemaFieldDefinitions<T>;
-};
-type SchemaFieldDefinitions<T> = {
-    [K in keyof T]: FieldDefinition<T[K]>;
-};
-interface SchemaValidator<T> {
-    validate(data: unknown): T;
+    value?: string | ExtractDescriptorFn | ExtractMap;
 }
-
-declare const createScraper: <T>({ fields, validator, }: ScrapeConfig<T>) => ((html: cheerio.CheerioAPI | string) => T);
-
-declare class EffectValidator<A, I = A> implements SchemaValidator<A> {
-    private schema;
-    constructor(schema: Schema.Schema<A, I>);
-    validate(data: unknown): A;
+type ExtractValue = string | ExtractDescriptor | [string | ExtractDescriptor];
+interface ExtractMap {
+    [key: string]: ExtractValue;
 }
 
-declare class ZodValidator<T> implements SchemaValidator<T> {
-    private schema;
-    constructor(schema: ZodSchema<T>);
-    validate(data: unknown): T;
-}
+type ValidatorType = 'zod';
+type ZodBuilder = typeof z;
+type SchemaBuilder<V extends ValidatorType> = V extends 'zod' ? ZodBuilder : never;
+type SchemaFunction<V extends ValidatorType, T> = (builder: SchemaBuilder<V>) => V extends 'zod' ? z.ZodSchema<T> : never;
+type ScraperConfig<T extends Record<string, unknown>, V extends ValidatorType, R extends T = T> = {
+    validator: V;
+    schema: SchemaFunction<V, T>;
+    extract: ExtractMap;
+    transform?: (data: T) => Promise<R> | R;
+};
+type BaseFieldOptions = {
+    attribute?: string;
+};
+type LeafFieldConfig = BaseFieldOptions & {
+    selector?: string;
+    selectorAll?: string;
+} & ({
+    selector: string;
+    selectorAll?: never;
+} | {
+    selector?: never;
+    selectorAll: string;
+});
+type FieldConfig<T> = T extends object ? T extends Array<infer U> ? LeafFieldConfig : {
+    fields: Fields<T>;
+} : LeafFieldConfig;
+type Fields<T> = {
+    [K in keyof T]: FieldConfig<T[K]>;
+};
+type ValidationResult<T> = {
+    success: boolean;
+    data?: T;
+    error?: unknown;
+};
+type ScraperResult<T> = {
+    data?: T;
+    error?: unknown;
+};
 
-declare class JoiValidator<T> implements SchemaValidator<T> {
-    private schema;
-    constructor(schema: Schema$1<T>);
-    validate(data: unknown): T;
-    private formatError;
-}
+/**
+ * Defines a scraper with the provided configuration.
+ *
+ * @template T - The shape of the extracted data.
+ * @template V - The type of the validator used for validation.
+ * @template R - The type of the result after optional transformation, defaults to T.
+ *
+ * @param config - The configuration object for the scraper.
+ * @returns A function that takes an HTML string and returns the scraping result, which could be
+ * a scraper result or a promise of a scraper result.
+ */
+declare function defineScraper<T extends Record<string, unknown>, V extends ValidatorType, R extends T = T>(config: ScraperConfig<T, V, R>): (html: string) => Promise<ScraperResult<R>>;
 
-export { EffectValidator, type FieldDefinition, JoiValidator, type SchemaFieldDefinitions, type SchemaValidator, type ScrapeConfig, ZodValidator, createScraper };
+export { type FieldConfig, type Fields, type LeafFieldConfig, type SchemaBuilder, type SchemaFunction, type ScraperConfig, type ScraperResult, type ValidationResult, type ValidatorType, defineScraper };

package/dist/index.js

@@ -1,97 +1,79 @@
-// src/createScraper.ts
+// src/defineScraper.ts
 import * as cheerio from "cheerio";
-var extractData = (fields, $context) => {
-  const data = {};
-  for (const key in fields) {
-    const fieldDef = fields[key];
-    if ("fields" in fieldDef) {
-      const nestedData = extractData(
-        fieldDef.fields,
-        $context
-      );
-      data[key] = nestedData;
-    } else {
-      const elements = $context(fieldDef.selector);
-      let values = [];
-      elements.each((_, element) => {
-        const value = fieldDef.attribute ? $context(element).attr(fieldDef.attribute) : $context(element).text().trim();
-        if (value !== void 0) {
-          values.push(value);
-        }
-      });
-      if (values.length === 0 && fieldDef.defaultValue !== void 0) {
-        data[key] = fieldDef.defaultValue;
-      } else if (fieldDef.multiple) {
-        data[key] = values.map(
-          (value) => fieldDef.transform ? fieldDef.transform(value) : value
-        );
-      } else {
-        const value = values[0];
-        data[key] = fieldDef.transform && value ? fieldDef.transform(value) : value;
-      }
-    }
-  }
-  return data;
-};
-var createScraper = ({
-  fields,
-  validator
-}) => {
-  return (html) => {
-    const $ = typeof html === "string" ? cheerio.load(html) : html;
-    const data = extractData(fields, $);
-    return validator.validate(data);
-  };
-};
 
-// src/validators/effect.ts
-import * as Schema from "effect/Schema";
-import { Effect } from "effect";
-var EffectValidator = class {
-  constructor(schema) {
+// src/validators.ts
+import { z } from "zod";
+var Validator = class {
+  constructor(schema, validateFunction) {
     this.schema = schema;
+    this.validateFunction = validateFunction;
   }
   validate(data) {
-    const result = Schema.decodeUnknown(this.schema)(data);
-    return Effect.runSync(result);
+    try {
+      const result = this.validateFunction(this.schema, data);
+      return { success: true, data: result };
+    } catch (error) {
+      return { success: false, error };
+    }
   }
 };
-
-// src/validators/zod.ts
-import "zod";
-var ZodValidator = class {
-  constructor(schema) {
-    this.schema = schema;
+function getSchemaBuilder(type) {
+  switch (type) {
+    case "zod":
+      return z;
+    default:
+      throw new Error(`Unsupported validator type: ${type}`);
   }
-  validate(data) {
-    return this.schema.parse(data);
+}
+function createValidator(type, schemaFn) {
+  const builder = getSchemaBuilder(type);
+  const schema = schemaFn(builder);
+  switch (type) {
+    case "zod":
+      return new Validator(
+        schema,
+        (schema2, data) => schema2.parse(data)
+      );
+    default:
+      throw new Error(`Unsupported validator type: ${type}`);
   }
-};
+}
 
-// src/validators/joi.ts
-var JoiValidator = class {
-  constructor(schema) {
-    this.schema = schema;
-  }
-  validate(data) {
-    const { error, value } = this.schema.validate(data, {
-      convert: true,
-      stripUnknown: true,
-      presence: "optional",
-      abortEarly: false
-    });
-    if (error) {
-      throw new Error(this.formatError(error));
+// src/defineScraper.ts
+function defineScraper(config) {
+  const validator = createValidator(config.validator, config.schema);
+  return async (html) => {
+    try {
+      const $ = cheerio.load(html);
+      const extractedData = $.extract(config.extract);
+      const validationResult = validator.validate(extractedData);
+      if (!validationResult.success) {
+        return { error: validationResult.error };
+      }
+      if (!validationResult.data) {
+        return {
+          error: new Error("Validation succeeded but no data was returned")
+        };
+      }
+      if (config.transform) {
+        try {
+          const transformed = await Promise.resolve(
+            config.transform(validationResult.data)
+          );
+          return { data: transformed };
+        } catch (error) {
+          return { error };
+        }
+      }
+      return { data: validationResult.data };
+    } catch (error) {
+      return { error };
     }
-    return value;
-  }
-  formatError(error) {
-    return error.details.map((detail) => detail.message).join("\n");
-  }
-};
+  };
+}
+
+// src/types/main.ts
+import "zod";
 export {
-  EffectValidator,
-  JoiValidator,
-  ZodValidator,
-  createScraper
+  defineScraper
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xscrape",
-  "version": "1.3.1",
+  "version": "2.0.0",
   "description": "A flexible and powerful library designed to extract and transform data from HTML documents using user-defined schemas",
   "main": "dist/index.js",
   "exports": {
@@ -39,26 +39,30 @@
   },
   "homepage": "https://github.com/johnie/xscrape#readme",
   "devDependencies": {
-    "@arethetypeswrong/cli": "^0.16.4",
-    "@changesets/changelog-github": "^0.5.0",
-    "@changesets/cli": "^2.27.9",
-    "prettier": "^3.3.3",
-    "tsup": "^8.3.5",
-    "typescript": "^5.6.3",
-    "vite": "^5.4.10",
-    "vitest": "^2.1.3"
+    "@arethetypeswrong/cli": "^0.17.3",
+    "@changesets/changelog-github": "^0.5.1",
+    "@changesets/cli": "^2.28.1",
+    "domhandler": "^5.0.3",
+    "jsdom": "^26.0.0",
+    "prettier": "^3.5.1",
+    "tsup": "^8.3.6",
+    "typescript": "^5.7.3",
+    "vite": "^6.1.1",
+    "vitest": "^3.0.6"
   },
   "dependencies": {
     "cheerio": "^1.0.0",
-    "effect": "^3.10.4",
+    "effect": "^3.13.2",
     "joi": "^17.13.3",
-    "zod": "^3.23.8"
+    "yup": "^1.6.1",
+    "zod": "^3.24.2"
   },
   "scripts": {
     "build": "tsup",
     "ci": "npm run build && npm run check-format && npm run check-exports && npm run lint && npm run test",
     "lint": "tsc",
     "test": "vitest run",
+    "test:watch": "vitest",
     "format": "prettier --write ./src",
     "check-format": "prettier --check ./src",
     "check-exports": "attw --pack .",