xscrape 3.0.4 → 3.2.1

package/README.md

@@ -66,7 +66,7 @@ const scraper = defineScraper({
     description: { selector: 'meta[name="description"]', value: 'content' },
     keywords: {
       selector: 'meta[name="keywords"]',
-      value: (el) => el.attribs['content']?.split(',') || [],
+      value: (node) => node.attr('content')?.split(',') || [],
     },
     views: { selector: 'meta[name="views"]', value: 'content' },
   },
@@ -218,16 +218,16 @@ const scraper = defineScraper({
   extract: {
     tags: {
       selector: 'meta[name="keywords"]',
-      value: (el) => el.attribs['content']?.split(',').map(tag => tag.trim()) || [],
+      value: (node) => node.attr('content')?.split(',').map(tag => tag.trim()) || [],
     },
     publishedDate: {
       selector: 'meta[name="published"]',
-      value: (el) => new Date(el.attribs['content']),
+      value: (node) => new Date(node.attr('content') ?? ''),
     },
     readingTime: {
       selector: 'article',
-      value: (el) => {
-        const text = el.text();
+      value: (node) => {
+        const text = node.text();
         const wordsPerMinute = 200;
         const wordCount = text.split(/\s+/).length;
         return Math.ceil(wordCount / wordsPerMinute);
@@ -253,7 +253,7 @@ const scraper = defineScraper({
     description: { selector: 'meta[name="description"]', value: 'content' },
     tags: {
       selector: 'meta[name="keywords"]',
-      value: (el) => el.attribs['content']?.split(',') || [],
+      value: (node) => node.attr('content')?.split(',') || [],
     },
   },
   transform: (data) => ({
@@ -336,14 +336,18 @@ A scraper function that takes HTML string and returns `Promise<{ data?: T, error
 The `extract` object defines how to extract data from HTML:
 
 ```typescript
-type ExtractConfig = {
-  [key: string]: ExtractDescriptor | [ExtractDescriptor];
-};
+interface ExtractNode {
+  attr(name: string): string | undefined;
+  text(): string;
+  html(): string | undefined;
+}
 
 type ExtractDescriptor = {
   selector: string;
-  value?: string | ((el: Element) => any) | ExtractConfig;
+  value?: string | ((node: ExtractNode) => unknown) | ExtractConfig;
 };
+
+type ExtractConfig = Record<string, string | ExtractDescriptor | [string | ExtractDescriptor]>;
 ```
 
 #### Properties
@@ -351,7 +355,7 @@ type ExtractDescriptor = {
 - `selector`: CSS selector to find elements
 - `value`: How to extract the value:
   - `string`: Attribute name (e.g., `'href'`, `'content'`)
-  - `function`: Custom extraction function
+  - `function`: Custom extraction function receiving an xscrape `ExtractNode`
   - `object`: Nested extraction configuration
   - `undefined`: Extract text content
 

package/dist/index.cjs

@@ -1,72 +1,81 @@
-"use strict";
-var __create = Object.create;
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __getProtoOf = Object.getPrototypeOf;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
-  // If the importer is in node compatibility mode or this is not an ESM
-  // file that has been converted to a CommonJS file using a Babel-
-  // compatible transform (i.e. "__esModule" has not been set), then set
-  // "default" to the CommonJS "module.exports" for node compatibility.
-  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
-  mod
-));
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-
-// src/index.ts
-var index_exports = {};
-__export(index_exports, {
-  defineScraper: () => defineScraper
-});
-module.exports = __toCommonJS(index_exports);
-
-// src/defineScraper.ts
-var cheerio = __toESM(require("cheerio"), 1);
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+let cheerio = require("cheerio");
+//#region src/internal/runtime.ts
+function createScraperRuntime(config) {
+	return async (html) => {
+		try {
+			const extractedData = extractHtml(html, config.extract);
+			const validation = await validateExtractedData(config.schema, extractedData);
+			if (!validation.ok) return { error: validation.error };
+			return { data: await transformValidatedData(validation.value, config.transform) };
+		} catch (error) {
+			return { error };
+		}
+	};
+}
+function extractHtml(html, extract) {
+	const $ = (0, cheerio.load)(html);
+	const plan = compileExtractConfig($, extract);
+	return $.extract(plan);
+}
+async function validateExtractedData(schema, extractedData) {
+	const validationResult = await Promise.resolve(schema["~standard"].validate(extractedData));
+	if (validationResult.issues) return {
+		ok: false,
+		error: validationResult.issues
+	};
+	if (!("value" in validationResult)) return {
+		ok: false,
+		error: /* @__PURE__ */ new Error("xscrape: Validation succeeded but no data was returned")
+	};
+	return {
+		ok: true,
+		value: validationResult.value
+	};
+}
+function transformValidatedData(value, transform) {
+	return Promise.resolve(transform ? transform(value) : value);
+}
+function compileExtractConfig($, extract) {
+	return Object.fromEntries(Object.entries(extract).map(([key, value]) => [key, compileExtractField($, value)]));
+}
+function compileExtractField($, field) {
+	if (Array.isArray(field)) {
+		const item = field[0];
+		return [typeof item === "string" ? item : compileExtractDescriptor($, item)];
+	}
+	if (typeof field === "string") return field;
+	return compileExtractDescriptor($, field);
+}
+function compileExtractDescriptor($, descriptor) {
+	const value = compileDescriptorValue($, descriptor.value);
+	return value === void 0 ? { selector: descriptor.selector } : {
+		selector: descriptor.selector,
+		value
+	};
+}
+function compileDescriptorValue($, value) {
+	if (value === void 0 || typeof value === "string") return value;
+	if (typeof value === "function") return (element, key, obj) => value(createExtractNode($, element), key, obj);
+	return compileExtractConfig($, value);
+}
+function createExtractNode($, element) {
+	return {
+		attr(name) {
+			return element.attribs[name] ?? void 0;
+		},
+		text() {
+			return $(element).text();
+		},
+		html() {
+			return $(element).html() ?? void 0;
+		}
+	};
+}
+//#endregion
+//#region src/index.ts
 function defineScraper(config) {
-  return async (html) => {
-    try {
-      const $ = cheerio.load(html);
-      const extractedData = $.extract(config.extract);
-      const validationResult = await Promise.resolve(
-        config.schema["~standard"].validate(extractedData)
-      );
-      if (validationResult.issues) {
-        return { error: validationResult.issues };
-      }
-      if (!("value" in validationResult)) {
-        return {
-          error: new Error(
-            "xscrape: Validation succeeded but no data was returned"
-          )
-        };
-      }
-      if (config.transform) {
-        const transformed = await Promise.resolve(
-          config.transform(validationResult.value)
-        );
-        return { data: transformed };
-      }
-      return { data: validationResult.value };
-    } catch (error) {
-      return { error };
-    }
-  };
+	return createScraperRuntime(config);
 }
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  defineScraper
-});
+//#endregion
+exports.defineScraper = defineScraper;

package/dist/index.d.cts

@@ -1,32 +1,36 @@
-import { StandardSchemaV1 } from '@standard-schema/spec';
-import { Element } from 'domhandler';
+import { StandardSchemaV1 } from "@standard-schema/spec";
 
-type ExtractDescriptorFn = (el: Element, key: string, obj: Record<string, unknown>) => unknown;
-interface ExtractDescriptor {
-    selector: string;
-    value?: string | ExtractDescriptorFn | ExtractMap;
+//#region src/types/extract.d.ts
+type ExtractObjectShape<T> = NonNullable<T> extends readonly unknown[] ? never : NonNullable<T> extends object ? NonNullable<T> : never;
+interface ExtractNode {
+  attr(name: string): string | undefined;
+  html(): string | undefined;
+  text(): string;
+}
+type ExtractValueCallback<T = unknown> = (node: ExtractNode, key: string, object: Record<string, unknown>) => T | undefined;
+interface ExtractDescriptor<T = unknown> {
+  selector: string;
+  value?: string | ExtractValueCallback<T> | ExtractConfig<ExtractObjectShape<T>>;
+}
+type ExtractField<T = unknown> = string | ExtractDescriptor<T> | (T extends readonly (infer Item)[] ? [string | ExtractDescriptor<NonNullable<Item>>] : never);
+type ExtractShape<T> = [T] extends [never] ? Record<string, unknown> : NonNullable<T> extends object ? NonNullable<T> : Record<string, unknown>;
+type ExtractConfig<T = Record<string, unknown>> = { [K in keyof ExtractShape<T>]: ExtractField<ExtractShape<T>[K]> };
+//#endregion
+//#region src/types/main.d.ts
+interface ScraperConfig<S extends StandardSchemaV1, R extends StandardSchemaV1.InferOutput<S> = StandardSchemaV1.InferOutput<S>> {
+  extract: ExtractConfig<StandardSchemaV1.InferOutput<S>>;
+  schema: S;
+  transform?: (data: StandardSchemaV1.InferOutput<S>) => Promise<R> | R;
 }
-type ExtractValue = string | ExtractDescriptor | [string | ExtractDescriptor];
-type ExtractMap = Record<string, ExtractValue>;
-
-type SchemaAwareExtractMap<T> = {
-    [K in keyof T]: ExtractMap[string];
-};
-type ScraperConfig<S extends StandardSchemaV1, R extends StandardSchemaV1.InferOutput<S> = StandardSchemaV1.InferOutput<S>> = {
-    schema: S;
-    extract: SchemaAwareExtractMap<StandardSchemaV1.InferOutput<S>>;
-    transform?: (data: StandardSchemaV1.InferOutput<S>) => Promise<R> | R;
-};
-type ValidationResult<T> = {
-    success: boolean;
-    data?: T;
-    error?: unknown;
-};
 type ScraperResult<T> = {
-    data?: T;
-    error?: unknown;
+  data: T;
+  error?: undefined;
+} | {
+  data?: undefined;
+  error: unknown;
 };
-
-declare function defineScraper<S extends StandardSchemaV1, T extends StandardSchemaV1.InferOutput<S> = StandardSchemaV1.InferOutput<S>, R extends T = T>(config: ScraperConfig<S, R>): (html: string) => Promise<ScraperResult<R>>;
-
-export { type ScraperConfig, type ScraperResult, type ValidationResult, defineScraper };
+//#endregion
+//#region src/index.d.ts
+declare function defineScraper<S extends StandardSchemaV1, R extends StandardSchemaV1.InferOutput<S> = StandardSchemaV1.InferOutput<S>>(config: ScraperConfig<S, R>): (html: string) => Promise<ScraperResult<R>>;
+//#endregion
+export { type ExtractConfig, type ExtractDescriptor, type ExtractField, type ExtractNode, type ExtractValueCallback, type ScraperConfig, type ScraperResult, defineScraper };

package/dist/index.d.mts

@@ -0,0 +1,36 @@
+import { StandardSchemaV1 } from "@standard-schema/spec";
+
+//#region src/types/extract.d.ts
+type ExtractObjectShape<T> = NonNullable<T> extends readonly unknown[] ? never : NonNullable<T> extends object ? NonNullable<T> : never;
+interface ExtractNode {
+  attr(name: string): string | undefined;
+  html(): string | undefined;
+  text(): string;
+}
+type ExtractValueCallback<T = unknown> = (node: ExtractNode, key: string, object: Record<string, unknown>) => T | undefined;
+interface ExtractDescriptor<T = unknown> {
+  selector: string;
+  value?: string | ExtractValueCallback<T> | ExtractConfig<ExtractObjectShape<T>>;
+}
+type ExtractField<T = unknown> = string | ExtractDescriptor<T> | (T extends readonly (infer Item)[] ? [string | ExtractDescriptor<NonNullable<Item>>] : never);
+type ExtractShape<T> = [T] extends [never] ? Record<string, unknown> : NonNullable<T> extends object ? NonNullable<T> : Record<string, unknown>;
+type ExtractConfig<T = Record<string, unknown>> = { [K in keyof ExtractShape<T>]: ExtractField<ExtractShape<T>[K]> };
+//#endregion
+//#region src/types/main.d.ts
+interface ScraperConfig<S extends StandardSchemaV1, R extends StandardSchemaV1.InferOutput<S> = StandardSchemaV1.InferOutput<S>> {
+  extract: ExtractConfig<StandardSchemaV1.InferOutput<S>>;
+  schema: S;
+  transform?: (data: StandardSchemaV1.InferOutput<S>) => Promise<R> | R;
+}
+type ScraperResult<T> = {
+  data: T;
+  error?: undefined;
+} | {
+  data?: undefined;
+  error: unknown;
+};
+//#endregion
+//#region src/index.d.ts
+declare function defineScraper<S extends StandardSchemaV1, R extends StandardSchemaV1.InferOutput<S> = StandardSchemaV1.InferOutput<S>>(config: ScraperConfig<S, R>): (html: string) => Promise<ScraperResult<R>>;
+//#endregion
+export { type ExtractConfig, type ExtractDescriptor, type ExtractField, type ExtractNode, type ExtractValueCallback, type ScraperConfig, type ScraperResult, defineScraper };

package/dist/index.mjs

@@ -0,0 +1,80 @@
+import { load } from "cheerio";
+//#region src/internal/runtime.ts
+function createScraperRuntime(config) {
+	return async (html) => {
+		try {
+			const extractedData = extractHtml(html, config.extract);
+			const validation = await validateExtractedData(config.schema, extractedData);
+			if (!validation.ok) return { error: validation.error };
+			return { data: await transformValidatedData(validation.value, config.transform) };
+		} catch (error) {
+			return { error };
+		}
+	};
+}
+function extractHtml(html, extract) {
+	const $ = load(html);
+	const plan = compileExtractConfig($, extract);
+	return $.extract(plan);
+}
+async function validateExtractedData(schema, extractedData) {
+	const validationResult = await Promise.resolve(schema["~standard"].validate(extractedData));
+	if (validationResult.issues) return {
+		ok: false,
+		error: validationResult.issues
+	};
+	if (!("value" in validationResult)) return {
+		ok: false,
+		error: /* @__PURE__ */ new Error("xscrape: Validation succeeded but no data was returned")
+	};
+	return {
+		ok: true,
+		value: validationResult.value
+	};
+}
+function transformValidatedData(value, transform) {
+	return Promise.resolve(transform ? transform(value) : value);
+}
+function compileExtractConfig($, extract) {
+	return Object.fromEntries(Object.entries(extract).map(([key, value]) => [key, compileExtractField($, value)]));
+}
+function compileExtractField($, field) {
+	if (Array.isArray(field)) {
+		const item = field[0];
+		return [typeof item === "string" ? item : compileExtractDescriptor($, item)];
+	}
+	if (typeof field === "string") return field;
+	return compileExtractDescriptor($, field);
+}
+function compileExtractDescriptor($, descriptor) {
+	const value = compileDescriptorValue($, descriptor.value);
+	return value === void 0 ? { selector: descriptor.selector } : {
+		selector: descriptor.selector,
+		value
+	};
+}
+function compileDescriptorValue($, value) {
+	if (value === void 0 || typeof value === "string") return value;
+	if (typeof value === "function") return (element, key, obj) => value(createExtractNode($, element), key, obj);
+	return compileExtractConfig($, value);
+}
+function createExtractNode($, element) {
+	return {
+		attr(name) {
+			return element.attribs[name] ?? void 0;
+		},
+		text() {
+			return $(element).text();
+		},
+		html() {
+			return $(element).html() ?? void 0;
+		}
+	};
+}
+//#endregion
+//#region src/index.ts
+function defineScraper(config) {
+	return createScraperRuntime(config);
+}
+//#endregion
+export { defineScraper };

package/package.json

@@ -1,19 +1,49 @@
 {
+  "dependencies": {
+    "@standard-schema/spec": "^1.1.0",
+    "cheerio": "^1.2.0",
+    "domhandler": "^6.0.1"
+  },
+  "devDependencies": {
+    "@arethetypeswrong/cli": "^0.18.2",
+    "@biomejs/biome": "2.4.14",
+    "@changesets/changelog-github": "^0.7.0",
+    "@changesets/cli": "^2.31.0",
+    "@types/node": "^25.6.0",
+    "arktype": "^2.2.0",
+    "effect": "^3.21.2",
+    "jsdom": "^29.1.1",
+    "lefthook": "^2.1.6",
+    "tsdown": "^0.22.0",
+    "typescript": "^6.0.3",
+    "ultracite": "7.6.3",
+    "valibot": "^1.4.0",
+    "vite": "^8.0.11",
+    "vitepress": "^1.6.4",
+    "vitest": "^4.1.5",
+    "zod": "^4.4.3"
+  },
   "name": "xscrape",
-  "version": "3.0.4",
+  "type": "module",
+  "version": "3.2.1",
   "description": "A flexible and powerful library designed to extract and transform data from HTML documents using user-defined schemas",
-  "main": "dist/index.js",
+  "main": "dist/index.cjs",
   "exports": {
     ".": {
-      "import": "./dist/index.js",
-      "require": "./dist/index.cjs"
+      "import": {
+        "types": "./dist/index.d.mts",
+        "default": "./dist/index.mjs"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
     }
   },
-  "typings": "dist/index.d.ts",
+  "typings": "dist/index.d.mts",
   "files": [
     "dist"
   ],
-  "type": "module",
   "keywords": [
     "web-scraping",
     "data-extraction",
@@ -39,38 +69,17 @@
     "url": "https://github.com/johnie/xscrape/issues"
   },
   "homepage": "https://github.com/johnie/xscrape#readme",
-  "devDependencies": {
-    "@arethetypeswrong/cli": "^0.18.2",
-    "@biomejs/biome": "2.1.2",
-    "@changesets/changelog-github": "^0.5.1",
-    "@changesets/cli": "^2.29.5",
-    "arktype": "^2.1.20",
-    "effect": "^3.17.0",
-    "jsdom": "^26.1.0",
-    "lefthook": "^1.12.2",
-    "tsup": "^8.5.0",
-    "typescript": "^5.8.3",
-    "valibot": "^1.1.0",
-    "vite": "^7.0.5",
-    "vitepress": "^1.6.3",
-    "vitest": "^3.2.4",
-    "zod": "^4.0.5"
-  },
-  "dependencies": {
-    "@standard-schema/spec": "^1.0.0",
-    "cheerio": "^1.1.2",
-    "domhandler": "^5.0.3"
-  },
   "scripts": {
-    "build": "tsup",
-    "ci": "pnpm run build && pnpm run lint && pnpm run typecheck && pnpm run check-exports && pnpm run test",
+    "build": "tsdown",
+    "ci": "pnpm run build && pnpm run check && pnpm run typecheck && pnpm run check-exports && pnpm run test",
     "typecheck": "tsc",
     "test": "vitest run",
     "test:watch": "vitest",
-    "format": "biome format --write ./src",
-    "lint": "biome check ./src",
+    "check": "ultracite check",
+    "fix": "ultracite fix",
     "check-exports": "attw --pack .",
-    "local-release": "pnpm run ci && changeset version && changeset publish",
+    "version": "changeset version && pnpm run fix",
+    "local-release": "pnpm run ci && pnpm run version && changeset publish",
     "release": "pnpm run ci && changeset publish",
     "docs:dev": "vitepress dev docs",
     "docs:build": "vitepress build docs",

package/dist/index.d.ts

@@ -1,32 +0,0 @@
-import { StandardSchemaV1 } from '@standard-schema/spec';
-import { Element } from 'domhandler';
-
-type ExtractDescriptorFn = (el: Element, key: string, obj: Record<string, unknown>) => unknown;
-interface ExtractDescriptor {
-    selector: string;
-    value?: string | ExtractDescriptorFn | ExtractMap;
-}
-type ExtractValue = string | ExtractDescriptor | [string | ExtractDescriptor];
-type ExtractMap = Record<string, ExtractValue>;
-
-type SchemaAwareExtractMap<T> = {
-    [K in keyof T]: ExtractMap[string];
-};
-type ScraperConfig<S extends StandardSchemaV1, R extends StandardSchemaV1.InferOutput<S> = StandardSchemaV1.InferOutput<S>> = {
-    schema: S;
-    extract: SchemaAwareExtractMap<StandardSchemaV1.InferOutput<S>>;
-    transform?: (data: StandardSchemaV1.InferOutput<S>) => Promise<R> | R;
-};
-type ValidationResult<T> = {
-    success: boolean;
-    data?: T;
-    error?: unknown;
-};
-type ScraperResult<T> = {
-    data?: T;
-    error?: unknown;
-};
-
-declare function defineScraper<S extends StandardSchemaV1, T extends StandardSchemaV1.InferOutput<S> = StandardSchemaV1.InferOutput<S>, R extends T = T>(config: ScraperConfig<S, R>): (html: string) => Promise<ScraperResult<R>>;
-
-export { type ScraperConfig, type ScraperResult, type ValidationResult, defineScraper };

package/dist/index.js

@@ -1,35 +0,0 @@
-// src/defineScraper.ts
-import * as cheerio from "cheerio";
-function defineScraper(config) {
-  return async (html) => {
-    try {
-      const $ = cheerio.load(html);
-      const extractedData = $.extract(config.extract);
-      const validationResult = await Promise.resolve(
-        config.schema["~standard"].validate(extractedData)
-      );
-      if (validationResult.issues) {
-        return { error: validationResult.issues };
-      }
-      if (!("value" in validationResult)) {
-        return {
-          error: new Error(
-            "xscrape: Validation succeeded but no data was returned"
-          )
-        };
-      }
-      if (config.transform) {
-        const transformed = await Promise.resolve(
-          config.transform(validationResult.value)
-        );
-        return { data: transformed };
-      }
-      return { data: validationResult.value };
-    } catch (error) {
-      return { error };
-    }
-  };
-}
-export {
-  defineScraper
-};