scrapex 0.5.3 → 1.0.0-beta.1

package/dist/index.mjs

@@ -0,0 +1,1956 @@
+import { c as BaseHttpProvider, d as Semaphore, f as withResilience, l as CircuitBreaker, n as enhance, p as ScrapeError, r as extract, u as RateLimiter } from "./enhancer-ByjRD-t5.mjs";
+import { t as RSSParser } from "./parsers-CwkYnyWY.mjs";
+import * as cheerio from "cheerio";
+import { createHash } from "node:crypto";
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+
+//#region src/core/context.ts
+let jsdomModule = null;
+/**
+* Preload JSDOM module (called once during scrape initialization)
+*/
+async function preloadJsdom() {
+	if (!jsdomModule) jsdomModule = await import("jsdom");
+}
+/**
+* Create an extraction context with lazy JSDOM loading.
+*
+* Cheerio is always available for fast DOM queries.
+* JSDOM is only loaded when getDocument() is called (for Readability).
+*/
+function createExtractionContext(url, finalUrl, html, options) {
+	let document = null;
+	return {
+		url,
+		finalUrl,
+		html,
+		$: cheerio.load(html),
+		options,
+		results: {},
+		getDocument() {
+			if (!document) {
+				if (!jsdomModule) throw new Error("JSDOM not preloaded. Call preloadJsdom() before using getDocument().");
+				document = new jsdomModule.JSDOM(html, { url: finalUrl }).window.document;
+			}
+			return document;
+		}
+	};
+}
+/**
+* Merge partial results into the context
+*/
+function mergeResults(context, extracted) {
+	return {
+		...context,
+		results: {
+			...context.results,
+			...extracted,
+			custom: extracted.custom || context.results.custom ? {
+				...context.results.custom,
+				...extracted.custom
+			} : void 0
+		}
+	};
+}
+
+//#endregion
+//#region src/embeddings/aggregation.ts
+/**
+* Aggregate multiple embedding vectors into a single vector or return all.
+*
+* @param vectors - Array of embedding vectors (must all have same dimensions)
+* @param strategy - Aggregation strategy
+* @returns Aggregated result based on strategy
+*/
+function aggregateVectors(vectors, strategy = "average") {
+	if (vectors.length === 0) throw new Error("Cannot aggregate empty vector array");
+	const firstVector = vectors[0];
+	if (!firstVector) throw new Error("Cannot aggregate empty vector array");
+	const dimensions = firstVector.length;
+	for (let i = 1; i < vectors.length; i++) {
+		const vec = vectors[i];
+		if (!vec || vec.length !== dimensions) throw new Error(`Vector dimension mismatch: expected ${dimensions}, got ${vec?.length ?? 0} at index ${i}`);
+	}
+	switch (strategy) {
+		case "average": return {
+			type: "single",
+			vector: averageVectors(vectors),
+			dimensions
+		};
+		case "max": return {
+			type: "single",
+			vector: maxPoolVectors(vectors),
+			dimensions
+		};
+		case "first": return {
+			type: "single",
+			vector: firstVector,
+			dimensions
+		};
+		case "all": return {
+			type: "multiple",
+			vectors,
+			dimensions
+		};
+		default: {
+			const _exhaustive = strategy;
+			throw new Error(`Unknown aggregation strategy: ${_exhaustive}`);
+		}
+	}
+}
+/**
+* Compute element-wise average of vectors.
+*/
+function averageVectors(vectors) {
+	const first = vectors[0];
+	if (!first || vectors.length === 1) return first ?? [];
+	const dimensions = first.length;
+	const count = vectors.length;
+	const result = new Array(dimensions).fill(0);
+	for (const vector of vectors) for (let i = 0; i < dimensions; i++) {
+		const val = result[i];
+		if (val !== void 0) result[i] = val + (vector[i] ?? 0);
+	}
+	for (let i = 0; i < dimensions; i++) {
+		const val = result[i];
+		if (val !== void 0) result[i] = val / count;
+	}
+	return result;
+}
+/**
+* Compute element-wise maximum of vectors (max pooling).
+*/
+function maxPoolVectors(vectors) {
+	const first = vectors[0];
+	if (!first || vectors.length === 1) return first ?? [];
+	const dimensions = first.length;
+	const result = [...first];
+	for (let v = 1; v < vectors.length; v++) {
+		const vec = vectors[v];
+		if (!vec) continue;
+		for (let i = 0; i < dimensions; i++) {
+			const val = vec[i] ?? 0;
+			if (val > (result[i] ?? 0)) result[i] = val;
+		}
+	}
+	return result;
+}
+/**
+* Compute cosine similarity between two vectors.
+* Both vectors should be normalized for accurate results.
+*/
+function cosineSimilarity(a, b) {
+	if (a.length !== b.length) throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+	let dot = 0;
+	let magnitudeA = 0;
+	let magnitudeB = 0;
+	for (let i = 0; i < a.length; i++) {
+		const aVal = a[i] ?? 0;
+		const bVal = b[i] ?? 0;
+		dot += aVal * bVal;
+		magnitudeA += aVal * aVal;
+		magnitudeB += bVal * bVal;
+	}
+	const magnitude = Math.sqrt(magnitudeA) * Math.sqrt(magnitudeB);
+	if (magnitude === 0) return 0;
+	return dot / magnitude;
+}
+/**
+* Get the dimensions of a vector or set of vectors.
+*/
+function getDimensions(vectors) {
+	if (vectors.length === 0) return 0;
+	const first = vectors[0];
+	if (typeof first === "number") return vectors.length;
+	return first?.length ?? 0;
+}
+
+//#endregion
+//#region src/embeddings/cache.ts
+/**
+* Default maximum cache entries.
+*/
+const DEFAULT_MAX_ENTRIES = 1e3;
+/**
+* Default TTL in milliseconds (1 hour).
+*/
+const DEFAULT_TTL_MS = 3600 * 1e3;
+/**
+* Generate a content-addressable cache key.
+* Key is based on content hash and embedding configuration.
+* Note: custom RegExp patterns are serialized by source+flags; different
+* constructions can yield different cache keys even if equivalent.
+*/
+function generateCacheKey(params) {
+	const hash = createHash("sha256");
+	const fingerprint = stableStringify({
+		providerKey: params.providerKey,
+		model: params.model ?? "provider-default",
+		dimensions: params.dimensions ?? "default",
+		aggregation: params.aggregation ?? "average",
+		input: serializeInputConfig(params.input),
+		chunking: serializeChunkingConfig(params.chunking),
+		safety: serializeSafetyConfig(params.safety),
+		cacheKeySalt: params.cacheKeySalt
+	});
+	hash.update(fingerprint);
+	hash.update("\0");
+	hash.update(params.content);
+	return hash.digest("hex");
+}
+/**
+* Generate a checksum for content verification.
+*/
+function generateChecksum(content) {
+	return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+function serializeInputConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		type: config.type ?? "textContent",
+		hasTransform: Boolean(config.transform),
+		hasCustomText: Boolean(config.customText)
+	});
+}
+function serializeChunkingConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		size: config.size,
+		overlap: config.overlap,
+		tokenizer: getTokenizerId(config.tokenizer),
+		maxInputLength: config.maxInputLength
+	});
+}
+function serializeSafetyConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		piiRedaction: serializePiiConfig(config.piiRedaction),
+		minTextLength: config.minTextLength,
+		maxTokens: config.maxTokens
+	});
+}
+function serializePiiConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		email: config.email ?? false,
+		phone: config.phone ?? false,
+		creditCard: config.creditCard ?? false,
+		ssn: config.ssn ?? false,
+		ipAddress: config.ipAddress ?? false,
+		customPatterns: config.customPatterns?.map((pattern) => `${pattern.source}/${pattern.flags}`)
+	});
+}
+function getTokenizerId(tokenizer) {
+	if (!tokenizer || tokenizer === "heuristic") return "heuristic";
+	if (tokenizer === "tiktoken") return "tiktoken";
+	return "custom";
+}
+function stableStringify(value) {
+	return stringifyNormalized(normalizeValue(value));
+}
+function normalizeValue(value) {
+	if (value === void 0) return void 0;
+	if (value === null) return null;
+	if (Array.isArray(value)) return value.map((entry) => normalizeValue(entry)).filter((entry) => entry !== void 0);
+	if (typeof value === "object") return normalizeObject(value);
+	return value;
+}
+function normalizeObject(value) {
+	const normalized = {};
+	for (const key of Object.keys(value).sort()) {
+		const entry = normalizeValue(value[key]);
+		if (entry !== void 0) normalized[key] = entry;
+	}
+	return normalized;
+}
+function stringifyNormalized(value) {
+	if (value === void 0) return "undefined";
+	if (value === null) return "null";
+	if (typeof value === "string") return JSON.stringify(value);
+	if (typeof value === "number" || typeof value === "boolean") return String(value);
+	if (Array.isArray(value)) return `[${value.map((entry) => stringifyNormalized(entry)).join(",")}]`;
+	if (typeof value === "object") {
+		const obj = value;
+		return `{${Object.keys(obj).sort().map((key) => `${JSON.stringify(key)}:${stringifyNormalized(obj[key])}`).join(",")}}`;
+	}
+	return JSON.stringify(value);
+}
+/**
+* In-memory LRU cache with TTL support.
+* Content-addressable: uses content hash as key, not URL.
+*/
+var InMemoryEmbeddingCache = class {
+	cache;
+	maxEntries;
+	defaultTtlMs;
+	constructor(options) {
+		this.cache = /* @__PURE__ */ new Map();
+		this.maxEntries = options?.maxEntries ?? DEFAULT_MAX_ENTRIES;
+		this.defaultTtlMs = options?.ttlMs ?? DEFAULT_TTL_MS;
+	}
+	async get(key) {
+		const entry = this.cache.get(key);
+		if (!entry) return;
+		const now = Date.now();
+		if (now > entry.expiresAt) {
+			this.cache.delete(key);
+			return;
+		}
+		entry.accessedAt = now;
+		return entry.value;
+	}
+	async set(key, value, options) {
+		const now = Date.now();
+		const ttl = options?.ttlMs ?? this.defaultTtlMs;
+		if (this.cache.size >= this.maxEntries && !this.cache.has(key)) this.evictLRU();
+		this.cache.set(key, {
+			value,
+			createdAt: now,
+			expiresAt: now + ttl,
+			accessedAt: now
+		});
+	}
+	async delete(key) {
+		return this.cache.delete(key);
+	}
+	async clear() {
+		this.cache.clear();
+	}
+	/**
+	* Get cache statistics.
+	*/
+	getStats() {
+		const now = Date.now();
+		let expired = 0;
+		for (const entry of this.cache.values()) if (now > entry.expiresAt) expired++;
+		return {
+			size: this.cache.size,
+			maxEntries: this.maxEntries,
+			expired,
+			utilization: this.cache.size / this.maxEntries
+		};
+	}
+	/**
+	* Evict expired entries.
+	*/
+	cleanup() {
+		const now = Date.now();
+		let evicted = 0;
+		for (const [key, entry] of this.cache.entries()) if (now > entry.expiresAt) {
+			this.cache.delete(key);
+			evicted++;
+		}
+		return evicted;
+	}
+	/**
+	* Evict least recently used entry.
+	*/
+	evictLRU() {
+		let oldestKey = null;
+		let oldestAccess = Number.POSITIVE_INFINITY;
+		for (const [key, entry] of this.cache.entries()) if (entry.accessedAt < oldestAccess) {
+			oldestAccess = entry.accessedAt;
+			oldestKey = key;
+		}
+		if (oldestKey) this.cache.delete(oldestKey);
+	}
+};
+/**
+* Default in-memory cache instance.
+* Optimized for moderate cache sizes (default 1000 entries).
+*/
+let defaultCache = null;
+/**
+* Get or create the default cache instance.
+*/
+function getDefaultCache() {
+	if (!defaultCache) defaultCache = new InMemoryEmbeddingCache();
+	return defaultCache;
+}
+
+//#endregion
+//#region src/embeddings/chunking.ts
+/**
+* Default chunk size in tokens.
+*/
+const DEFAULT_CHUNK_SIZE$1 = 500;
+/**
+* Default overlap in tokens.
+*/
+const DEFAULT_OVERLAP = 50;
+/**
+* Default maximum input length in characters.
+*/
+const DEFAULT_MAX_INPUT_LENGTH = 1e5;
+/**
+* Heuristic token counting: approximately 4 characters per token.
+* This is a reasonable approximation for English text.
+*/
+function heuristicTokenCount(text) {
+	return Math.ceil(text.length / 4);
+}
+/**
+* Convert token count to approximate character count.
+*/
+function tokensToChars(tokens) {
+	return tokens * 4;
+}
+/**
+* Create a tokenizer function based on configuration.
+*/
+function createTokenizer(config) {
+	if (!config || config === "heuristic") return heuristicTokenCount;
+	if (config === "tiktoken") return heuristicTokenCount;
+	return config;
+}
+/**
+* Find a natural break point in text (sentence or word boundary).
+* Prefers common sentence boundaries (Latin + CJK), falls back to word boundaries.
+*/
+function findBreakPoint(text, targetIndex) {
+	const searchStart = Math.max(0, targetIndex - Math.floor(targetIndex * .2));
+	const searchEnd = Math.min(text.length, targetIndex + Math.floor(targetIndex * .2));
+	const searchText = text.slice(searchStart, searchEnd);
+	const sentenceMatch = /[.!?。！？]\s*/g;
+	let lastSentenceEnd = -1;
+	for (const match of searchText.matchAll(sentenceMatch)) {
+		const absolutePos = searchStart + match.index + match[0].length;
+		if (absolutePos <= targetIndex) lastSentenceEnd = absolutePos;
+	}
+	if (lastSentenceEnd !== -1) return lastSentenceEnd;
+	const wordBoundary = text.lastIndexOf(" ", targetIndex);
+	if (wordBoundary > searchStart) return wordBoundary + 1;
+	return targetIndex;
+}
+/**
+* Split text into overlapping chunks optimized for embedding.
+* Respects sentence boundaries when possible.
+*/
+function chunkText(text, config) {
+	const chunkSize = config?.size ?? DEFAULT_CHUNK_SIZE$1;
+	const rawOverlap = config?.overlap ?? DEFAULT_OVERLAP;
+	const safeOverlap = Math.max(0, rawOverlap);
+	const overlap = Math.min(safeOverlap, Math.max(0, chunkSize - 1));
+	const maxInputLength = config?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
+	const tokenizer = createTokenizer(config?.tokenizer);
+	const normalizedText = (text.length > maxInputLength ? text.slice(0, maxInputLength) : text).replace(/\s+/g, " ").trim();
+	if (!normalizedText) return [];
+	const totalTokens = tokenizer(normalizedText);
+	if (totalTokens <= chunkSize) return [{
+		text: normalizedText,
+		startIndex: 0,
+		endIndex: normalizedText.length,
+		tokens: totalTokens
+	}];
+	const chunks = [];
+	const chunkSizeChars = tokensToChars(chunkSize);
+	const overlapChars = tokensToChars(overlap);
+	let startIndex = 0;
+	while (startIndex < normalizedText.length) {
+		const targetEnd = Math.min(startIndex + chunkSizeChars, normalizedText.length);
+		const endIndex = targetEnd < normalizedText.length ? findBreakPoint(normalizedText, targetEnd) : targetEnd;
+		const chunkText$1 = normalizedText.slice(startIndex, endIndex).trim();
+		if (chunkText$1) chunks.push({
+			text: chunkText$1,
+			startIndex,
+			endIndex,
+			tokens: tokenizer(chunkText$1)
+		});
+		if (endIndex >= normalizedText.length) break;
+		const nextStart = endIndex - overlapChars;
+		startIndex = Math.max(nextStart, startIndex + 1);
+		if (startIndex < normalizedText.length) {
+			const spaceIndex = normalizedText.indexOf(" ", startIndex);
+			if (spaceIndex !== -1 && spaceIndex < startIndex + overlapChars) startIndex = spaceIndex + 1;
+		}
+	}
+	return chunks;
+}
+/**
+* Estimate total tokens for a text without chunking.
+*/
+function estimateTokens(text, tokenizer) {
+	return createTokenizer(tokenizer)(text);
+}
+
+//#endregion
+//#region src/embeddings/input.ts
+/**
+* Select and prepare input text for embedding based on configuration.
+*
+* @param data - Scraped data to extract input from
+* @param config - Input configuration
+* @returns Selected and prepared text, or undefined if no valid input
+*/
+function selectInput(data, config) {
+	if (config?.transform) return normalizeText(config.transform(data));
+	if (config?.type === "custom" && config.customText) return normalizeText(config.customText);
+	const type = config?.type ?? "textContent";
+	switch (type) {
+		case "textContent": return selectTextContent(data);
+		case "title+summary": return selectTitleSummary(data);
+		case "custom": return selectTextContent(data);
+		default: {
+			const _exhaustive = type;
+			throw new Error(`Unknown input type: ${_exhaustive}`);
+		}
+	}
+}
+/**
+* Select textContent as input.
+*/
+function selectTextContent(data) {
+	if (data.textContent) return normalizeText(data.textContent);
+	if (data.content) return normalizeText(stripMarkdown(data.content));
+	if (data.excerpt) return normalizeText(data.excerpt);
+	if (data.description) return normalizeText(data.description);
+}
+/**
+* Select title + summary (or fallbacks) as input.
+* Optimized for semantic search and classification.
+*/
+function selectTitleSummary(data) {
+	const parts = [];
+	if (data.title) parts.push(data.title);
+	if (data.summary) parts.push(data.summary);
+	else if (data.excerpt) parts.push(data.excerpt);
+	else if (data.description) parts.push(data.description);
+	if (parts.length === 0) return;
+	return normalizeText(parts.join("\n\n"));
+}
+/**
+* Normalize text for embedding:
+* - Collapse whitespace
+* - Trim leading/trailing whitespace
+* - Remove control characters
+*/
+function normalizeText(text) {
+	if (!text) return "";
+	return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").split("\n").map((line) => line.trim()).join("\n").trim();
+}
+/**
+* Basic markdown stripping for when we need plain text from content.
+* Not comprehensive, but handles common cases.
+*/
+function stripMarkdown(markdown) {
+	return markdown.replace(/```[\s\S]*?```/g, "").replace(/`[^`]+`/g, "").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replace(/^#{1,6}\s+/gm, "").replace(/\*\*([^*]+)\*\*/g, "$1").replace(/\*([^*]+)\*/g, "$1").replace(/__([^_]+)__/g, "$1").replace(/_([^_]+)_/g, "$1").replace(/^>\s+/gm, "").replace(/^[-*_]{3,}$/gm, "").replace(/^[\s]*[-*+]\s+/gm, "").replace(/^[\s]*\d+\.\s+/gm, "");
+}
+/**
+* Check if the selected input meets minimum requirements.
+*/
+function validateInput(text, minLength = 10) {
+	if (!text) return {
+		valid: false,
+		reason: "No input text available"
+	};
+	if (text.length < minLength) return {
+		valid: false,
+		reason: `Input too short (${text.length} < ${minLength} characters)`
+	};
+	const wordCount = text.split(/\s+/).filter((w) => w.length > 1).length;
+	if (wordCount < 3) return {
+		valid: false,
+		reason: `Input has too few words (${wordCount} < 3)`
+	};
+	return {
+		valid: true,
+		text,
+		wordCount,
+		charCount: text.length
+	};
+}
+
+//#endregion
+//#region src/embeddings/providers/base.ts
+/**
+* Generate a stable cache key identifier for provider configuration.
+*/
+function getProviderCacheKey(config) {
+	switch (config.type) {
+		case "http": return `http:${config.config.baseUrl.replace(/\/$/, "")}:${config.config.model}`;
+		case "custom": return `custom:${config.provider.name}`;
+		default: {
+			const _exhaustive = config;
+			return String(_exhaustive);
+		}
+	}
+}
+
+//#endregion
+//#region src/embeddings/providers/http.ts
+/**
+* HTTP-based Embedding Provider using native fetch.
+* Provides a unified interface for any REST-based embedding API.
+*/
+/**
+* HTTP-based embedding provider.
+* Works with any REST API using native fetch.
+*/
+var HttpEmbeddingProvider = class extends BaseHttpProvider {
+	name = "http-embedding";
+	requestBuilder;
+	responseMapper;
+	constructor(config) {
+		super(config);
+		this.requestBuilder = config.requestBuilder ?? ((texts, model) => ({
+			input: texts,
+			model
+		}));
+		this.responseMapper = config.responseMapper ?? ((response) => {
+			const resp = response;
+			if (Array.isArray(resp.data)) return resp.data.map((item) => item.embedding);
+			if (Array.isArray(resp.embeddings)) return resp.embeddings;
+			if (Array.isArray(resp.embedding)) return [resp.embedding];
+			if (Array.isArray(response)) return response;
+			throw new ScrapeError("Unable to parse embedding response. Provide a custom responseMapper.", "VALIDATION_ERROR");
+		});
+	}
+	/**
+	* Generate embeddings for one or more texts.
+	*/
+	async embed(texts, options) {
+		const model = options.model || this.model;
+		const body = this.requestBuilder(texts, model);
+		const { data } = await this.fetch(this.baseUrl, {
+			body,
+			signal: options.signal
+		});
+		const embeddings = this.responseMapper(data);
+		if (embeddings.length !== texts.length) throw new ScrapeError(`Embedding count mismatch: expected ${texts.length}, got ${embeddings.length}`, "VALIDATION_ERROR");
+		return { embeddings };
+	}
+};
+/**
+* Create a generic HTTP embedding provider.
+*/
+function createHttpEmbedding(config) {
+	return new HttpEmbeddingProvider(config);
+}
+
+//#endregion
+//#region src/embeddings/providers/presets.ts
+/**
+* Create an OpenAI embedding provider.
+*
+* @example
+* ```ts
+* const provider = createOpenAIEmbedding({ apiKey: 'sk-...' });
+* const { embeddings } = await provider.embed(['Hello'], { model: 'text-embedding-3-small' });
+* ```
+*/
+function createOpenAIEmbedding(options) {
+	const apiKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
+	if (!apiKey) throw new Error("OpenAI API key required. Set OPENAI_API_KEY env var or pass apiKey option.");
+	const headers = { Authorization: `Bearer ${apiKey}` };
+	if (options?.organization) headers["OpenAI-Organization"] = options.organization;
+	return new HttpEmbeddingProvider({
+		baseUrl: options?.baseUrl ?? "https://api.openai.com/v1/embeddings",
+		model: options?.model ?? "text-embedding-3-small",
+		headers,
+		requestBuilder: (texts, model) => ({
+			input: texts,
+			model
+		}),
+		responseMapper: (res) => res.data.map((item) => item.embedding)
+	});
+}
+/**
+* Create an Azure OpenAI embedding provider.
+*
+* @example
+* ```ts
+* const provider = createAzureEmbedding({
+*   endpoint: 'https://my-resource.openai.azure.com',
+*   deploymentName: 'text-embedding-ada-002',
+*   apiVersion: '2023-05-15',
+* });
+* ```
+*/
+function createAzureEmbedding(options) {
+	const apiKey = options.apiKey ?? process.env.AZURE_OPENAI_API_KEY;
+	if (!apiKey) throw new Error("Azure OpenAI API key required. Set AZURE_OPENAI_API_KEY env var or pass apiKey option.");
+	return new HttpEmbeddingProvider({
+		baseUrl: `${options.endpoint.replace(/\/$/, "")}/openai/deployments/${options.deploymentName}/embeddings?api-version=${options.apiVersion}`,
+		model: options.deploymentName,
+		headers: { "api-key": apiKey },
+		requestBuilder: (texts) => ({ input: texts }),
+		responseMapper: (res) => res.data.map((item) => item.embedding)
+	});
+}
+/**
+* Create an Ollama embedding provider for local models.
+*
+* LIMITATION: Ollama's /api/embeddings endpoint processes one text at a time,
+* not batches. When multiple chunks are embedded, each chunk triggers a
+* separate HTTP request. This is handled transparently by the pipeline's
+* sequential chunk processing, but may be slower than batch-capable providers.
+* For high-throughput scenarios, consider using OpenAI, Cohere, or HuggingFace
+* which support batch embedding in a single request.
+*
+* @example
+* ```ts
+* const provider = createOllamaEmbedding({ model: 'nomic-embed-text' });
+* ```
+*/
+function createOllamaEmbedding(options) {
+	return new HttpEmbeddingProvider({
+		baseUrl: options?.baseUrl ?? "http://localhost:11434/api/embeddings",
+		model: options?.model ?? "nomic-embed-text",
+		requireHttps: false,
+		allowPrivate: true,
+		requestBuilder: (texts, model) => ({
+			model,
+			prompt: texts[0]
+		}),
+		responseMapper: (res) => [res.embedding]
+	});
+}
+/**
+* Create a HuggingFace Inference API embedding provider.
+*
+* @example
+* ```ts
+* const provider = createHuggingFaceEmbedding({
+*   model: 'sentence-transformers/all-MiniLM-L6-v2',
+* });
+* ```
+*/
+function createHuggingFaceEmbedding(options) {
+	const apiKey = options.apiKey ?? process.env.HF_TOKEN ?? process.env.HUGGINGFACE_API_KEY;
+	const headers = {};
+	if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
+	return new HttpEmbeddingProvider({
+		baseUrl: `https://api-inference.huggingface.co/models/${options.model}`,
+		model: options.model,
+		headers,
+		requestBuilder: (texts) => ({ inputs: texts }),
+		responseMapper: (response) => {
+			if (Array.isArray(response)) {
+				if (Array.isArray(response[0]) && typeof response[0][0] === "number") return response;
+				return [response];
+			}
+			throw new Error("Unexpected HuggingFace response format");
+		}
+	});
+}
+/**
+* Create a local Transformers.js embedding provider.
+* Uses dependency injection - user provides the imported transformers module.
+*
+* @example
+* ```typescript
+* import * as transformers from '@huggingface/transformers';
+* import { createTransformersEmbedding } from 'scrapex/embeddings';
+*
+* const provider = createTransformersEmbedding(transformers, {
+*   model: 'Xenova/all-MiniLM-L6-v2',
+* });
+* ```
+*
+* Required Node.js dependencies:
+* ```
+* npm install @huggingface/transformers onnxruntime-node
+* ```
+*/
+function createTransformersEmbedding(transformers, options) {
+	let pipeline = null;
+	let currentModel = null;
+	const config = {
+		model: options?.model ?? "Xenova/all-MiniLM-L6-v2",
+		quantized: options?.quantized ?? true,
+		pooling: options?.pooling ?? "mean",
+		normalize: options?.normalize ?? true
+	};
+	return {
+		name: "transformers",
+		async embed(texts, request) {
+			const model = request.model || config.model;
+			if (!pipeline || currentModel !== model) {
+				const cacheDir = options?.cacheDir;
+				const env = transformers.env;
+				const priorCacheDir = env?.cacheDir;
+				if (cacheDir && env) env.cacheDir = cacheDir;
+				try {
+					pipeline = await transformers.pipeline("feature-extraction", model, { quantized: config.quantized });
+				} finally {
+					if (cacheDir && env) if (priorCacheDir === void 0) delete env.cacheDir;
+					else env.cacheDir = priorCacheDir;
+				}
+				currentModel = model;
+			}
+			const embeddings = [];
+			for (const text of texts) {
+				const output = await pipeline(text, {
+					pooling: config.pooling,
+					normalize: config.normalize
+				});
+				embeddings.push(Array.from(output.data));
+			}
+			return { embeddings };
+		}
+	};
+}
+/** Recommended models for Transformers.js */
+const TRANSFORMERS_MODELS = {
+	DEFAULT: "Xenova/all-MiniLM-L6-v2",
+	QUALITY: "Xenova/all-mpnet-base-v2",
+	RETRIEVAL: "Xenova/bge-small-en-v1.5",
+	MULTILINGUAL: "Xenova/multilingual-e5-small"
+};
+
+//#endregion
+//#region src/embeddings/providers/index.ts
+/**
+* Create an embedding provider from configuration.
+* This is the main factory function for creating providers.
+*/
+function createEmbeddingProvider(config) {
+	switch (config.type) {
+		case "http": return createHttpEmbedding(config.config);
+		case "custom": return config.provider;
+		default: throw new ScrapeError(`Unknown embedding provider type: ${config.type}`, "VALIDATION_ERROR");
+	}
+}
+
+//#endregion
+//#region src/embeddings/safety.ts
+/**
+* PII redaction patterns with high precision to minimize false positives.
+* Patterns are designed to match common formats while avoiding over-matching.
+*/
+const EMAIL_PATTERN = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
+const PHONE_PATTERN = /(?:\+?1[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}\b/g;
+const CREDIT_CARD_PATTERN = /\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12}|(?:[0-9]{4}[-\s]){3}[0-9]{4}|[0-9]{13,19})\b/g;
+const SSN_PATTERN = /\b[0-9]{3}-[0-9]{2}-[0-9]{4}\b/g;
+const IPV4_PATTERN = /\b(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/g;
+const REDACTED = "[REDACTED]";
+/**
+* Create a redaction function based on configuration.
+* Returns a function that applies all configured PII patterns.
+*/
+function createPiiRedactor(config) {
+	const patterns = [];
+	if (config.creditCard) patterns.push({
+		name: "creditCard",
+		pattern: CREDIT_CARD_PATTERN
+	});
+	if (config.email) patterns.push({
+		name: "email",
+		pattern: EMAIL_PATTERN
+	});
+	if (config.phone) patterns.push({
+		name: "phone",
+		pattern: PHONE_PATTERN
+	});
+	if (config.ssn) patterns.push({
+		name: "ssn",
+		pattern: SSN_PATTERN
+	});
+	if (config.ipAddress) patterns.push({
+		name: "ipAddress",
+		pattern: IPV4_PATTERN
+	});
+	if (config.customPatterns) for (let i = 0; i < config.customPatterns.length; i++) {
+		const customPattern = config.customPatterns[i];
+		if (customPattern) patterns.push({
+			name: `custom_${i}`,
+			pattern: customPattern
+		});
+	}
+	return (text) => {
+		let redactedText = text;
+		let totalRedactions = 0;
+		const redactionsByType = {};
+		for (const { name, pattern } of patterns) {
+			pattern.lastIndex = 0;
+			const matchCount = text.match(pattern)?.length ?? 0;
+			if (matchCount > 0) {
+				redactedText = redactedText.replace(pattern, REDACTED);
+				totalRedactions += matchCount;
+				redactionsByType[name] = (redactionsByType[name] ?? 0) + matchCount;
+			}
+		}
+		return {
+			text: redactedText,
+			redacted: totalRedactions > 0,
+			redactionCount: totalRedactions,
+			redactionsByType
+		};
+	};
+}
+/**
+* Simple redaction that applies all default patterns.
+* Use createPiiRedactor() for fine-grained control.
+*/
+function redactPii(text) {
+	return createPiiRedactor({
+		email: true,
+		phone: true,
+		creditCard: true,
+		ssn: true,
+		ipAddress: true
+	})(text);
+}
+
+//#endregion
+//#region src/embeddings/pipeline.ts
+const DEFAULT_CHUNK_SIZE = 500;
+/**
+* Get the effective model for embedding.
+* Prioritizes: explicit options.model > provider config model
+*/
+function getEffectiveModel(providerConfig, explicitModel) {
+	if (explicitModel) return explicitModel;
+	if (providerConfig.type === "http") return providerConfig.config.model;
+}
+/**
+* Generate embeddings for scraped data.
+* This is the main entry point for the embedding pipeline.
+*/
+async function generateEmbeddings(data, options) {
+	const startTime = Date.now();
+	try {
+		const provider = createEmbeddingProvider(options.provider);
+		const model = getEffectiveModel(options.provider, options.model);
+		const validation = validateInput(selectInput(data, options.input), options.safety?.minTextLength ?? 10);
+		if (!validation.valid) return createSkippedResult(validation.reason, { model });
+		const originalInput = validation.text;
+		let inputText = validation.text;
+		let piiRedacted = false;
+		if (options.safety?.piiRedaction) {
+			const redactionResult = createPiiRedactor(options.safety.piiRedaction)(inputText);
+			inputText = redactionResult.text;
+			piiRedacted = redactionResult.redacted;
+		}
+		const effectiveChunking = applyMaxTokensToChunking(options.chunking, options.safety?.maxTokens);
+		const cacheKey = generateCacheKey({
+			providerKey: getProviderCacheKey(options.provider),
+			model,
+			dimensions: options.output?.dimensions,
+			aggregation: options.output?.aggregation,
+			input: options.input,
+			chunking: effectiveChunking,
+			safety: options.safety,
+			cacheKeySalt: options.cache?.cacheKeySalt,
+			content: inputText
+		});
+		const cache = options.cache?.store ?? getDefaultCache();
+		const cachedResult = await cache.get(cacheKey);
+		if (cachedResult && cachedResult.status === "success") {
+			if (options.onMetrics) options.onMetrics({
+				provider: provider.name,
+				model,
+				inputTokens: estimateTokens(inputText),
+				outputDimensions: getDimensions(cachedResult.aggregation === "all" ? cachedResult.vectors : cachedResult.vector),
+				chunks: cachedResult.source.chunks,
+				latencyMs: Date.now() - startTime,
+				cached: true,
+				retries: 0,
+				piiRedacted
+			});
+			return {
+				...cachedResult,
+				source: {
+					...cachedResult.source,
+					cached: true
+				}
+			};
+		}
+		const chunks = chunkText(inputText, effectiveChunking);
+		const callbackChunks = options.onChunk && options.safety?.allowSensitiveCallbacks ? chunkText(originalInput, effectiveChunking) : null;
+		if (chunks.length === 0) return createSkippedResult("No content after chunking", { model });
+		const sharedState = options.resilience?.state;
+		const rateLimiter = sharedState?.rateLimiter ?? (options.resilience?.rateLimit ? new RateLimiter(options.resilience.rateLimit) : null);
+		const circuitBreaker = sharedState?.circuitBreaker ?? (options.resilience?.circuitBreaker ? new CircuitBreaker(options.resilience.circuitBreaker) : null);
+		const concurrency = options.resilience?.concurrency ?? 1;
+		const semaphore = sharedState?.semaphore ?? new Semaphore(concurrency);
+		const embeddings = [];
+		let totalTokens = 0;
+		let retryCount = 0;
+		for (let i = 0; i < chunks.length; i++) {
+			const chunk = chunks[i];
+			if (!chunk) continue;
+			if (rateLimiter) await rateLimiter.acquire();
+			if (circuitBreaker?.isOpen()) return createSkippedResult("Circuit breaker is open", {
+				model,
+				chunks: i
+			});
+			await semaphore.execute(async () => {
+				const { result: result$1 } = await withResilience(async (signal) => {
+					return provider.embed([chunk.text], {
+						model,
+						dimensions: options.output?.dimensions,
+						signal
+					});
+				}, options.resilience, {
+					circuitBreaker: circuitBreaker ?? void 0,
+					rateLimiter: void 0,
+					semaphore: void 0
+				}, { onRetry: () => {
+					retryCount++;
+				} });
+				if (result$1.usage) totalTokens += result$1.usage.totalTokens;
+				else totalTokens += chunk.tokens;
+				const embedding = result$1.embeddings[0];
+				if (embedding) {
+					embeddings.push(embedding);
+					if (options.onChunk) {
+						const callbackText = callbackChunks?.[i]?.text ?? chunk.text;
+						options.onChunk(callbackText, embedding);
+					}
+				}
+			});
+		}
+		const aggregation = options.output?.aggregation ?? "average";
+		const aggregated = aggregateVectors(embeddings, aggregation);
+		const source = {
+			model,
+			chunks: chunks.length,
+			tokens: totalTokens || estimateTokens(inputText),
+			checksum: generateChecksum(inputText),
+			cached: false,
+			latencyMs: Date.now() - startTime
+		};
+		let result;
+		if (aggregated.type === "single") result = {
+			status: "success",
+			aggregation,
+			vector: aggregated.vector,
+			source
+		};
+		else result = {
+			status: "success",
+			aggregation: "all",
+			vectors: aggregated.vectors,
+			source
+		};
+		await cache.set(cacheKey, result, { ttlMs: options.cache?.ttlMs });
+		if (options.onMetrics) {
+			const metrics = {
+				provider: provider.name,
+				model,
+				inputTokens: source.tokens,
+				outputDimensions: aggregated.dimensions,
+				chunks: chunks.length,
+				latencyMs: source.latencyMs,
+				cached: false,
+				retries: retryCount,
+				piiRedacted
+			};
+			options.onMetrics(metrics);
+		}
+		return result;
+	} catch (error) {
+		const reason = error instanceof Error ? error.message : String(error);
+		if (error instanceof ScrapeError && ["INVALID_URL", "BLOCKED"].includes(error.code)) throw error;
+		return createSkippedResult(reason, { latencyMs: Date.now() - startTime });
+	}
+}
+function applyMaxTokensToChunking(chunking, maxTokens) {
+	if (!maxTokens || maxTokens <= 0) return chunking;
+	const baseSize = chunking?.size ?? DEFAULT_CHUNK_SIZE;
+	const baseOverlap = chunking?.overlap ?? 50;
+	const clampedSize = Math.min(baseSize, maxTokens);
+	const clampedOverlap = Math.min(baseOverlap, Math.max(0, clampedSize - 1));
+	return {
+		...chunking,
+		size: clampedSize,
+		overlap: clampedOverlap
+	};
+}
+/**
+* Embed arbitrary text directly.
+* Standalone function for embedding text outside of scrape().
+*/
+async function embed(text, options) {
+	return generateEmbeddings({ textContent: text }, {
+		...options,
+		input: {
+			...options.input,
+			type: "textContent"
+		}
+	});
+}
+/**
+* Embed from existing ScrapedData.
+* Useful when you've already scraped and want to add embeddings later.
+*/
+async function embedScrapedData(data, options) {
+	return generateEmbeddings(data, options);
+}
+/**
+* Create a skipped result with reason.
+*/
+function createSkippedResult(reason, partialSource) {
+	return {
+		status: "skipped",
+		reason,
+		source: partialSource ?? {}
+	};
+}
+
+//#endregion
+//#region src/extractors/content.ts
+const turndown = new TurndownService({
+	headingStyle: "atx",
+	codeBlockStyle: "fenced",
+	bulletListMarker: "-",
+	emDelimiter: "_",
+	strongDelimiter: "**",
+	linkStyle: "inlined"
+});
+turndown.remove([
+	"script",
+	"style",
+	"noscript",
+	"iframe",
+	"nav",
+	"footer"
+]);
+/**
+* Extracts main content using Mozilla Readability.
+* Converts HTML to Markdown for LLM consumption.
+*/
+var ContentExtractor = class {
+	name = "content";
+	priority = 50;
+	async extract(context) {
+		const { options } = context;
+		if (options.extractContent === false) return {};
+		const article = new Readability(context.getDocument().cloneNode(true)).parse();
+		if (!article || !article.content) return this.extractFallback(context);
+		let content = turndown.turndown(article.content);
+		const maxLength = options.maxContentLength ?? 5e4;
+		if (content.length > maxLength) content = `${content.slice(0, maxLength)}\n\n[Content truncated...]`;
+		const textContent = (article.textContent ?? "").trim();
+		const excerpt = this.createExcerpt(textContent);
+		const wordCount = textContent.split(/\s+/).filter(Boolean).length;
+		const contentType = this.detectContentType(context);
+		return {
+			content,
+			textContent,
+			excerpt: article.excerpt || excerpt,
+			wordCount,
+			contentType,
+			title: article.title || void 0,
+			author: article.byline || void 0,
+			siteName: article.siteName || void 0
+		};
+	}
+	extractFallback(context) {
+		const { $ } = context;
+		const bodyHtml = $("body").html() || "";
+		const content = turndown.turndown(bodyHtml);
+		const textContent = $("body").text().replace(/\s+/g, " ").trim();
+		return {
+			content: content.slice(0, context.options.maxContentLength ?? 5e4),
+			textContent,
+			excerpt: this.createExcerpt(textContent),
+			wordCount: textContent.split(/\s+/).filter(Boolean).length,
+			contentType: "unknown"
+		};
+	}
+	createExcerpt(text, maxLength = 300) {
+		if (text.length <= maxLength) return text;
+		const truncated = text.slice(0, maxLength);
+		const lastSpace = truncated.lastIndexOf(" ");
+		return `${lastSpace > 0 ? truncated.slice(0, lastSpace) : truncated}...`;
+	}
+	detectContentType(context) {
+		const { $, finalUrl } = context;
+		const url = finalUrl.toLowerCase();
+		if (url.includes("github.com") && !url.includes("/blob/") && !url.includes("/issues/")) {
+			if ($("meta[property=\"og:type\"]").attr("content") === "object" || url.match(/github\.com\/[^/]+\/[^/]+\/?$/)) return "repo";
+		}
+		if (url.includes("npmjs.com/package/")) return "package";
+		if (url.includes("pypi.org/project/")) return "package";
+		if (url.includes("/docs/") || url.includes(".readthedocs.") || url.includes("/documentation/")) return "docs";
+		if (url.includes("youtube.com") || url.includes("vimeo.com") || url.includes("youtu.be")) return "video";
+		const hasPrice = $("[class*=\"price\"], [data-price], [itemprop=\"price\"]").length > 0;
+		const hasAddToCart = $("[class*=\"cart\"], [class*=\"buy\"], button:contains(\"Add\")").length > 0;
+		if (hasPrice || hasAddToCart) return "product";
+		const ogType = $("meta[property=\"og:type\"]").attr("content")?.toLowerCase();
+		if (ogType === "article" || ogType === "blog" || ogType === "news") return "article";
+		const hasArticleTag = $("article").length > 0;
+		const hasDateline = $("time[datetime], [class*=\"date\"], [class*=\"byline\"]").length > 0;
+		if (hasArticleTag && hasDateline) return "article";
+		return "unknown";
+	}
+};
+
+//#endregion
+//#region src/utils/url.ts
+/**
+* Common tracking parameters to remove from URLs
+*/
+const TRACKING_PARAMS = [
+	"utm_source",
+	"utm_medium",
+	"utm_campaign",
+	"utm_term",
+	"utm_content",
+	"utm_id",
+	"ref",
+	"fbclid",
+	"gclid",
+	"gclsrc",
+	"dclid",
+	"msclkid",
+	"mc_cid",
+	"mc_eid",
+	"_ga",
+	"_gl",
+	"source",
+	"referrer"
+];
+/**
+* Validate if a string is a valid URL
+*/
+function isValidUrl(url) {
+	try {
+		const parsed = new URL(url);
+		return ["http:", "https:"].includes(parsed.protocol);
+	} catch {
+		return false;
+	}
+}
+/**
+* Normalize URL by removing tracking params and trailing slashes
+*/
+function normalizeUrl(url) {
+	try {
+		const parsed = new URL(url);
+		for (const param of TRACKING_PARAMS) parsed.searchParams.delete(param);
+		let normalized = parsed.toString();
+		if (normalized.endsWith("/") && parsed.pathname !== "/") normalized = normalized.slice(0, -1);
+		return normalized;
+	} catch {
+		return url;
+	}
+}
+/**
+* Extract domain from URL (without www prefix)
+*/
+function extractDomain(url) {
+	try {
+		return new URL(url).hostname.replace(/^www\./, "");
+	} catch {
+		return "";
+	}
+}
+/**
+* Resolve a potentially relative URL against a base URL
+*/
+function resolveUrl(url, baseUrl) {
+	if (!url) return void 0;
+	try {
+		return new URL(url, baseUrl).href;
+	} catch {
+		return url;
+	}
+}
+/**
+* Check if a URL is external relative to a domain
+*/
+function isExternalUrl(url, baseDomain) {
+	try {
+		return new URL(url).hostname.replace(/^www\./, "") !== baseDomain;
+	} catch {
+		return false;
+	}
+}
+/**
+* Extract protocol from URL
+*/
+function getProtocol(url) {
+	try {
+		return new URL(url).protocol;
+	} catch {
+		return "";
+	}
+}
+/**
+* Get the path portion of a URL
+*/
+function getPath(url) {
+	try {
+		return new URL(url).pathname;
+	} catch {
+		return "";
+	}
+}
+/**
+* Check if URL matches a pattern (supports * wildcard)
+*/
+function matchesUrlPattern(url, pattern) {
+	if (!pattern.includes("*")) return url === pattern || url.startsWith(pattern);
+	const regexPattern = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+	return (/* @__PURE__ */ new RegExp(`^${regexPattern}`)).test(url);
+}
+
+//#endregion
+//#region src/extractors/favicon.ts
+/**
+* Extracts favicon URL from the page.
+* Checks multiple sources in order of preference.
+*/
+var FaviconExtractor = class {
+	name = "favicon";
+	priority = 70;
+	async extract(context) {
+		const { $, finalUrl } = context;
+		for (const selector of [
+			"link[rel=\"icon\"][type=\"image/svg+xml\"]",
+			"link[rel=\"icon\"][sizes=\"192x192\"]",
+			"link[rel=\"icon\"][sizes=\"180x180\"]",
+			"link[rel=\"icon\"][sizes=\"128x128\"]",
+			"link[rel=\"icon\"][sizes=\"96x96\"]",
+			"link[rel=\"apple-touch-icon\"][sizes=\"180x180\"]",
+			"link[rel=\"apple-touch-icon\"]",
+			"link[rel=\"icon\"][sizes=\"32x32\"]",
+			"link[rel=\"icon\"]",
+			"link[rel=\"shortcut icon\"]"
+		]) {
+			const href = $(selector).first().attr("href");
+			if (href) return { favicon: resolveUrl(finalUrl, href) };
+		}
+		try {
+			const url = new URL(finalUrl);
+			return { favicon: `${url.protocol}//${url.host}/favicon.ico` };
+		} catch {
+			return {};
+		}
+	}
+};
+
+//#endregion
+//#region src/extractors/jsonld.ts
+/**
+* Extracts JSON-LD structured data from the page.
+* Also extracts additional metadata from structured data.
+*/
+var JsonLdExtractor = class {
+	name = "jsonld";
+	priority = 80;
+	async extract(context) {
+		const { $ } = context;
+		const jsonLd = [];
+		$("script[type=\"application/ld+json\"]").each((_, el) => {
+			const content = $(el).html();
+			if (!content) return;
+			try {
+				const parsed = JSON.parse(content);
+				if (Array.isArray(parsed)) jsonLd.push(...parsed);
+				else if (typeof parsed === "object" && parsed !== null) jsonLd.push(parsed);
+			} catch {}
+		});
+		if (jsonLd.length === 0) return {};
+		return {
+			jsonLd,
+			...this.extractMetadata(jsonLd)
+		};
+	}
+	extractMetadata(jsonLd) {
+		const result = {};
+		for (const item of jsonLd) {
+			const type = this.getType(item);
+			if (type?.match(/Article|BlogPosting|NewsArticle|WebPage/i)) {
+				result.title = result.title || this.getString(item, "headline", "name");
+				result.description = result.description || this.getString(item, "description");
+				result.author = result.author || this.getAuthor(item);
+				result.publishedAt = result.publishedAt || this.getString(item, "datePublished");
+				result.modifiedAt = result.modifiedAt || this.getString(item, "dateModified");
+				result.image = result.image || this.getImage(item);
+			}
+			if (type === "Organization") result.siteName = result.siteName || this.getString(item, "name");
+			if (type === "Product") {
+				result.title = result.title || this.getString(item, "name");
+				result.description = result.description || this.getString(item, "description");
+				result.image = result.image || this.getImage(item);
+			}
+			if (type === "SoftwareApplication") {
+				result.title = result.title || this.getString(item, "name");
+				result.description = result.description || this.getString(item, "description");
+			}
+			const keywords = this.getKeywords(item);
+			if (keywords.length > 0) result.keywords = [...result.keywords || [], ...keywords];
+		}
+		if (result.keywords) result.keywords = [...new Set(result.keywords)];
+		return result;
+	}
+	getType(item) {
+		const type = item["@type"];
+		if (typeof type === "string") return type;
+		if (Array.isArray(type)) return type[0];
+	}
+	getString(item, ...keys) {
+		for (const key of keys) {
+			const value = item[key];
+			if (typeof value === "string") return value;
+			if (typeof value === "object" && value !== null && "@value" in value) return String(value["@value"]);
+		}
+	}
+	getAuthor(item) {
+		const author = item.author;
+		if (typeof author === "string") return author;
+		if (Array.isArray(author)) return author.map((a) => typeof a === "string" ? a : this.getString(a, "name")).filter(Boolean).join(", ") || void 0;
+		if (typeof author === "object" && author !== null) {
+			const authorObj = author;
+			return this.getString(authorObj, "name") || void 0;
+		}
+	}
+	getImage(item) {
+		const image = item.image;
+		if (typeof image === "string") return image;
+		if (Array.isArray(image) && image.length > 0) return this.getImage({ image: image[0] });
+		if (typeof image === "object" && image !== null) {
+			const imageObj = image;
+			return this.getString(imageObj, "url", "contentUrl") || void 0;
+		}
+	}
+	getKeywords(item) {
+		const keywords = item.keywords;
+		if (typeof keywords === "string") return keywords.split(",").map((k) => k.trim()).filter(Boolean);
+		if (Array.isArray(keywords)) return keywords.filter((k) => typeof k === "string");
+		return [];
+	}
+};
+
+//#endregion
+//#region src/extractors/links.ts
+/**
+* Extracts links from the page content.
+* Filters out navigation/footer links and focuses on content links.
+*/
+var LinksExtractor = class {
+	name = "links";
+	priority = 30;
+	async extract(context) {
+		const { $, finalUrl } = context;
+		const links = [];
+		const seen = /* @__PURE__ */ new Set();
+		const contentArea = $("article, main, [role=\"main\"]").first();
+		const container = contentArea.length > 0 ? contentArea : $("body");
+		const skipSelectors = "nav, header, footer, aside, [role=\"navigation\"], [class*=\"nav\"], [class*=\"footer\"], [class*=\"header\"], [class*=\"sidebar\"], [class*=\"menu\"]";
+		container.find("a[href]").each((_, el) => {
+			const $el = $(el);
+			if ($el.closest(skipSelectors).length > 0) return;
+			const href = $el.attr("href");
+			if (!href) return;
+			if (href.startsWith("#") || href.startsWith("javascript:") || href.startsWith("mailto:") || href.startsWith("tel:")) return;
+			const resolvedUrl = resolveUrl(href, finalUrl);
+			if (!resolvedUrl || !isValidUrl(resolvedUrl)) return;
+			if (seen.has(resolvedUrl)) return;
+			seen.add(resolvedUrl);
+			const text = $el.text().trim() || $el.attr("title") || $el.attr("aria-label") || "";
+			if (text.length < 2) return;
+			const baseDomain = extractDomain(finalUrl);
+			links.push({
+				url: resolvedUrl,
+				text: text.slice(0, 200),
+				isExternal: isExternalUrl(resolvedUrl, baseDomain)
+			});
+		});
+		return { links: links.slice(0, 100) };
+	}
+};
+
+//#endregion
+//#region src/extractors/meta.ts
+/**
+* Extracts metadata from HTML meta tags, Open Graph, and Twitter cards.
+* Runs first to provide basic metadata for other extractors.
+*/
+var MetaExtractor = class {
+	name = "meta";
+	priority = 100;
+	async extract(context) {
+		const { $ } = context;
+		const getMeta = (nameOrProperty) => {
+			return ($(`meta[name="${nameOrProperty}"]`).attr("content") || $(`meta[property="${nameOrProperty}"]`).attr("content") || $(`meta[itemprop="${nameOrProperty}"]`).attr("content"))?.trim() || void 0;
+		};
+		const title = getMeta("og:title") || getMeta("twitter:title") || $("title").first().text().trim() || "";
+		const description = getMeta("og:description") || getMeta("twitter:description") || getMeta("description") || "";
+		const image = getMeta("og:image") || getMeta("twitter:image") || getMeta("twitter:image:src") || void 0;
+		const canonicalUrl = $("link[rel=\"canonical\"]").attr("href") || getMeta("og:url") || context.finalUrl;
+		const author = getMeta("author") || getMeta("article:author") || getMeta("twitter:creator") || $("[rel=\"author\"]").first().text().trim() || void 0;
+		const siteName = getMeta("og:site_name") || getMeta("application-name") || void 0;
+		const publishedAt = getMeta("article:published_time") || getMeta("datePublished") || getMeta("date") || $("time[datetime]").first().attr("datetime") || void 0;
+		const modifiedAt = getMeta("article:modified_time") || getMeta("dateModified") || void 0;
+		const language = $("html").attr("lang") || getMeta("og:locale") || getMeta("language") || void 0;
+		const keywordsRaw = getMeta("keywords") || getMeta("article:tag") || "";
+		return {
+			title,
+			description,
+			image,
+			canonicalUrl,
+			author,
+			siteName,
+			publishedAt,
+			modifiedAt,
+			language,
+			keywords: keywordsRaw ? keywordsRaw.split(",").map((k) => k.trim()).filter(Boolean) : []
+		};
+	}
+};
+
+//#endregion
+//#region src/extractors/index.ts
+/**
+* Default extractors in priority order.
+* Higher priority runs first.
+*/
+function createDefaultExtractors() {
+	return [
+		new MetaExtractor(),
+		new JsonLdExtractor(),
+		new FaviconExtractor(),
+		new ContentExtractor(),
+		new LinksExtractor()
+	];
+}
+/**
+* Sort extractors by priority (higher first).
+*/
+function sortExtractors(extractors) {
+	return [...extractors].sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
+}
+
+//#endregion
+//#region src/fetchers/types.ts
+/**
+* Default user agent string
+*/
+const DEFAULT_USER_AGENT = "Scrapex-Bot/2.0 (+https://github.com/developer-rakeshpaul/scrapex)";
+/**
+* Default timeout in milliseconds
+*/
+const DEFAULT_TIMEOUT = 1e4;
+
+//#endregion
+//#region src/fetchers/fetch.ts
+/**
+* Default fetcher using native fetch API.
+* Works in Node.js 18+ without polyfills.
+*/
+var NativeFetcher = class {
+	name = "native-fetch";
+	async fetch(url, options = {}) {
+		const { timeout = DEFAULT_TIMEOUT, userAgent = DEFAULT_USER_AGENT, headers = {} } = options;
+		let parsedUrl;
+		try {
+			parsedUrl = new URL(url);
+		} catch {
+			throw new ScrapeError(`Invalid URL: ${url}`, "INVALID_URL");
+		}
+		if (!["http:", "https:"].includes(parsedUrl.protocol)) throw new ScrapeError(`Invalid protocol: ${parsedUrl.protocol}`, "INVALID_URL");
+		const controller = new AbortController();
+		const timeoutId = setTimeout(() => controller.abort(), timeout);
+		try {
+			const response = await fetch(url, {
+				signal: controller.signal,
+				headers: {
+					"User-Agent": userAgent,
+					Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+					"Accept-Language": "en-US,en;q=0.5",
+					...headers
+				},
+				redirect: "follow"
+			});
+			clearTimeout(timeoutId);
+			if (!response.ok) {
+				if (response.status === 404) throw new ScrapeError(`Page not found: ${url}`, "NOT_FOUND", 404);
+				if (response.status === 403 || response.status === 401) throw new ScrapeError(`Access blocked: ${url}`, "BLOCKED", response.status);
+				if (response.status === 429) throw new ScrapeError(`Rate limited: ${url}`, "BLOCKED", 429);
+				throw new ScrapeError(`HTTP error ${response.status}: ${url}`, "FETCH_FAILED", response.status);
+			}
+			const contentType = response.headers.get("content-type") || "";
+			if (options.allowedContentTypes) {
+				if (!options.allowedContentTypes.some((type) => contentType.toLowerCase().includes(type.toLowerCase()))) throw new ScrapeError(`Unexpected content type: ${contentType}`, "PARSE_ERROR");
+			} else if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) throw new ScrapeError(`Unexpected content type: ${contentType}`, "PARSE_ERROR");
+			const html = await response.text();
+			const responseHeaders = {};
+			response.headers.forEach((value, key) => {
+				responseHeaders[key] = value;
+			});
+			return {
+				html,
+				finalUrl: response.url,
+				statusCode: response.status,
+				contentType,
+				headers: responseHeaders
+			};
+		} catch (error) {
+			clearTimeout(timeoutId);
+			if (error instanceof ScrapeError) throw error;
+			if (error instanceof Error && error.name === "AbortError") throw new ScrapeError(`Request timed out after ${timeout}ms`, "TIMEOUT");
+			if (error instanceof Error) throw new ScrapeError(`Fetch failed: ${error.message}`, "FETCH_FAILED", void 0, error);
+			throw new ScrapeError("Unknown fetch error", "FETCH_FAILED");
+		}
+	}
+};
+/**
+* Default fetcher instance
+*/
+const defaultFetcher = new NativeFetcher();
+
+//#endregion
+//#region src/fetchers/robots.ts
+/**
+* Check if URL is allowed by robots.txt
+*
+* @param url - The URL to check
+* @param userAgent - User agent to check rules for
+* @returns Whether the URL is allowed and optional reason
+*/
+async function checkRobotsTxt(url, userAgent = DEFAULT_USER_AGENT) {
+	try {
+		const parsedUrl = new URL(url);
+		const robotsUrl = `${parsedUrl.protocol}//${parsedUrl.host}/robots.txt`;
+		const response = await fetch(robotsUrl, {
+			headers: { "User-Agent": userAgent },
+			signal: AbortSignal.timeout(5e3)
+		});
+		if (!response.ok) return { allowed: true };
+		const allowed = isPathAllowed(parseRobotsTxt(await response.text(), userAgent), parsedUrl.pathname + parsedUrl.search);
+		return {
+			allowed,
+			reason: allowed ? void 0 : "Blocked by robots.txt"
+		};
+	} catch {
+		return { allowed: true };
+	}
+}
+/**
+* Parse robots.txt content for a specific user agent
+*/
+function parseRobotsTxt(content, userAgent) {
+	const rules = {
+		disallow: [],
+		allow: []
+	};
+	const lines = content.split("\n");
+	const botName = userAgent.split(/[\s/]/)[0]?.toLowerCase() || "";
+	let currentAgent = "";
+	let isMatchingAgent = false;
+	let hasFoundSpecificAgent = false;
+	for (const rawLine of lines) {
+		const line = rawLine.trim();
+		if (!line || line.startsWith("#")) continue;
+		const colonIndex = line.indexOf(":");
+		if (colonIndex === -1) continue;
+		const directive = line.slice(0, colonIndex).trim().toLowerCase();
+		const value = line.slice(colonIndex + 1).trim();
+		if (directive === "user-agent") {
+			currentAgent = value.toLowerCase();
+			isMatchingAgent = currentAgent === "*" || currentAgent === botName || botName.includes(currentAgent);
+			if (currentAgent !== "*" && isMatchingAgent) {
+				hasFoundSpecificAgent = true;
+				rules.disallow = [];
+				rules.allow = [];
+			}
+		} else if (isMatchingAgent && (!hasFoundSpecificAgent || currentAgent !== "*")) {
+			if (directive === "disallow" && value) rules.disallow.push(value);
+			else if (directive === "allow" && value) rules.allow.push(value);
+		}
+	}
+	return rules;
+}
+/**
+* Check if a path is allowed based on robots.txt rules
+*/
+function isPathAllowed(rules, path) {
+	if (rules.disallow.length === 0 && rules.allow.length === 0) return true;
+	for (const pattern of rules.allow) if (matchesPattern(path, pattern)) return true;
+	for (const pattern of rules.disallow) if (matchesPattern(path, pattern)) return false;
+	return true;
+}
+/**
+* Check if a path matches a robots.txt pattern
+*/
+function matchesPattern(path, pattern) {
+	if (!pattern) return false;
+	if (pattern.endsWith("*")) return path.startsWith(pattern.slice(0, -1));
+	if (pattern.endsWith("$")) return path === pattern.slice(0, -1);
+	if (pattern.includes("*")) return (/* @__PURE__ */ new RegExp(`^${pattern.replace(/\*/g, ".*").replace(/\?/g, "\\?")}.*`)).test(path);
+	return path.startsWith(pattern);
+}
+
+//#endregion
+//#region src/core/scrape.ts
+/**
+* Scrape a URL and extract metadata and content.
+*
+* @param url - The URL to scrape
+* @param options - Scraping options
+* @returns Scraped data with metadata and content
+*
+* @example
+* ```ts
+* const result = await scrape('https://example.com/article');
+* console.log(result.title, result.content);
+* ```
+*/
+async function scrape(url, options = {}) {
+	const startTime = Date.now();
+	if (!isValidUrl(url)) throw new ScrapeError("Invalid URL provided", "INVALID_URL");
+	const normalizedUrl = normalizeUrl(url);
+	if (options.respectRobots) {
+		const robotsResult = await checkRobotsTxt(normalizedUrl, options.userAgent);
+		if (!robotsResult.allowed) throw new ScrapeError(`URL blocked by robots.txt: ${robotsResult.reason || "disallowed"}`, "ROBOTS_BLOCKED");
+	}
+	const fetchResult = await (options.fetcher ?? defaultFetcher).fetch(normalizedUrl, {
+		timeout: options.timeout,
+		userAgent: options.userAgent
+	});
+	await preloadJsdom();
+	let context = createExtractionContext(normalizedUrl, fetchResult.finalUrl, fetchResult.html, options);
+	let extractors;
+	if (options.replaceDefaultExtractors) extractors = options.extractors ?? [];
+	else {
+		const defaults = createDefaultExtractors();
+		extractors = options.extractors ? [...defaults, ...options.extractors] : defaults;
+	}
+	extractors = sortExtractors(extractors);
+	for (const extractor of extractors) try {
+		const extracted = await extractor.extract(context);
+		context = mergeResults(context, extracted);
+	} catch (error) {
+		console.error(`Extractor "${extractor.name}" failed:`, error);
+		context = mergeResults(context, { error: context.results.error ? `${context.results.error}; ${extractor.name}: ${error instanceof Error ? error.message : String(error)}` : `${extractor.name}: ${error instanceof Error ? error.message : String(error)}` });
+	}
+	const intermediateResult = {
+		url: normalizedUrl,
+		canonicalUrl: context.results.canonicalUrl || fetchResult.finalUrl,
+		domain: extractDomain(fetchResult.finalUrl),
+		title: context.results.title || "",
+		description: context.results.description || "",
+		image: context.results.image,
+		favicon: context.results.favicon,
+		content: context.results.content || "",
+		textContent: context.results.textContent || "",
+		excerpt: context.results.excerpt || "",
+		wordCount: context.results.wordCount || 0,
+		author: context.results.author,
+		publishedAt: context.results.publishedAt,
+		modifiedAt: context.results.modifiedAt,
+		siteName: context.results.siteName,
+		language: context.results.language,
+		contentType: context.results.contentType || "unknown",
+		keywords: context.results.keywords || [],
+		jsonLd: context.results.jsonLd,
+		links: context.results.links,
+		custom: context.results.custom,
+		scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		scrapeTimeMs: 0,
+		error: context.results.error
+	};
+	if (options.llm && options.enhance && options.enhance.length > 0) try {
+		const enhanced = await enhance(intermediateResult, options.llm, options.enhance);
+		Object.assign(intermediateResult, enhanced);
+	} catch (error) {
+		console.error("LLM enhancement failed:", error);
+		intermediateResult.error = intermediateResult.error ? `${intermediateResult.error}; LLM: ${error instanceof Error ? error.message : String(error)}` : `LLM: ${error instanceof Error ? error.message : String(error)}`;
+	}
+	if (options.llm && options.extract) try {
+		intermediateResult.extracted = await extract(intermediateResult, options.llm, options.extract);
+	} catch (error) {
+		console.error("LLM extraction failed:", error);
+		intermediateResult.error = intermediateResult.error ? `${intermediateResult.error}; LLM extraction: ${error instanceof Error ? error.message : String(error)}` : `LLM extraction: ${error instanceof Error ? error.message : String(error)}`;
+	}
+	if (options.embeddings) intermediateResult.embeddings = await generateEmbeddings(intermediateResult, options.embeddings);
+	const scrapeTimeMs = Date.now() - startTime;
+	return {
+		...intermediateResult,
+		scrapeTimeMs
+	};
+}
+/**
+* Scrape from raw HTML string (no fetch).
+*
+* @param html - The HTML content
+* @param url - The URL (for resolving relative links)
+* @param options - Scraping options
+* @returns Scraped data with metadata and content
+*
+* @example
+* ```ts
+* const html = await fetchSomehow('https://example.com');
+* const result = await scrapeHtml(html, 'https://example.com');
+* ```
+*/
+async function scrapeHtml(html, url, options = {}) {
+	const startTime = Date.now();
+	if (!isValidUrl(url)) throw new ScrapeError("Invalid URL provided", "INVALID_URL");
+	const normalizedUrl = normalizeUrl(url);
+	await preloadJsdom();
+	let context = createExtractionContext(normalizedUrl, normalizedUrl, html, options);
+	let extractors;
+	if (options.replaceDefaultExtractors) extractors = options.extractors ?? [];
+	else {
+		const defaults = createDefaultExtractors();
+		extractors = options.extractors ? [...defaults, ...options.extractors] : defaults;
+	}
+	extractors = sortExtractors(extractors);
+	for (const extractor of extractors) try {
+		const extracted = await extractor.extract(context);
+		context = mergeResults(context, extracted);
+	} catch (error) {
+		console.error(`Extractor "${extractor.name}" failed:`, error);
+		context = mergeResults(context, { error: context.results.error ? `${context.results.error}; ${extractor.name}: ${error instanceof Error ? error.message : String(error)}` : `${extractor.name}: ${error instanceof Error ? error.message : String(error)}` });
+	}
+	const domain = extractDomain(normalizedUrl);
+	const intermediateResult = {
+		url: normalizedUrl,
+		canonicalUrl: context.results.canonicalUrl || normalizedUrl,
+		domain,
+		title: context.results.title || "",
+		description: context.results.description || "",
+		image: context.results.image,
+		favicon: context.results.favicon,
+		content: context.results.content || "",
+		textContent: context.results.textContent || "",
+		excerpt: context.results.excerpt || "",
+		wordCount: context.results.wordCount || 0,
+		author: context.results.author,
+		publishedAt: context.results.publishedAt,
+		modifiedAt: context.results.modifiedAt,
+		siteName: context.results.siteName,
+		language: context.results.language,
+		contentType: context.results.contentType || "unknown",
+		keywords: context.results.keywords || [],
+		jsonLd: context.results.jsonLd,
+		links: context.results.links,
+		summary: context.results.summary,
+		suggestedTags: context.results.suggestedTags,
+		entities: context.results.entities,
+		extracted: context.results.extracted,
+		custom: context.results.custom,
+		scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		scrapeTimeMs: 0,
+		error: context.results.error
+	};
+	if (options.embeddings) intermediateResult.embeddings = await generateEmbeddings(intermediateResult, options.embeddings);
+	const scrapeTimeMs = Date.now() - startTime;
+	return {
+		...intermediateResult,
+		scrapeTimeMs
+	};
+}
+
+//#endregion
+//#region src/utils/feed.ts
+/**
+* Fetch and parse an RSS/Atom feed from a URL.
+* Uses scrapex's fetcher infrastructure for consistent behavior.
+*/
+async function fetchFeed(url, options) {
+	const result = await (options?.fetcher || defaultFetcher).fetch(url, {
+		timeout: options?.timeout,
+		userAgent: options?.userAgent,
+		allowedContentTypes: [
+			"application/rss+xml",
+			"application/atom+xml",
+			"application/rdf+xml",
+			"application/xml",
+			"text/xml",
+			"text/html"
+		]
+	});
+	return new RSSParser(options?.parserOptions).parse(result.html, url);
+}
+/**
+* Detect RSS/Atom feed URLs from HTML.
+* Supports RSS, Atom, and RDF feed types.
+*/
+function discoverFeeds(html, baseUrl) {
+	const $ = cheerio.load(html);
+	const feeds = [];
+	const seen = /* @__PURE__ */ new Set();
+	$([
+		"link[type=\"application/rss+xml\"]",
+		"link[type=\"application/atom+xml\"]",
+		"link[type=\"application/rdf+xml\"]",
+		"link[rel=\"alternate\"][type*=\"xml\"]"
+	].join(", ")).each((_, el) => {
+		const href = $(el).attr("href");
+		if (href) try {
+			const resolved = new URL(href, baseUrl).href;
+			if (!seen.has(resolved)) {
+				seen.add(resolved);
+				feeds.push(resolved);
+			}
+		} catch {}
+	});
+	return feeds;
+}
+/**
+* Filter feed items by date range.
+* Items without publishedAt are included by default.
+*/
+function filterByDate(items, options) {
+	const { after, before, includeUndated = true } = options;
+	return items.filter((item) => {
+		if (!item.publishedAt) return includeUndated;
+		const date = new Date(item.publishedAt);
+		if (after && date < after) return false;
+		if (before && date > before) return false;
+		return true;
+	});
+}
+/**
+* Convert feed items to markdown for LLM consumption.
+* Uses ISO 8601 date format for consistency across environments.
+*/
+function feedToMarkdown(feed, options) {
+	const { includeContent = false, maxItems } = options || {};
+	const lines = [`# ${feed.title}`, ""];
+	if (feed.description) lines.push(feed.description, "");
+	const items = maxItems ? feed.items.slice(0, maxItems) : feed.items;
+	for (const item of items) {
+		lines.push(`## ${item.title}`);
+		if (item.publishedAt) {
+			const date = item.publishedAt.split("T")[0];
+			lines.push(`*${date}*`);
+		}
+		lines.push("");
+		if (includeContent && item.content) lines.push(item.content);
+		else if (item.description) lines.push(item.description);
+		if (item.link) lines.push(`[Read more](${item.link})`, "");
+		else lines.push("");
+	}
+	return lines.join("\n");
+}
+/**
+* Extract plain text from feed items for LLM processing.
+* Concatenates title, description, and content.
+*/
+function feedToText(feed, options) {
+	const { maxItems, separator = "\n\n---\n\n" } = options || {};
+	return (maxItems ? feed.items.slice(0, maxItems) : feed.items).map((item) => {
+		const parts = [item.title];
+		if (item.description) parts.push(item.description);
+		if (item.content) parts.push(item.content);
+		return parts.join("\n\n");
+	}).join(separator);
+}
+/**
+* Paginate through a feed using rel="next" links (RFC 5005).
+* Returns an async generator that yields each page.
+*/
+async function* paginateFeed(url, options) {
+	const { maxPages = 10, ...fetchOptions } = options || {};
+	let currentUrl = url;
+	let pageCount = 0;
+	while (currentUrl && pageCount < maxPages) {
+		const result = await fetchFeed(currentUrl, fetchOptions);
+		yield result.data;
+		currentUrl = result.data.next;
+		pageCount++;
+	}
+}
+
+//#endregion
+export { ContentExtractor, DEFAULT_TIMEOUT, DEFAULT_USER_AGENT, FaviconExtractor, InMemoryEmbeddingCache, JsonLdExtractor, LinksExtractor, MetaExtractor, NativeFetcher, RSSParser, ScrapeError, TRANSFORMERS_MODELS, aggregateVectors, checkRobotsTxt, chunkText, cosineSimilarity, createAzureEmbedding, createDefaultExtractors, createEmbeddingProvider, createExtractionContext, createHttpEmbedding, createHuggingFaceEmbedding, createOllamaEmbedding, createOpenAIEmbedding, createPiiRedactor, createTransformersEmbedding, defaultFetcher, discoverFeeds, embed, embedScrapedData, estimateTokens, extractDomain, feedToMarkdown, feedToText, fetchFeed, filterByDate, generateEmbeddings, getPath, getProtocol, isExternalUrl, isValidUrl, matchesUrlPattern, mergeResults, normalizeUrl, paginateFeed, redactPii, resolveUrl, scrape, scrapeHtml, sortExtractors };
+//# sourceMappingURL=index.mjs.map