scrapex 1.0.0-alpha.1 → 1.0.0-beta.1

package/dist/index.mjs

@@ -1,5 +1,7 @@
-import { c as ScrapeError, n as enhance, r as extract } from "./enhancer-Q6CSc1gA.mjs";
+import { c as BaseHttpProvider, d as Semaphore, f as withResilience, l as CircuitBreaker, n as enhance, p as ScrapeError, r as extract, u as RateLimiter } from "./enhancer-ByjRD-t5.mjs";
+import { t as RSSParser } from "./parsers-CwkYnyWY.mjs";
 import * as cheerio from "cheerio";
+import { createHash } from "node:crypto";
 import { Readability } from "@mozilla/readability";
 import TurndownService from "turndown";
 
@@ -52,6 +54,1042 @@ function mergeResults(context, extracted) {
 	};
 }
 
+//#endregion
+//#region src/embeddings/aggregation.ts
+/**
+* Aggregate multiple embedding vectors into a single vector or return all.
+*
+* @param vectors - Array of embedding vectors (must all have same dimensions)
+* @param strategy - Aggregation strategy
+* @returns Aggregated result based on strategy
+*/
+function aggregateVectors(vectors, strategy = "average") {
+	if (vectors.length === 0) throw new Error("Cannot aggregate empty vector array");
+	const firstVector = vectors[0];
+	if (!firstVector) throw new Error("Cannot aggregate empty vector array");
+	const dimensions = firstVector.length;
+	for (let i = 1; i < vectors.length; i++) {
+		const vec = vectors[i];
+		if (!vec || vec.length !== dimensions) throw new Error(`Vector dimension mismatch: expected ${dimensions}, got ${vec?.length ?? 0} at index ${i}`);
+	}
+	switch (strategy) {
+		case "average": return {
+			type: "single",
+			vector: averageVectors(vectors),
+			dimensions
+		};
+		case "max": return {
+			type: "single",
+			vector: maxPoolVectors(vectors),
+			dimensions
+		};
+		case "first": return {
+			type: "single",
+			vector: firstVector,
+			dimensions
+		};
+		case "all": return {
+			type: "multiple",
+			vectors,
+			dimensions
+		};
+		default: {
+			const _exhaustive = strategy;
+			throw new Error(`Unknown aggregation strategy: ${_exhaustive}`);
+		}
+	}
+}
+/**
+* Compute element-wise average of vectors.
+*/
+function averageVectors(vectors) {
+	const first = vectors[0];
+	if (!first || vectors.length === 1) return first ?? [];
+	const dimensions = first.length;
+	const count = vectors.length;
+	const result = new Array(dimensions).fill(0);
+	for (const vector of vectors) for (let i = 0; i < dimensions; i++) {
+		const val = result[i];
+		if (val !== void 0) result[i] = val + (vector[i] ?? 0);
+	}
+	for (let i = 0; i < dimensions; i++) {
+		const val = result[i];
+		if (val !== void 0) result[i] = val / count;
+	}
+	return result;
+}
+/**
+* Compute element-wise maximum of vectors (max pooling).
+*/
+function maxPoolVectors(vectors) {
+	const first = vectors[0];
+	if (!first || vectors.length === 1) return first ?? [];
+	const dimensions = first.length;
+	const result = [...first];
+	for (let v = 1; v < vectors.length; v++) {
+		const vec = vectors[v];
+		if (!vec) continue;
+		for (let i = 0; i < dimensions; i++) {
+			const val = vec[i] ?? 0;
+			if (val > (result[i] ?? 0)) result[i] = val;
+		}
+	}
+	return result;
+}
+/**
+* Compute cosine similarity between two vectors.
+* Both vectors should be normalized for accurate results.
+*/
+function cosineSimilarity(a, b) {
+	if (a.length !== b.length) throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+	let dot = 0;
+	let magnitudeA = 0;
+	let magnitudeB = 0;
+	for (let i = 0; i < a.length; i++) {
+		const aVal = a[i] ?? 0;
+		const bVal = b[i] ?? 0;
+		dot += aVal * bVal;
+		magnitudeA += aVal * aVal;
+		magnitudeB += bVal * bVal;
+	}
+	const magnitude = Math.sqrt(magnitudeA) * Math.sqrt(magnitudeB);
+	if (magnitude === 0) return 0;
+	return dot / magnitude;
+}
+/**
+* Get the dimensions of a vector or set of vectors.
+*/
+function getDimensions(vectors) {
+	if (vectors.length === 0) return 0;
+	const first = vectors[0];
+	if (typeof first === "number") return vectors.length;
+	return first?.length ?? 0;
+}
+
+//#endregion
+//#region src/embeddings/cache.ts
+/**
+* Default maximum cache entries.
+*/
+const DEFAULT_MAX_ENTRIES = 1e3;
+/**
+* Default TTL in milliseconds (1 hour).
+*/
+const DEFAULT_TTL_MS = 3600 * 1e3;
+/**
+* Generate a content-addressable cache key.
+* Key is based on content hash and embedding configuration.
+* Note: custom RegExp patterns are serialized by source+flags; different
+* constructions can yield different cache keys even if equivalent.
+*/
+function generateCacheKey(params) {
+	const hash = createHash("sha256");
+	const fingerprint = stableStringify({
+		providerKey: params.providerKey,
+		model: params.model ?? "provider-default",
+		dimensions: params.dimensions ?? "default",
+		aggregation: params.aggregation ?? "average",
+		input: serializeInputConfig(params.input),
+		chunking: serializeChunkingConfig(params.chunking),
+		safety: serializeSafetyConfig(params.safety),
+		cacheKeySalt: params.cacheKeySalt
+	});
+	hash.update(fingerprint);
+	hash.update("\0");
+	hash.update(params.content);
+	return hash.digest("hex");
+}
+/**
+* Generate a checksum for content verification.
+*/
+function generateChecksum(content) {
+	return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+function serializeInputConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		type: config.type ?? "textContent",
+		hasTransform: Boolean(config.transform),
+		hasCustomText: Boolean(config.customText)
+	});
+}
+function serializeChunkingConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		size: config.size,
+		overlap: config.overlap,
+		tokenizer: getTokenizerId(config.tokenizer),
+		maxInputLength: config.maxInputLength
+	});
+}
+function serializeSafetyConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		piiRedaction: serializePiiConfig(config.piiRedaction),
+		minTextLength: config.minTextLength,
+		maxTokens: config.maxTokens
+	});
+}
+function serializePiiConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		email: config.email ?? false,
+		phone: config.phone ?? false,
+		creditCard: config.creditCard ?? false,
+		ssn: config.ssn ?? false,
+		ipAddress: config.ipAddress ?? false,
+		customPatterns: config.customPatterns?.map((pattern) => `${pattern.source}/${pattern.flags}`)
+	});
+}
+function getTokenizerId(tokenizer) {
+	if (!tokenizer || tokenizer === "heuristic") return "heuristic";
+	if (tokenizer === "tiktoken") return "tiktoken";
+	return "custom";
+}
+function stableStringify(value) {
+	return stringifyNormalized(normalizeValue(value));
+}
+function normalizeValue(value) {
+	if (value === void 0) return void 0;
+	if (value === null) return null;
+	if (Array.isArray(value)) return value.map((entry) => normalizeValue(entry)).filter((entry) => entry !== void 0);
+	if (typeof value === "object") return normalizeObject(value);
+	return value;
+}
+function normalizeObject(value) {
+	const normalized = {};
+	for (const key of Object.keys(value).sort()) {
+		const entry = normalizeValue(value[key]);
+		if (entry !== void 0) normalized[key] = entry;
+	}
+	return normalized;
+}
+function stringifyNormalized(value) {
+	if (value === void 0) return "undefined";
+	if (value === null) return "null";
+	if (typeof value === "string") return JSON.stringify(value);
+	if (typeof value === "number" || typeof value === "boolean") return String(value);
+	if (Array.isArray(value)) return `[${value.map((entry) => stringifyNormalized(entry)).join(",")}]`;
+	if (typeof value === "object") {
+		const obj = value;
+		return `{${Object.keys(obj).sort().map((key) => `${JSON.stringify(key)}:${stringifyNormalized(obj[key])}`).join(",")}}`;
+	}
+	return JSON.stringify(value);
+}
+/**
+* In-memory LRU cache with TTL support.
+* Content-addressable: uses content hash as key, not URL.
+*/
+var InMemoryEmbeddingCache = class {
+	cache;
+	maxEntries;
+	defaultTtlMs;
+	constructor(options) {
+		this.cache = /* @__PURE__ */ new Map();
+		this.maxEntries = options?.maxEntries ?? DEFAULT_MAX_ENTRIES;
+		this.defaultTtlMs = options?.ttlMs ?? DEFAULT_TTL_MS;
+	}
+	async get(key) {
+		const entry = this.cache.get(key);
+		if (!entry) return;
+		const now = Date.now();
+		if (now > entry.expiresAt) {
+			this.cache.delete(key);
+			return;
+		}
+		entry.accessedAt = now;
+		return entry.value;
+	}
+	async set(key, value, options) {
+		const now = Date.now();
+		const ttl = options?.ttlMs ?? this.defaultTtlMs;
+		if (this.cache.size >= this.maxEntries && !this.cache.has(key)) this.evictLRU();
+		this.cache.set(key, {
+			value,
+			createdAt: now,
+			expiresAt: now + ttl,
+			accessedAt: now
+		});
+	}
+	async delete(key) {
+		return this.cache.delete(key);
+	}
+	async clear() {
+		this.cache.clear();
+	}
+	/**
+	* Get cache statistics.
+	*/
+	getStats() {
+		const now = Date.now();
+		let expired = 0;
+		for (const entry of this.cache.values()) if (now > entry.expiresAt) expired++;
+		return {
+			size: this.cache.size,
+			maxEntries: this.maxEntries,
+			expired,
+			utilization: this.cache.size / this.maxEntries
+		};
+	}
+	/**
+	* Evict expired entries.
+	*/
+	cleanup() {
+		const now = Date.now();
+		let evicted = 0;
+		for (const [key, entry] of this.cache.entries()) if (now > entry.expiresAt) {
+			this.cache.delete(key);
+			evicted++;
+		}
+		return evicted;
+	}
+	/**
+	* Evict least recently used entry.
+	*/
+	evictLRU() {
+		let oldestKey = null;
+		let oldestAccess = Number.POSITIVE_INFINITY;
+		for (const [key, entry] of this.cache.entries()) if (entry.accessedAt < oldestAccess) {
+			oldestAccess = entry.accessedAt;
+			oldestKey = key;
+		}
+		if (oldestKey) this.cache.delete(oldestKey);
+	}
+};
+/**
+* Default in-memory cache instance.
+* Optimized for moderate cache sizes (default 1000 entries).
+*/
+let defaultCache = null;
+/**
+* Get or create the default cache instance.
+*/
+function getDefaultCache() {
+	if (!defaultCache) defaultCache = new InMemoryEmbeddingCache();
+	return defaultCache;
+}
+
+//#endregion
+//#region src/embeddings/chunking.ts
+/**
+* Default chunk size in tokens.
+*/
+const DEFAULT_CHUNK_SIZE$1 = 500;
+/**
+* Default overlap in tokens.
+*/
+const DEFAULT_OVERLAP = 50;
+/**
+* Default maximum input length in characters.
+*/
+const DEFAULT_MAX_INPUT_LENGTH = 1e5;
+/**
+* Heuristic token counting: approximately 4 characters per token.
+* This is a reasonable approximation for English text.
+*/
+function heuristicTokenCount(text) {
+	return Math.ceil(text.length / 4);
+}
+/**
+* Convert token count to approximate character count.
+*/
+function tokensToChars(tokens) {
+	return tokens * 4;
+}
+/**
+* Create a tokenizer function based on configuration.
+*/
+function createTokenizer(config) {
+	if (!config || config === "heuristic") return heuristicTokenCount;
+	if (config === "tiktoken") return heuristicTokenCount;
+	return config;
+}
+/**
+* Find a natural break point in text (sentence or word boundary).
+* Prefers common sentence boundaries (Latin + CJK), falls back to word boundaries.
+*/
+function findBreakPoint(text, targetIndex) {
+	const searchStart = Math.max(0, targetIndex - Math.floor(targetIndex * .2));
+	const searchEnd = Math.min(text.length, targetIndex + Math.floor(targetIndex * .2));
+	const searchText = text.slice(searchStart, searchEnd);
+	const sentenceMatch = /[.!?。！？]\s*/g;
+	let lastSentenceEnd = -1;
+	for (const match of searchText.matchAll(sentenceMatch)) {
+		const absolutePos = searchStart + match.index + match[0].length;
+		if (absolutePos <= targetIndex) lastSentenceEnd = absolutePos;
+	}
+	if (lastSentenceEnd !== -1) return lastSentenceEnd;
+	const wordBoundary = text.lastIndexOf(" ", targetIndex);
+	if (wordBoundary > searchStart) return wordBoundary + 1;
+	return targetIndex;
+}
+/**
+* Split text into overlapping chunks optimized for embedding.
+* Respects sentence boundaries when possible.
+*/
+function chunkText(text, config) {
+	const chunkSize = config?.size ?? DEFAULT_CHUNK_SIZE$1;
+	const rawOverlap = config?.overlap ?? DEFAULT_OVERLAP;
+	const safeOverlap = Math.max(0, rawOverlap);
+	const overlap = Math.min(safeOverlap, Math.max(0, chunkSize - 1));
+	const maxInputLength = config?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
+	const tokenizer = createTokenizer(config?.tokenizer);
+	const normalizedText = (text.length > maxInputLength ? text.slice(0, maxInputLength) : text).replace(/\s+/g, " ").trim();
+	if (!normalizedText) return [];
+	const totalTokens = tokenizer(normalizedText);
+	if (totalTokens <= chunkSize) return [{
+		text: normalizedText,
+		startIndex: 0,
+		endIndex: normalizedText.length,
+		tokens: totalTokens
+	}];
+	const chunks = [];
+	const chunkSizeChars = tokensToChars(chunkSize);
+	const overlapChars = tokensToChars(overlap);
+	let startIndex = 0;
+	while (startIndex < normalizedText.length) {
+		const targetEnd = Math.min(startIndex + chunkSizeChars, normalizedText.length);
+		const endIndex = targetEnd < normalizedText.length ? findBreakPoint(normalizedText, targetEnd) : targetEnd;
+		const chunkText$1 = normalizedText.slice(startIndex, endIndex).trim();
+		if (chunkText$1) chunks.push({
+			text: chunkText$1,
+			startIndex,
+			endIndex,
+			tokens: tokenizer(chunkText$1)
+		});
+		if (endIndex >= normalizedText.length) break;
+		const nextStart = endIndex - overlapChars;
+		startIndex = Math.max(nextStart, startIndex + 1);
+		if (startIndex < normalizedText.length) {
+			const spaceIndex = normalizedText.indexOf(" ", startIndex);
+			if (spaceIndex !== -1 && spaceIndex < startIndex + overlapChars) startIndex = spaceIndex + 1;
+		}
+	}
+	return chunks;
+}
+/**
+* Estimate total tokens for a text without chunking.
+*/
+function estimateTokens(text, tokenizer) {
+	return createTokenizer(tokenizer)(text);
+}
+
+//#endregion
+//#region src/embeddings/input.ts
+/**
+* Select and prepare input text for embedding based on configuration.
+*
+* @param data - Scraped data to extract input from
+* @param config - Input configuration
+* @returns Selected and prepared text, or undefined if no valid input
+*/
+function selectInput(data, config) {
+	if (config?.transform) return normalizeText(config.transform(data));
+	if (config?.type === "custom" && config.customText) return normalizeText(config.customText);
+	const type = config?.type ?? "textContent";
+	switch (type) {
+		case "textContent": return selectTextContent(data);
+		case "title+summary": return selectTitleSummary(data);
+		case "custom": return selectTextContent(data);
+		default: {
+			const _exhaustive = type;
+			throw new Error(`Unknown input type: ${_exhaustive}`);
+		}
+	}
+}
+/**
+* Select textContent as input.
+*/
+function selectTextContent(data) {
+	if (data.textContent) return normalizeText(data.textContent);
+	if (data.content) return normalizeText(stripMarkdown(data.content));
+	if (data.excerpt) return normalizeText(data.excerpt);
+	if (data.description) return normalizeText(data.description);
+}
+/**
+* Select title + summary (or fallbacks) as input.
+* Optimized for semantic search and classification.
+*/
+function selectTitleSummary(data) {
+	const parts = [];
+	if (data.title) parts.push(data.title);
+	if (data.summary) parts.push(data.summary);
+	else if (data.excerpt) parts.push(data.excerpt);
+	else if (data.description) parts.push(data.description);
+	if (parts.length === 0) return;
+	return normalizeText(parts.join("\n\n"));
+}
+/**
+* Normalize text for embedding:
+* - Collapse whitespace
+* - Trim leading/trailing whitespace
+* - Remove control characters
+*/
+function normalizeText(text) {
+	if (!text) return "";
+	return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").split("\n").map((line) => line.trim()).join("\n").trim();
+}
+/**
+* Basic markdown stripping for when we need plain text from content.
+* Not comprehensive, but handles common cases.
+*/
+function stripMarkdown(markdown) {
+	return markdown.replace(/```[\s\S]*?```/g, "").replace(/`[^`]+`/g, "").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replace(/^#{1,6}\s+/gm, "").replace(/\*\*([^*]+)\*\*/g, "$1").replace(/\*([^*]+)\*/g, "$1").replace(/__([^_]+)__/g, "$1").replace(/_([^_]+)_/g, "$1").replace(/^>\s+/gm, "").replace(/^[-*_]{3,}$/gm, "").replace(/^[\s]*[-*+]\s+/gm, "").replace(/^[\s]*\d+\.\s+/gm, "");
+}
+/**
+* Check if the selected input meets minimum requirements.
+*/
+function validateInput(text, minLength = 10) {
+	if (!text) return {
+		valid: false,
+		reason: "No input text available"
+	};
+	if (text.length < minLength) return {
+		valid: false,
+		reason: `Input too short (${text.length} < ${minLength} characters)`
+	};
+	const wordCount = text.split(/\s+/).filter((w) => w.length > 1).length;
+	if (wordCount < 3) return {
+		valid: false,
+		reason: `Input has too few words (${wordCount} < 3)`
+	};
+	return {
+		valid: true,
+		text,
+		wordCount,
+		charCount: text.length
+	};
+}
+
+//#endregion
+//#region src/embeddings/providers/base.ts
+/**
+* Generate a stable cache key identifier for provider configuration.
+*/
+function getProviderCacheKey(config) {
+	switch (config.type) {
+		case "http": return `http:${config.config.baseUrl.replace(/\/$/, "")}:${config.config.model}`;
+		case "custom": return `custom:${config.provider.name}`;
+		default: {
+			const _exhaustive = config;
+			return String(_exhaustive);
+		}
+	}
+}
+
+//#endregion
+//#region src/embeddings/providers/http.ts
+/**
+* HTTP-based Embedding Provider using native fetch.
+* Provides a unified interface for any REST-based embedding API.
+*/
+/**
+* HTTP-based embedding provider.
+* Works with any REST API using native fetch.
+*/
+var HttpEmbeddingProvider = class extends BaseHttpProvider {
+	name = "http-embedding";
+	requestBuilder;
+	responseMapper;
+	constructor(config) {
+		super(config);
+		this.requestBuilder = config.requestBuilder ?? ((texts, model) => ({
+			input: texts,
+			model
+		}));
+		this.responseMapper = config.responseMapper ?? ((response) => {
+			const resp = response;
+			if (Array.isArray(resp.data)) return resp.data.map((item) => item.embedding);
+			if (Array.isArray(resp.embeddings)) return resp.embeddings;
+			if (Array.isArray(resp.embedding)) return [resp.embedding];
+			if (Array.isArray(response)) return response;
+			throw new ScrapeError("Unable to parse embedding response. Provide a custom responseMapper.", "VALIDATION_ERROR");
+		});
+	}
+	/**
+	* Generate embeddings for one or more texts.
+	*/
+	async embed(texts, options) {
+		const model = options.model || this.model;
+		const body = this.requestBuilder(texts, model);
+		const { data } = await this.fetch(this.baseUrl, {
+			body,
+			signal: options.signal
+		});
+		const embeddings = this.responseMapper(data);
+		if (embeddings.length !== texts.length) throw new ScrapeError(`Embedding count mismatch: expected ${texts.length}, got ${embeddings.length}`, "VALIDATION_ERROR");
+		return { embeddings };
+	}
+};
+/**
+* Create a generic HTTP embedding provider.
+*/
+function createHttpEmbedding(config) {
+	return new HttpEmbeddingProvider(config);
+}
+
+//#endregion
+//#region src/embeddings/providers/presets.ts
+/**
+* Create an OpenAI embedding provider.
+*
+* @example
+* ```ts
+* const provider = createOpenAIEmbedding({ apiKey: 'sk-...' });
+* const { embeddings } = await provider.embed(['Hello'], { model: 'text-embedding-3-small' });
+* ```
+*/
+function createOpenAIEmbedding(options) {
+	const apiKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
+	if (!apiKey) throw new Error("OpenAI API key required. Set OPENAI_API_KEY env var or pass apiKey option.");
+	const headers = { Authorization: `Bearer ${apiKey}` };
+	if (options?.organization) headers["OpenAI-Organization"] = options.organization;
+	return new HttpEmbeddingProvider({
+		baseUrl: options?.baseUrl ?? "https://api.openai.com/v1/embeddings",
+		model: options?.model ?? "text-embedding-3-small",
+		headers,
+		requestBuilder: (texts, model) => ({
+			input: texts,
+			model
+		}),
+		responseMapper: (res) => res.data.map((item) => item.embedding)
+	});
+}
+/**
+* Create an Azure OpenAI embedding provider.
+*
+* @example
+* ```ts
+* const provider = createAzureEmbedding({
+*   endpoint: 'https://my-resource.openai.azure.com',
+*   deploymentName: 'text-embedding-ada-002',
+*   apiVersion: '2023-05-15',
+* });
+* ```
+*/
+function createAzureEmbedding(options) {
+	const apiKey = options.apiKey ?? process.env.AZURE_OPENAI_API_KEY;
+	if (!apiKey) throw new Error("Azure OpenAI API key required. Set AZURE_OPENAI_API_KEY env var or pass apiKey option.");
+	return new HttpEmbeddingProvider({
+		baseUrl: `${options.endpoint.replace(/\/$/, "")}/openai/deployments/${options.deploymentName}/embeddings?api-version=${options.apiVersion}`,
+		model: options.deploymentName,
+		headers: { "api-key": apiKey },
+		requestBuilder: (texts) => ({ input: texts }),
+		responseMapper: (res) => res.data.map((item) => item.embedding)
+	});
+}
+/**
+* Create an Ollama embedding provider for local models.
+*
+* LIMITATION: Ollama's /api/embeddings endpoint processes one text at a time,
+* not batches. When multiple chunks are embedded, each chunk triggers a
+* separate HTTP request. This is handled transparently by the pipeline's
+* sequential chunk processing, but may be slower than batch-capable providers.
+* For high-throughput scenarios, consider using OpenAI, Cohere, or HuggingFace
+* which support batch embedding in a single request.
+*
+* @example
+* ```ts
+* const provider = createOllamaEmbedding({ model: 'nomic-embed-text' });
+* ```
+*/
+function createOllamaEmbedding(options) {
+	return new HttpEmbeddingProvider({
+		baseUrl: options?.baseUrl ?? "http://localhost:11434/api/embeddings",
+		model: options?.model ?? "nomic-embed-text",
+		requireHttps: false,
+		allowPrivate: true,
+		requestBuilder: (texts, model) => ({
+			model,
+			prompt: texts[0]
+		}),
+		responseMapper: (res) => [res.embedding]
+	});
+}
+/**
+* Create a HuggingFace Inference API embedding provider.
+*
+* @example
+* ```ts
+* const provider = createHuggingFaceEmbedding({
+*   model: 'sentence-transformers/all-MiniLM-L6-v2',
+* });
+* ```
+*/
+function createHuggingFaceEmbedding(options) {
+	const apiKey = options.apiKey ?? process.env.HF_TOKEN ?? process.env.HUGGINGFACE_API_KEY;
+	const headers = {};
+	if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
+	return new HttpEmbeddingProvider({
+		baseUrl: `https://api-inference.huggingface.co/models/${options.model}`,
+		model: options.model,
+		headers,
+		requestBuilder: (texts) => ({ inputs: texts }),
+		responseMapper: (response) => {
+			if (Array.isArray(response)) {
+				if (Array.isArray(response[0]) && typeof response[0][0] === "number") return response;
+				return [response];
+			}
+			throw new Error("Unexpected HuggingFace response format");
+		}
+	});
+}
+/**
+* Create a local Transformers.js embedding provider.
+* Uses dependency injection - user provides the imported transformers module.
+*
+* @example
+* ```typescript
+* import * as transformers from '@huggingface/transformers';
+* import { createTransformersEmbedding } from 'scrapex/embeddings';
+*
+* const provider = createTransformersEmbedding(transformers, {
+*   model: 'Xenova/all-MiniLM-L6-v2',
+* });
+* ```
+*
+* Required Node.js dependencies:
+* ```
+* npm install @huggingface/transformers onnxruntime-node
+* ```
+*/
+function createTransformersEmbedding(transformers, options) {
+	let pipeline = null;
+	let currentModel = null;
+	const config = {
+		model: options?.model ?? "Xenova/all-MiniLM-L6-v2",
+		quantized: options?.quantized ?? true,
+		pooling: options?.pooling ?? "mean",
+		normalize: options?.normalize ?? true
+	};
+	return {
+		name: "transformers",
+		async embed(texts, request) {
+			const model = request.model || config.model;
+			if (!pipeline || currentModel !== model) {
+				const cacheDir = options?.cacheDir;
+				const env = transformers.env;
+				const priorCacheDir = env?.cacheDir;
+				if (cacheDir && env) env.cacheDir = cacheDir;
+				try {
+					pipeline = await transformers.pipeline("feature-extraction", model, { quantized: config.quantized });
+				} finally {
+					if (cacheDir && env) if (priorCacheDir === void 0) delete env.cacheDir;
+					else env.cacheDir = priorCacheDir;
+				}
+				currentModel = model;
+			}
+			const embeddings = [];
+			for (const text of texts) {
+				const output = await pipeline(text, {
+					pooling: config.pooling,
+					normalize: config.normalize
+				});
+				embeddings.push(Array.from(output.data));
+			}
+			return { embeddings };
+		}
+	};
+}
+/** Recommended models for Transformers.js */
+const TRANSFORMERS_MODELS = {
+	DEFAULT: "Xenova/all-MiniLM-L6-v2",
+	QUALITY: "Xenova/all-mpnet-base-v2",
+	RETRIEVAL: "Xenova/bge-small-en-v1.5",
+	MULTILINGUAL: "Xenova/multilingual-e5-small"
+};
+
+//#endregion
+//#region src/embeddings/providers/index.ts
+/**
+* Create an embedding provider from configuration.
+* This is the main factory function for creating providers.
+*/
+function createEmbeddingProvider(config) {
+	switch (config.type) {
+		case "http": return createHttpEmbedding(config.config);
+		case "custom": return config.provider;
+		default: throw new ScrapeError(`Unknown embedding provider type: ${config.type}`, "VALIDATION_ERROR");
+	}
+}
+
+//#endregion
+//#region src/embeddings/safety.ts
+/**
+* PII redaction patterns with high precision to minimize false positives.
+* Patterns are designed to match common formats while avoiding over-matching.
+*/
+const EMAIL_PATTERN = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
+const PHONE_PATTERN = /(?:\+?1[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}\b/g;
+const CREDIT_CARD_PATTERN = /\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12}|(?:[0-9]{4}[-\s]){3}[0-9]{4}|[0-9]{13,19})\b/g;
+const SSN_PATTERN = /\b[0-9]{3}-[0-9]{2}-[0-9]{4}\b/g;
+const IPV4_PATTERN = /\b(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/g;
+const REDACTED = "[REDACTED]";
+/**
+* Create a redaction function based on configuration.
+* Returns a function that applies all configured PII patterns.
+*/
+function createPiiRedactor(config) {
+	const patterns = [];
+	if (config.creditCard) patterns.push({
+		name: "creditCard",
+		pattern: CREDIT_CARD_PATTERN
+	});
+	if (config.email) patterns.push({
+		name: "email",
+		pattern: EMAIL_PATTERN
+	});
+	if (config.phone) patterns.push({
+		name: "phone",
+		pattern: PHONE_PATTERN
+	});
+	if (config.ssn) patterns.push({
+		name: "ssn",
+		pattern: SSN_PATTERN
+	});
+	if (config.ipAddress) patterns.push({
+		name: "ipAddress",
+		pattern: IPV4_PATTERN
+	});
+	if (config.customPatterns) for (let i = 0; i < config.customPatterns.length; i++) {
+		const customPattern = config.customPatterns[i];
+		if (customPattern) patterns.push({
+			name: `custom_${i}`,
+			pattern: customPattern
+		});
+	}
+	return (text) => {
+		let redactedText = text;
+		let totalRedactions = 0;
+		const redactionsByType = {};
+		for (const { name, pattern } of patterns) {
+			pattern.lastIndex = 0;
+			const matchCount = text.match(pattern)?.length ?? 0;
+			if (matchCount > 0) {
+				redactedText = redactedText.replace(pattern, REDACTED);
+				totalRedactions += matchCount;
+				redactionsByType[name] = (redactionsByType[name] ?? 0) + matchCount;
+			}
+		}
+		return {
+			text: redactedText,
+			redacted: totalRedactions > 0,
+			redactionCount: totalRedactions,
+			redactionsByType
+		};
+	};
+}
+/**
+* Simple redaction that applies all default patterns.
+* Use createPiiRedactor() for fine-grained control.
+*/
+function redactPii(text) {
+	return createPiiRedactor({
+		email: true,
+		phone: true,
+		creditCard: true,
+		ssn: true,
+		ipAddress: true
+	})(text);
+}
+
+//#endregion
+//#region src/embeddings/pipeline.ts
+const DEFAULT_CHUNK_SIZE = 500;
+/**
+* Get the effective model for embedding.
+* Prioritizes: explicit options.model > provider config model
+*/
+function getEffectiveModel(providerConfig, explicitModel) {
+	if (explicitModel) return explicitModel;
+	if (providerConfig.type === "http") return providerConfig.config.model;
+}
+/**
+* Generate embeddings for scraped data.
+* This is the main entry point for the embedding pipeline.
+*/
+async function generateEmbeddings(data, options) {
+	const startTime = Date.now();
+	try {
+		const provider = createEmbeddingProvider(options.provider);
+		const model = getEffectiveModel(options.provider, options.model);
+		const validation = validateInput(selectInput(data, options.input), options.safety?.minTextLength ?? 10);
+		if (!validation.valid) return createSkippedResult(validation.reason, { model });
+		const originalInput = validation.text;
+		let inputText = validation.text;
+		let piiRedacted = false;
+		if (options.safety?.piiRedaction) {
+			const redactionResult = createPiiRedactor(options.safety.piiRedaction)(inputText);
+			inputText = redactionResult.text;
+			piiRedacted = redactionResult.redacted;
+		}
+		const effectiveChunking = applyMaxTokensToChunking(options.chunking, options.safety?.maxTokens);
+		const cacheKey = generateCacheKey({
+			providerKey: getProviderCacheKey(options.provider),
+			model,
+			dimensions: options.output?.dimensions,
+			aggregation: options.output?.aggregation,
+			input: options.input,
+			chunking: effectiveChunking,
+			safety: options.safety,
+			cacheKeySalt: options.cache?.cacheKeySalt,
+			content: inputText
+		});
+		const cache = options.cache?.store ?? getDefaultCache();
+		const cachedResult = await cache.get(cacheKey);
+		if (cachedResult && cachedResult.status === "success") {
+			if (options.onMetrics) options.onMetrics({
+				provider: provider.name,
+				model,
+				inputTokens: estimateTokens(inputText),
+				outputDimensions: getDimensions(cachedResult.aggregation === "all" ? cachedResult.vectors : cachedResult.vector),
+				chunks: cachedResult.source.chunks,
+				latencyMs: Date.now() - startTime,
+				cached: true,
+				retries: 0,
+				piiRedacted
+			});
+			return {
+				...cachedResult,
+				source: {
+					...cachedResult.source,
+					cached: true
+				}
+			};
+		}
+		const chunks = chunkText(inputText, effectiveChunking);
+		const callbackChunks = options.onChunk && options.safety?.allowSensitiveCallbacks ? chunkText(originalInput, effectiveChunking) : null;
+		if (chunks.length === 0) return createSkippedResult("No content after chunking", { model });
+		const sharedState = options.resilience?.state;
+		const rateLimiter = sharedState?.rateLimiter ?? (options.resilience?.rateLimit ? new RateLimiter(options.resilience.rateLimit) : null);
+		const circuitBreaker = sharedState?.circuitBreaker ?? (options.resilience?.circuitBreaker ? new CircuitBreaker(options.resilience.circuitBreaker) : null);
+		const concurrency = options.resilience?.concurrency ?? 1;
+		const semaphore = sharedState?.semaphore ?? new Semaphore(concurrency);
+		const embeddings = [];
+		let totalTokens = 0;
+		let retryCount = 0;
+		for (let i = 0; i < chunks.length; i++) {
+			const chunk = chunks[i];
+			if (!chunk) continue;
+			if (rateLimiter) await rateLimiter.acquire();
+			if (circuitBreaker?.isOpen()) return createSkippedResult("Circuit breaker is open", {
+				model,
+				chunks: i
+			});
+			await semaphore.execute(async () => {
+				const { result: result$1 } = await withResilience(async (signal) => {
+					return provider.embed([chunk.text], {
+						model,
+						dimensions: options.output?.dimensions,
+						signal
+					});
+				}, options.resilience, {
+					circuitBreaker: circuitBreaker ?? void 0,
+					rateLimiter: void 0,
+					semaphore: void 0
+				}, { onRetry: () => {
+					retryCount++;
+				} });
+				if (result$1.usage) totalTokens += result$1.usage.totalTokens;
+				else totalTokens += chunk.tokens;
+				const embedding = result$1.embeddings[0];
+				if (embedding) {
+					embeddings.push(embedding);
+					if (options.onChunk) {
+						const callbackText = callbackChunks?.[i]?.text ?? chunk.text;
+						options.onChunk(callbackText, embedding);
+					}
+				}
+			});
+		}
+		const aggregation = options.output?.aggregation ?? "average";
+		const aggregated = aggregateVectors(embeddings, aggregation);
+		const source = {
+			model,
+			chunks: chunks.length,
+			tokens: totalTokens || estimateTokens(inputText),
+			checksum: generateChecksum(inputText),
+			cached: false,
+			latencyMs: Date.now() - startTime
+		};
+		let result;
+		if (aggregated.type === "single") result = {
+			status: "success",
+			aggregation,
+			vector: aggregated.vector,
+			source
+		};
+		else result = {
+			status: "success",
+			aggregation: "all",
+			vectors: aggregated.vectors,
+			source
+		};
+		await cache.set(cacheKey, result, { ttlMs: options.cache?.ttlMs });
+		if (options.onMetrics) {
+			const metrics = {
+				provider: provider.name,
+				model,
+				inputTokens: source.tokens,
+				outputDimensions: aggregated.dimensions,
+				chunks: chunks.length,
+				latencyMs: source.latencyMs,
+				cached: false,
+				retries: retryCount,
+				piiRedacted
+			};
+			options.onMetrics(metrics);
+		}
+		return result;
+	} catch (error) {
+		const reason = error instanceof Error ? error.message : String(error);
+		if (error instanceof ScrapeError && ["INVALID_URL", "BLOCKED"].includes(error.code)) throw error;
+		return createSkippedResult(reason, { latencyMs: Date.now() - startTime });
+	}
+}
+function applyMaxTokensToChunking(chunking, maxTokens) {
+	if (!maxTokens || maxTokens <= 0) return chunking;
+	const baseSize = chunking?.size ?? DEFAULT_CHUNK_SIZE;
+	const baseOverlap = chunking?.overlap ?? 50;
+	const clampedSize = Math.min(baseSize, maxTokens);
+	const clampedOverlap = Math.min(baseOverlap, Math.max(0, clampedSize - 1));
+	return {
+		...chunking,
+		size: clampedSize,
+		overlap: clampedOverlap
+	};
+}
+/**
+* Embed arbitrary text directly.
+* Standalone function for embedding text outside of scrape().
+*/
+async function embed(text, options) {
+	return generateEmbeddings({ textContent: text }, {
+		...options,
+		input: {
+			...options.input,
+			type: "textContent"
+		}
+	});
+}
+/**
+* Embed from existing ScrapedData.
+* Useful when you've already scraped and want to add embeddings later.
+*/
+async function embedScrapedData(data, options) {
+	return generateEmbeddings(data, options);
+}
+/**
+* Create a skipped result with reason.
+*/
+function createSkippedResult(reason, partialSource) {
+	return {
+		status: "skipped",
+		reason,
+		source: partialSource ?? {}
+	};
+}
+
 //#endregion
 //#region src/extractors/content.ts
 const turndown = new TurndownService({
@@ -528,7 +1566,9 @@ var NativeFetcher = class {
 				throw new ScrapeError(`HTTP error ${response.status}: ${url}`, "FETCH_FAILED", response.status);
 			}
 			const contentType = response.headers.get("content-type") || "";
-			if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) throw new ScrapeError(`Unexpected content type: ${contentType}`, "PARSE_ERROR");
+			if (options.allowedContentTypes) {
+				if (!options.allowedContentTypes.some((type) => contentType.toLowerCase().includes(type.toLowerCase()))) throw new ScrapeError(`Unexpected content type: ${contentType}`, "PARSE_ERROR");
+			} else if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) throw new ScrapeError(`Unexpected content type: ${contentType}`, "PARSE_ERROR");
 			const html = await response.text();
 			const responseHeaders = {};
 			response.headers.forEach((value, key) => {
@@ -719,6 +1759,7 @@ async function scrape(url, options = {}) {
 		console.error("LLM extraction failed:", error);
 		intermediateResult.error = intermediateResult.error ? `${intermediateResult.error}; LLM extraction: ${error instanceof Error ? error.message : String(error)}` : `LLM extraction: ${error instanceof Error ? error.message : String(error)}`;
 	}
+	if (options.embeddings) intermediateResult.embeddings = await generateEmbeddings(intermediateResult, options.embeddings);
 	const scrapeTimeMs = Date.now() - startTime;
 	return {
 		...intermediateResult,
@@ -759,9 +1800,8 @@ async function scrapeHtml(html, url, options = {}) {
 		console.error(`Extractor "${extractor.name}" failed:`, error);
 		context = mergeResults(context, { error: context.results.error ? `${context.results.error}; ${extractor.name}: ${error instanceof Error ? error.message : String(error)}` : `${extractor.name}: ${error instanceof Error ? error.message : String(error)}` });
 	}
-	const scrapeTimeMs = Date.now() - startTime;
 	const domain = extractDomain(normalizedUrl);
-	return {
+	const intermediateResult = {
 		url: normalizedUrl,
 		canonicalUrl: context.results.canonicalUrl || normalizedUrl,
 		domain,
@@ -788,11 +1828,129 @@ async function scrapeHtml(html, url, options = {}) {
 		extracted: context.results.extracted,
 		custom: context.results.custom,
 		scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
-		scrapeTimeMs,
+		scrapeTimeMs: 0,
 		error: context.results.error
 	};
+	if (options.embeddings) intermediateResult.embeddings = await generateEmbeddings(intermediateResult, options.embeddings);
+	const scrapeTimeMs = Date.now() - startTime;
+	return {
+		...intermediateResult,
+		scrapeTimeMs
+	};
+}
+
+//#endregion
+//#region src/utils/feed.ts
+/**
+* Fetch and parse an RSS/Atom feed from a URL.
+* Uses scrapex's fetcher infrastructure for consistent behavior.
+*/
+async function fetchFeed(url, options) {
+	const result = await (options?.fetcher || defaultFetcher).fetch(url, {
+		timeout: options?.timeout,
+		userAgent: options?.userAgent,
+		allowedContentTypes: [
+			"application/rss+xml",
+			"application/atom+xml",
+			"application/rdf+xml",
+			"application/xml",
+			"text/xml",
+			"text/html"
+		]
+	});
+	return new RSSParser(options?.parserOptions).parse(result.html, url);
+}
+/**
+* Detect RSS/Atom feed URLs from HTML.
+* Supports RSS, Atom, and RDF feed types.
+*/
+function discoverFeeds(html, baseUrl) {
+	const $ = cheerio.load(html);
+	const feeds = [];
+	const seen = /* @__PURE__ */ new Set();
+	$([
+		"link[type=\"application/rss+xml\"]",
+		"link[type=\"application/atom+xml\"]",
+		"link[type=\"application/rdf+xml\"]",
+		"link[rel=\"alternate\"][type*=\"xml\"]"
+	].join(", ")).each((_, el) => {
+		const href = $(el).attr("href");
+		if (href) try {
+			const resolved = new URL(href, baseUrl).href;
+			if (!seen.has(resolved)) {
+				seen.add(resolved);
+				feeds.push(resolved);
+			}
+		} catch {}
+	});
+	return feeds;
+}
+/**
+* Filter feed items by date range.
+* Items without publishedAt are included by default.
+*/
+function filterByDate(items, options) {
+	const { after, before, includeUndated = true } = options;
+	return items.filter((item) => {
+		if (!item.publishedAt) return includeUndated;
+		const date = new Date(item.publishedAt);
+		if (after && date < after) return false;
+		if (before && date > before) return false;
+		return true;
+	});
+}
+/**
+* Convert feed items to markdown for LLM consumption.
+* Uses ISO 8601 date format for consistency across environments.
+*/
+function feedToMarkdown(feed, options) {
+	const { includeContent = false, maxItems } = options || {};
+	const lines = [`# ${feed.title}`, ""];
+	if (feed.description) lines.push(feed.description, "");
+	const items = maxItems ? feed.items.slice(0, maxItems) : feed.items;
+	for (const item of items) {
+		lines.push(`## ${item.title}`);
+		if (item.publishedAt) {
+			const date = item.publishedAt.split("T")[0];
+			lines.push(`*${date}*`);
+		}
+		lines.push("");
+		if (includeContent && item.content) lines.push(item.content);
+		else if (item.description) lines.push(item.description);
+		if (item.link) lines.push(`[Read more](${item.link})`, "");
+		else lines.push("");
+	}
+	return lines.join("\n");
+}
+/**
+* Extract plain text from feed items for LLM processing.
+* Concatenates title, description, and content.
+*/
+function feedToText(feed, options) {
+	const { maxItems, separator = "\n\n---\n\n" } = options || {};
+	return (maxItems ? feed.items.slice(0, maxItems) : feed.items).map((item) => {
+		const parts = [item.title];
+		if (item.description) parts.push(item.description);
+		if (item.content) parts.push(item.content);
+		return parts.join("\n\n");
+	}).join(separator);
+}
+/**
+* Paginate through a feed using rel="next" links (RFC 5005).
+* Returns an async generator that yields each page.
+*/
+async function* paginateFeed(url, options) {
+	const { maxPages = 10, ...fetchOptions } = options || {};
+	let currentUrl = url;
+	let pageCount = 0;
+	while (currentUrl && pageCount < maxPages) {
+		const result = await fetchFeed(currentUrl, fetchOptions);
+		yield result.data;
+		currentUrl = result.data.next;
+		pageCount++;
+	}
 }
 
 //#endregion
-export { ContentExtractor, DEFAULT_TIMEOUT, DEFAULT_USER_AGENT, FaviconExtractor, JsonLdExtractor, LinksExtractor, MetaExtractor, NativeFetcher, ScrapeError, checkRobotsTxt, createDefaultExtractors, createExtractionContext, defaultFetcher, extractDomain, getPath, getProtocol, isExternalUrl, isValidUrl, matchesUrlPattern, mergeResults, normalizeUrl, resolveUrl, scrape, scrapeHtml, sortExtractors };
+export { ContentExtractor, DEFAULT_TIMEOUT, DEFAULT_USER_AGENT, FaviconExtractor, InMemoryEmbeddingCache, JsonLdExtractor, LinksExtractor, MetaExtractor, NativeFetcher, RSSParser, ScrapeError, TRANSFORMERS_MODELS, aggregateVectors, checkRobotsTxt, chunkText, cosineSimilarity, createAzureEmbedding, createDefaultExtractors, createEmbeddingProvider, createExtractionContext, createHttpEmbedding, createHuggingFaceEmbedding, createOllamaEmbedding, createOpenAIEmbedding, createPiiRedactor, createTransformersEmbedding, defaultFetcher, discoverFeeds, embed, embedScrapedData, estimateTokens, extractDomain, feedToMarkdown, feedToText, fetchFeed, filterByDate, generateEmbeddings, getPath, getProtocol, isExternalUrl, isValidUrl, matchesUrlPattern, mergeResults, normalizeUrl, paginateFeed, redactPii, resolveUrl, scrape, scrapeHtml, sortExtractors };
 //# sourceMappingURL=index.mjs.map