scrapex 1.0.0-beta.1 → 1.0.0-beta.2

package/dist/embeddings-Bsymy_jA.mjs

@@ -0,0 +1,1215 @@
+import { a as Semaphore, c as withResilience, d as ScrapeError, i as RateLimiter, n as CircuitBreaker, t as BaseHttpProvider } from "./http-base-DM7YNo6X.mjs";
+import { createHash } from "node:crypto";
+
+//#region src/embeddings/aggregation.ts
+/**
+* Aggregate multiple embedding vectors into a single vector or return all.
+*
+* @param vectors - Array of embedding vectors (must all have same dimensions)
+* @param strategy - Aggregation strategy
+* @returns Aggregated result based on strategy
+*/
+function aggregateVectors(vectors, strategy = "average") {
+	if (vectors.length === 0) throw new Error("Cannot aggregate empty vector array");
+	const firstVector = vectors[0];
+	if (!firstVector) throw new Error("Cannot aggregate empty vector array");
+	const dimensions = firstVector.length;
+	for (let i = 1; i < vectors.length; i++) {
+		const vec = vectors[i];
+		if (!vec || vec.length !== dimensions) throw new Error(`Vector dimension mismatch: expected ${dimensions}, got ${vec?.length ?? 0} at index ${i}`);
+	}
+	switch (strategy) {
+		case "average": return {
+			type: "single",
+			vector: averageVectors(vectors),
+			dimensions
+		};
+		case "max": return {
+			type: "single",
+			vector: maxPoolVectors(vectors),
+			dimensions
+		};
+		case "first": return {
+			type: "single",
+			vector: firstVector,
+			dimensions
+		};
+		case "all": return {
+			type: "multiple",
+			vectors,
+			dimensions
+		};
+		default: {
+			const _exhaustive = strategy;
+			throw new Error(`Unknown aggregation strategy: ${_exhaustive}`);
+		}
+	}
+}
+/**
+* Compute element-wise average of vectors.
+*/
+function averageVectors(vectors) {
+	const first = vectors[0];
+	if (!first || vectors.length === 1) return first ?? [];
+	const dimensions = first.length;
+	const count = vectors.length;
+	const result = new Array(dimensions).fill(0);
+	for (const vector of vectors) for (let i = 0; i < dimensions; i++) {
+		const val = result[i];
+		if (val !== void 0) result[i] = val + (vector[i] ?? 0);
+	}
+	for (let i = 0; i < dimensions; i++) {
+		const val = result[i];
+		if (val !== void 0) result[i] = val / count;
+	}
+	return result;
+}
+/**
+* Compute element-wise maximum of vectors (max pooling).
+*/
+function maxPoolVectors(vectors) {
+	const first = vectors[0];
+	if (!first || vectors.length === 1) return first ?? [];
+	const dimensions = first.length;
+	const result = [...first];
+	for (let v = 1; v < vectors.length; v++) {
+		const vec = vectors[v];
+		if (!vec) continue;
+		for (let i = 0; i < dimensions; i++) {
+			const val = vec[i] ?? 0;
+			if (val > (result[i] ?? 0)) result[i] = val;
+		}
+	}
+	return result;
+}
+/**
+* Normalize a vector to unit length (L2 normalization).
+*/
+function normalizeVector(vector) {
+	const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
+	if (magnitude === 0) return vector;
+	return vector.map((val) => val / magnitude);
+}
+/**
+* Compute cosine similarity between two vectors.
+* Both vectors should be normalized for accurate results.
+*/
+function cosineSimilarity(a, b) {
+	if (a.length !== b.length) throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+	let dot = 0;
+	let magnitudeA = 0;
+	let magnitudeB = 0;
+	for (let i = 0; i < a.length; i++) {
+		const aVal = a[i] ?? 0;
+		const bVal = b[i] ?? 0;
+		dot += aVal * bVal;
+		magnitudeA += aVal * aVal;
+		magnitudeB += bVal * bVal;
+	}
+	const magnitude = Math.sqrt(magnitudeA) * Math.sqrt(magnitudeB);
+	if (magnitude === 0) return 0;
+	return dot / magnitude;
+}
+/**
+* Compute euclidean distance between two vectors.
+*/
+function euclideanDistance(a, b) {
+	if (a.length !== b.length) throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+	let sum = 0;
+	for (let i = 0; i < a.length; i++) {
+		const diff = (a[i] ?? 0) - (b[i] ?? 0);
+		sum += diff * diff;
+	}
+	return Math.sqrt(sum);
+}
+/**
+* Compute dot product of two vectors.
+*/
+function dotProduct(a, b) {
+	if (a.length !== b.length) throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+	let result = 0;
+	for (let i = 0; i < a.length; i++) {
+		const aVal = a[i] ?? 0;
+		const bVal = b[i] ?? 0;
+		result += aVal * bVal;
+	}
+	return result;
+}
+/**
+* Get the dimensions of a vector or set of vectors.
+*/
+function getDimensions(vectors) {
+	if (vectors.length === 0) return 0;
+	const first = vectors[0];
+	if (typeof first === "number") return vectors.length;
+	return first?.length ?? 0;
+}
+
+//#endregion
+//#region src/embeddings/cache.ts
+/**
+* Default maximum cache entries.
+*/
+const DEFAULT_MAX_ENTRIES = 1e3;
+/**
+* Default TTL in milliseconds (1 hour).
+*/
+const DEFAULT_TTL_MS = 3600 * 1e3;
+/**
+* Generate a content-addressable cache key.
+* Key is based on content hash and embedding configuration.
+* Note: custom RegExp patterns are serialized by source+flags; different
+* constructions can yield different cache keys even if equivalent.
+*/
+function generateCacheKey(params) {
+	const hash = createHash("sha256");
+	const fingerprint = stableStringify({
+		providerKey: params.providerKey,
+		model: params.model ?? "provider-default",
+		dimensions: params.dimensions ?? "default",
+		aggregation: params.aggregation ?? "average",
+		input: serializeInputConfig(params.input),
+		chunking: serializeChunkingConfig(params.chunking),
+		safety: serializeSafetyConfig(params.safety),
+		cacheKeySalt: params.cacheKeySalt
+	});
+	hash.update(fingerprint);
+	hash.update("\0");
+	hash.update(params.content);
+	return hash.digest("hex");
+}
+/**
+* Generate a checksum for content verification.
+*/
+function generateChecksum(content) {
+	return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+function serializeInputConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		type: config.type ?? "textContent",
+		hasTransform: Boolean(config.transform),
+		hasCustomText: Boolean(config.customText)
+	});
+}
+function serializeChunkingConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		size: config.size,
+		overlap: config.overlap,
+		tokenizer: getTokenizerId(config.tokenizer),
+		maxInputLength: config.maxInputLength
+	});
+}
+function serializeSafetyConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		piiRedaction: serializePiiConfig(config.piiRedaction),
+		minTextLength: config.minTextLength,
+		maxTokens: config.maxTokens
+	});
+}
+function serializePiiConfig(config) {
+	if (!config) return void 0;
+	return normalizeObject({
+		email: config.email ?? false,
+		phone: config.phone ?? false,
+		creditCard: config.creditCard ?? false,
+		ssn: config.ssn ?? false,
+		ipAddress: config.ipAddress ?? false,
+		customPatterns: config.customPatterns?.map((pattern) => `${pattern.source}/${pattern.flags}`)
+	});
+}
+function getTokenizerId(tokenizer) {
+	if (!tokenizer || tokenizer === "heuristic") return "heuristic";
+	if (tokenizer === "tiktoken") return "tiktoken";
+	return "custom";
+}
+function stableStringify(value) {
+	return stringifyNormalized(normalizeValue(value));
+}
+function normalizeValue(value) {
+	if (value === void 0) return void 0;
+	if (value === null) return null;
+	if (Array.isArray(value)) return value.map((entry) => normalizeValue(entry)).filter((entry) => entry !== void 0);
+	if (typeof value === "object") return normalizeObject(value);
+	return value;
+}
+function normalizeObject(value) {
+	const normalized = {};
+	for (const key of Object.keys(value).sort()) {
+		const entry = normalizeValue(value[key]);
+		if (entry !== void 0) normalized[key] = entry;
+	}
+	return normalized;
+}
+function stringifyNormalized(value) {
+	if (value === void 0) return "undefined";
+	if (value === null) return "null";
+	if (typeof value === "string") return JSON.stringify(value);
+	if (typeof value === "number" || typeof value === "boolean") return String(value);
+	if (Array.isArray(value)) return `[${value.map((entry) => stringifyNormalized(entry)).join(",")}]`;
+	if (typeof value === "object") {
+		const obj = value;
+		return `{${Object.keys(obj).sort().map((key) => `${JSON.stringify(key)}:${stringifyNormalized(obj[key])}`).join(",")}}`;
+	}
+	return JSON.stringify(value);
+}
+/**
+* In-memory LRU cache with TTL support.
+* Content-addressable: uses content hash as key, not URL.
+*/
+var InMemoryEmbeddingCache = class {
+	cache;
+	maxEntries;
+	defaultTtlMs;
+	constructor(options) {
+		this.cache = /* @__PURE__ */ new Map();
+		this.maxEntries = options?.maxEntries ?? DEFAULT_MAX_ENTRIES;
+		this.defaultTtlMs = options?.ttlMs ?? DEFAULT_TTL_MS;
+	}
+	async get(key) {
+		const entry = this.cache.get(key);
+		if (!entry) return;
+		const now = Date.now();
+		if (now > entry.expiresAt) {
+			this.cache.delete(key);
+			return;
+		}
+		entry.accessedAt = now;
+		return entry.value;
+	}
+	async set(key, value, options) {
+		const now = Date.now();
+		const ttl = options?.ttlMs ?? this.defaultTtlMs;
+		if (this.cache.size >= this.maxEntries && !this.cache.has(key)) this.evictLRU();
+		this.cache.set(key, {
+			value,
+			createdAt: now,
+			expiresAt: now + ttl,
+			accessedAt: now
+		});
+	}
+	async delete(key) {
+		return this.cache.delete(key);
+	}
+	async clear() {
+		this.cache.clear();
+	}
+	/**
+	* Get cache statistics.
+	*/
+	getStats() {
+		const now = Date.now();
+		let expired = 0;
+		for (const entry of this.cache.values()) if (now > entry.expiresAt) expired++;
+		return {
+			size: this.cache.size,
+			maxEntries: this.maxEntries,
+			expired,
+			utilization: this.cache.size / this.maxEntries
+		};
+	}
+	/**
+	* Evict expired entries.
+	*/
+	cleanup() {
+		const now = Date.now();
+		let evicted = 0;
+		for (const [key, entry] of this.cache.entries()) if (now > entry.expiresAt) {
+			this.cache.delete(key);
+			evicted++;
+		}
+		return evicted;
+	}
+	/**
+	* Evict least recently used entry.
+	*/
+	evictLRU() {
+		let oldestKey = null;
+		let oldestAccess = Number.POSITIVE_INFINITY;
+		for (const [key, entry] of this.cache.entries()) if (entry.accessedAt < oldestAccess) {
+			oldestAccess = entry.accessedAt;
+			oldestKey = key;
+		}
+		if (oldestKey) this.cache.delete(oldestKey);
+	}
+};
+/**
+* Validate that a cached result matches expected parameters.
+*/
+function validateCachedResult(result, expectedDimensions) {
+	if (result.status !== "success") return true;
+	if (!expectedDimensions) return true;
+	if (result.aggregation === "all") {
+		const firstVec = result.vectors[0];
+		if (!firstVec) return false;
+		return firstVec.length === expectedDimensions;
+	}
+	return result.vector.length === expectedDimensions;
+}
+/**
+* Create a no-op cache that never stores anything.
+* Useful for disabling caching while maintaining interface compatibility.
+*/
+function createNoOpCache() {
+	return {
+		async get() {},
+		async set() {},
+		async delete() {
+			return false;
+		},
+		async clear() {}
+	};
+}
+/**
+* Default in-memory cache instance.
+* Optimized for moderate cache sizes (default 1000 entries).
+*/
+let defaultCache = null;
+/**
+* Get or create the default cache instance.
+*/
+function getDefaultCache() {
+	if (!defaultCache) defaultCache = new InMemoryEmbeddingCache();
+	return defaultCache;
+}
+/**
+* Reset the default cache (mainly for testing).
+*/
+async function resetDefaultCache() {
+	if (defaultCache) await defaultCache.clear();
+	defaultCache = null;
+}
+
+//#endregion
+//#region src/embeddings/chunking.ts
+/**
+* Default chunk size in tokens.
+*/
+const DEFAULT_CHUNK_SIZE$1 = 500;
+/**
+* Default overlap in tokens.
+*/
+const DEFAULT_OVERLAP = 50;
+/**
+* Default maximum input length in characters.
+*/
+const DEFAULT_MAX_INPUT_LENGTH = 1e5;
+/**
+* Heuristic token counting: approximately 4 characters per token.
+* This is a reasonable approximation for English text.
+*/
+function heuristicTokenCount(text) {
+	return Math.ceil(text.length / 4);
+}
+/**
+* Convert token count to approximate character count.
+*/
+function tokensToChars(tokens) {
+	return tokens * 4;
+}
+/**
+* Create a tokenizer function based on configuration.
+*/
+function createTokenizer(config) {
+	if (!config || config === "heuristic") return heuristicTokenCount;
+	if (config === "tiktoken") return heuristicTokenCount;
+	return config;
+}
+/**
+* Find a natural break point in text (sentence or word boundary).
+* Prefers common sentence boundaries (Latin + CJK), falls back to word boundaries.
+*/
+function findBreakPoint(text, targetIndex) {
+	const searchStart = Math.max(0, targetIndex - Math.floor(targetIndex * .2));
+	const searchEnd = Math.min(text.length, targetIndex + Math.floor(targetIndex * .2));
+	const searchText = text.slice(searchStart, searchEnd);
+	const sentenceMatch = /[.!?。！？]\s*/g;
+	let lastSentenceEnd = -1;
+	for (const match of searchText.matchAll(sentenceMatch)) {
+		const absolutePos = searchStart + match.index + match[0].length;
+		if (absolutePos <= targetIndex) lastSentenceEnd = absolutePos;
+	}
+	if (lastSentenceEnd !== -1) return lastSentenceEnd;
+	const wordBoundary = text.lastIndexOf(" ", targetIndex);
+	if (wordBoundary > searchStart) return wordBoundary + 1;
+	return targetIndex;
+}
+/**
+* Split text into overlapping chunks optimized for embedding.
+* Respects sentence boundaries when possible.
+*/
+function chunkText(text, config) {
+	const chunkSize = config?.size ?? DEFAULT_CHUNK_SIZE$1;
+	const rawOverlap = config?.overlap ?? DEFAULT_OVERLAP;
+	const safeOverlap = Math.max(0, rawOverlap);
+	const overlap = Math.min(safeOverlap, Math.max(0, chunkSize - 1));
+	const maxInputLength = config?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
+	const tokenizer = createTokenizer(config?.tokenizer);
+	const normalizedText = (text.length > maxInputLength ? text.slice(0, maxInputLength) : text).replace(/\s+/g, " ").trim();
+	if (!normalizedText) return [];
+	const totalTokens = tokenizer(normalizedText);
+	if (totalTokens <= chunkSize) return [{
+		text: normalizedText,
+		startIndex: 0,
+		endIndex: normalizedText.length,
+		tokens: totalTokens
+	}];
+	const chunks = [];
+	const chunkSizeChars = tokensToChars(chunkSize);
+	const overlapChars = tokensToChars(overlap);
+	let startIndex = 0;
+	while (startIndex < normalizedText.length) {
+		const targetEnd = Math.min(startIndex + chunkSizeChars, normalizedText.length);
+		const endIndex = targetEnd < normalizedText.length ? findBreakPoint(normalizedText, targetEnd) : targetEnd;
+		const chunkText$1 = normalizedText.slice(startIndex, endIndex).trim();
+		if (chunkText$1) chunks.push({
+			text: chunkText$1,
+			startIndex,
+			endIndex,
+			tokens: tokenizer(chunkText$1)
+		});
+		if (endIndex >= normalizedText.length) break;
+		const nextStart = endIndex - overlapChars;
+		startIndex = Math.max(nextStart, startIndex + 1);
+		if (startIndex < normalizedText.length) {
+			const spaceIndex = normalizedText.indexOf(" ", startIndex);
+			if (spaceIndex !== -1 && spaceIndex < startIndex + overlapChars) startIndex = spaceIndex + 1;
+		}
+	}
+	return chunks;
+}
+/**
+* Estimate total tokens for a text without chunking.
+*/
+function estimateTokens(text, tokenizer) {
+	return createTokenizer(tokenizer)(text);
+}
+/**
+* Check if text needs chunking based on token count.
+*/
+function needsChunking(text, maxTokens = DEFAULT_CHUNK_SIZE$1, tokenizer) {
+	return createTokenizer(tokenizer)(text) > maxTokens;
+}
+/**
+* Get statistics about potential chunking.
+*/
+function getChunkingStats(text, config) {
+	const maxInputLength = config?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
+	const chunkSize = config?.size ?? DEFAULT_CHUNK_SIZE$1;
+	const overlap = config?.overlap ?? DEFAULT_OVERLAP;
+	const tokenizer = createTokenizer(config?.tokenizer);
+	const inputLength = text.length;
+	const willTruncate = inputLength > maxInputLength;
+	const processedLength = willTruncate ? maxInputLength : inputLength;
+	const estimatedTokens = tokenizer(text.slice(0, processedLength).replace(/\s+/g, " ").trim());
+	let estimatedChunks = 1;
+	if (estimatedTokens > chunkSize) {
+		const clampedOverlap = Math.min(overlap, Math.max(0, chunkSize - 1));
+		const effectiveChunkSize = Math.max(1, chunkSize - clampedOverlap);
+		estimatedChunks = Math.ceil((estimatedTokens - clampedOverlap) / effectiveChunkSize);
+	}
+	return {
+		inputLength,
+		estimatedTokens,
+		estimatedChunks,
+		willTruncate
+	};
+}
+
+//#endregion
+//#region src/embeddings/input.ts
+/**
+* Select and prepare input text for embedding based on configuration.
+*
+* @param data - Scraped data to extract input from
+* @param config - Input configuration
+* @returns Selected and prepared text, or undefined if no valid input
+*/
+function selectInput(data, config) {
+	if (config?.transform) return normalizeText(config.transform(data));
+	if (config?.type === "custom" && config.customText) return normalizeText(config.customText);
+	const type = config?.type ?? "textContent";
+	switch (type) {
+		case "textContent": return selectTextContent(data);
+		case "title+summary": return selectTitleSummary(data);
+		case "custom": return selectTextContent(data);
+		default: {
+			const _exhaustive = type;
+			throw new Error(`Unknown input type: ${_exhaustive}`);
+		}
+	}
+}
+/**
+* Select textContent as input.
+*/
+function selectTextContent(data) {
+	if (data.textContent) return normalizeText(data.textContent);
+	if (data.content) return normalizeText(stripMarkdown(data.content));
+	if (data.excerpt) return normalizeText(data.excerpt);
+	if (data.description) return normalizeText(data.description);
+}
+/**
+* Select title + summary (or fallbacks) as input.
+* Optimized for semantic search and classification.
+*/
+function selectTitleSummary(data) {
+	const parts = [];
+	if (data.title) parts.push(data.title);
+	if (data.summary) parts.push(data.summary);
+	else if (data.excerpt) parts.push(data.excerpt);
+	else if (data.description) parts.push(data.description);
+	if (parts.length === 0) return;
+	return normalizeText(parts.join("\n\n"));
+}
+/**
+* Normalize text for embedding:
+* - Collapse whitespace
+* - Trim leading/trailing whitespace
+* - Remove control characters
+*/
+function normalizeText(text) {
+	if (!text) return "";
+	return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").split("\n").map((line) => line.trim()).join("\n").trim();
+}
+/**
+* Basic markdown stripping for when we need plain text from content.
+* Not comprehensive, but handles common cases.
+*/
+function stripMarkdown(markdown) {
+	return markdown.replace(/```[\s\S]*?```/g, "").replace(/`[^`]+`/g, "").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replace(/^#{1,6}\s+/gm, "").replace(/\*\*([^*]+)\*\*/g, "$1").replace(/\*([^*]+)\*/g, "$1").replace(/__([^_]+)__/g, "$1").replace(/_([^_]+)_/g, "$1").replace(/^>\s+/gm, "").replace(/^[-*_]{3,}$/gm, "").replace(/^[\s]*[-*+]\s+/gm, "").replace(/^[\s]*\d+\.\s+/gm, "");
+}
+/**
+* Check if the selected input meets minimum requirements.
+*/
+function validateInput(text, minLength = 10) {
+	if (!text) return {
+		valid: false,
+		reason: "No input text available"
+	};
+	if (text.length < minLength) return {
+		valid: false,
+		reason: `Input too short (${text.length} < ${minLength} characters)`
+	};
+	const wordCount = text.split(/\s+/).filter((w) => w.length > 1).length;
+	if (wordCount < 3) return {
+		valid: false,
+		reason: `Input has too few words (${wordCount} < 3)`
+	};
+	return {
+		valid: true,
+		text,
+		wordCount,
+		charCount: text.length
+	};
+}
+/**
+* Get a preview of what input would be selected.
+* Useful for debugging and testing.
+*/
+function previewInput(data, config, maxLength = 200) {
+	const input = selectInput(data, config);
+	if (!input) return "[No input available]";
+	if (input.length <= maxLength) return input;
+	return `${input.slice(0, maxLength)}...`;
+}
+
+//#endregion
+//#region src/embeddings/providers/base.ts
+/**
+* Generate a stable cache key identifier for provider configuration.
+*/
+function getProviderCacheKey(config) {
+	switch (config.type) {
+		case "http": return `http:${config.config.baseUrl.replace(/\/$/, "")}:${config.config.model}`;
+		case "custom": return `custom:${config.provider.name}`;
+		default: {
+			const _exhaustive = config;
+			return String(_exhaustive);
+		}
+	}
+}
+/**
+* Get default model for a provider type.
+*/
+function getDefaultModel(providerType) {
+	switch (providerType) {
+		case "openai": return "text-embedding-3-small";
+		case "azure": return "text-embedding-ada-002";
+		case "transformers": return "Xenova/all-MiniLM-L6-v2";
+		default: return "default";
+	}
+}
+
+//#endregion
+//#region src/embeddings/providers/http.ts
+/**
+* HTTP-based Embedding Provider using native fetch.
+* Provides a unified interface for any REST-based embedding API.
+*/
+/**
+* HTTP-based embedding provider.
+* Works with any REST API using native fetch.
+*/
+var HttpEmbeddingProvider = class extends BaseHttpProvider {
+	name = "http-embedding";
+	requestBuilder;
+	responseMapper;
+	constructor(config) {
+		super(config);
+		this.requestBuilder = config.requestBuilder ?? ((texts, model) => ({
+			input: texts,
+			model
+		}));
+		this.responseMapper = config.responseMapper ?? ((response) => {
+			const resp = response;
+			if (Array.isArray(resp.data)) return resp.data.map((item) => item.embedding);
+			if (Array.isArray(resp.embeddings)) return resp.embeddings;
+			if (Array.isArray(resp.embedding)) return [resp.embedding];
+			if (Array.isArray(response)) return response;
+			throw new ScrapeError("Unable to parse embedding response. Provide a custom responseMapper.", "VALIDATION_ERROR");
+		});
+	}
+	/**
+	* Generate embeddings for one or more texts.
+	*/
+	async embed(texts, options) {
+		const model = options.model || this.model;
+		const body = this.requestBuilder(texts, model);
+		const { data } = await this.fetch(this.baseUrl, {
+			body,
+			signal: options.signal
+		});
+		const embeddings = this.responseMapper(data);
+		if (embeddings.length !== texts.length) throw new ScrapeError(`Embedding count mismatch: expected ${texts.length}, got ${embeddings.length}`, "VALIDATION_ERROR");
+		return { embeddings };
+	}
+};
+/**
+* Create a generic HTTP embedding provider.
+*/
+function createHttpEmbedding(config) {
+	return new HttpEmbeddingProvider(config);
+}
+
+//#endregion
+//#region src/embeddings/providers/presets.ts
+/**
+* Create an OpenAI embedding provider.
+*
+* @example
+* ```ts
+* const provider = createOpenAIEmbedding({ apiKey: 'sk-...' });
+* const { embeddings } = await provider.embed(['Hello'], { model: 'text-embedding-3-small' });
+* ```
+*/
+function createOpenAIEmbedding(options) {
+	const apiKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
+	if (!apiKey) throw new Error("OpenAI API key required. Set OPENAI_API_KEY env var or pass apiKey option.");
+	const headers = { Authorization: `Bearer ${apiKey}` };
+	if (options?.organization) headers["OpenAI-Organization"] = options.organization;
+	return new HttpEmbeddingProvider({
+		baseUrl: options?.baseUrl ?? "https://api.openai.com/v1/embeddings",
+		model: options?.model ?? "text-embedding-3-small",
+		headers,
+		requestBuilder: (texts, model) => ({
+			input: texts,
+			model
+		}),
+		responseMapper: (res) => res.data.map((item) => item.embedding)
+	});
+}
+/**
+* Create an Azure OpenAI embedding provider.
+*
+* @example
+* ```ts
+* const provider = createAzureEmbedding({
+*   endpoint: 'https://my-resource.openai.azure.com',
+*   deploymentName: 'text-embedding-ada-002',
+*   apiVersion: '2023-05-15',
+* });
+* ```
+*/
+function createAzureEmbedding(options) {
+	const apiKey = options.apiKey ?? process.env.AZURE_OPENAI_API_KEY;
+	if (!apiKey) throw new Error("Azure OpenAI API key required. Set AZURE_OPENAI_API_KEY env var or pass apiKey option.");
+	return new HttpEmbeddingProvider({
+		baseUrl: `${options.endpoint.replace(/\/$/, "")}/openai/deployments/${options.deploymentName}/embeddings?api-version=${options.apiVersion}`,
+		model: options.deploymentName,
+		headers: { "api-key": apiKey },
+		requestBuilder: (texts) => ({ input: texts }),
+		responseMapper: (res) => res.data.map((item) => item.embedding)
+	});
+}
+/**
+* Create an Ollama embedding provider for local models.
+*
+* LIMITATION: Ollama's /api/embeddings endpoint processes one text at a time,
+* not batches. When multiple chunks are embedded, each chunk triggers a
+* separate HTTP request. This is handled transparently by the pipeline's
+* sequential chunk processing, but may be slower than batch-capable providers.
+* For high-throughput scenarios, consider using OpenAI, Cohere, or HuggingFace
+* which support batch embedding in a single request.
+*
+* @example
+* ```ts
+* const provider = createOllamaEmbedding({ model: 'nomic-embed-text' });
+* ```
+*/
+function createOllamaEmbedding(options) {
+	return new HttpEmbeddingProvider({
+		baseUrl: options?.baseUrl ?? "http://localhost:11434/api/embeddings",
+		model: options?.model ?? "nomic-embed-text",
+		requireHttps: false,
+		allowPrivate: true,
+		requestBuilder: (texts, model) => ({
+			model,
+			prompt: texts[0]
+		}),
+		responseMapper: (res) => [res.embedding]
+	});
+}
+/**
+* Create a HuggingFace Inference API embedding provider.
+*
+* @example
+* ```ts
+* const provider = createHuggingFaceEmbedding({
+*   model: 'sentence-transformers/all-MiniLM-L6-v2',
+* });
+* ```
+*/
+function createHuggingFaceEmbedding(options) {
+	const apiKey = options.apiKey ?? process.env.HF_TOKEN ?? process.env.HUGGINGFACE_API_KEY;
+	const headers = {};
+	if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
+	return new HttpEmbeddingProvider({
+		baseUrl: `https://api-inference.huggingface.co/models/${options.model}`,
+		model: options.model,
+		headers,
+		requestBuilder: (texts) => ({ inputs: texts }),
+		responseMapper: (response) => {
+			if (Array.isArray(response)) {
+				if (Array.isArray(response[0]) && typeof response[0][0] === "number") return response;
+				return [response];
+			}
+			throw new Error("Unexpected HuggingFace response format");
+		}
+	});
+}
+/**
+* Create a Cohere embedding provider.
+*
+* @example
+* ```ts
+* const provider = createCohereEmbedding({ model: 'embed-english-v3.0' });
+* ```
+*/
+function createCohereEmbedding(options) {
+	const apiKey = options?.apiKey ?? process.env.COHERE_API_KEY;
+	if (!apiKey) throw new Error("Cohere API key required. Set COHERE_API_KEY env var or pass apiKey option.");
+	return new HttpEmbeddingProvider({
+		baseUrl: "https://api.cohere.ai/v1/embed",
+		model: options?.model ?? "embed-english-v3.0",
+		headers: { Authorization: `Bearer ${apiKey}` },
+		requestBuilder: (texts, model) => ({
+			texts,
+			model,
+			input_type: options?.inputType ?? "search_document"
+		}),
+		responseMapper: (res) => res.embeddings
+	});
+}
+/**
+* Create a local Transformers.js embedding provider.
+* Uses dependency injection - user provides the imported transformers module.
+*
+* @example
+* ```typescript
+* import * as transformers from '@huggingface/transformers';
+* import { createTransformersEmbedding } from 'scrapex/embeddings';
+*
+* const provider = createTransformersEmbedding(transformers, {
+*   model: 'Xenova/all-MiniLM-L6-v2',
+* });
+* ```
+*
+* Required Node.js dependencies:
+* ```
+* npm install @huggingface/transformers onnxruntime-node
+* ```
+*/
+function createTransformersEmbedding(transformers, options) {
+	let pipeline = null;
+	let currentModel = null;
+	const config = {
+		model: options?.model ?? "Xenova/all-MiniLM-L6-v2",
+		quantized: options?.quantized ?? true,
+		pooling: options?.pooling ?? "mean",
+		normalize: options?.normalize ?? true
+	};
+	return {
+		name: "transformers",
+		async embed(texts, request) {
+			const model = request.model || config.model;
+			if (!pipeline || currentModel !== model) {
+				const cacheDir = options?.cacheDir;
+				const env = transformers.env;
+				const priorCacheDir = env?.cacheDir;
+				if (cacheDir && env) env.cacheDir = cacheDir;
+				try {
+					pipeline = await transformers.pipeline("feature-extraction", model, { quantized: config.quantized });
+				} finally {
+					if (cacheDir && env) if (priorCacheDir === void 0) delete env.cacheDir;
+					else env.cacheDir = priorCacheDir;
+				}
+				currentModel = model;
+			}
+			const embeddings = [];
+			for (const text of texts) {
+				const output = await pipeline(text, {
+					pooling: config.pooling,
+					normalize: config.normalize
+				});
+				embeddings.push(Array.from(output.data));
+			}
+			return { embeddings };
+		}
+	};
+}
+/** Recommended models for Transformers.js */
+const TRANSFORMERS_MODELS = {
+	DEFAULT: "Xenova/all-MiniLM-L6-v2",
+	QUALITY: "Xenova/all-mpnet-base-v2",
+	RETRIEVAL: "Xenova/bge-small-en-v1.5",
+	MULTILINGUAL: "Xenova/multilingual-e5-small"
+};
+
+//#endregion
+//#region src/embeddings/providers/index.ts
+/**
+* Create an embedding provider from configuration.
+* This is the main factory function for creating providers.
+*/
+function createEmbeddingProvider(config) {
+	switch (config.type) {
+		case "http": return createHttpEmbedding(config.config);
+		case "custom": return config.provider;
+		default: throw new ScrapeError(`Unknown embedding provider type: ${config.type}`, "VALIDATION_ERROR");
+	}
+}
+/**
+* Type guard to check if a value is an EmbeddingProvider.
+*/
+function isEmbeddingProvider(value) {
+	return typeof value === "object" && value !== null && "name" in value && typeof value.name === "string" && "embed" in value && typeof value.embed === "function";
+}
+
+//#endregion
+//#region src/embeddings/safety.ts
+/**
+* PII redaction patterns with high precision to minimize false positives.
+* Patterns are designed to match common formats while avoiding over-matching.
+*/
+const EMAIL_PATTERN = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
+const PHONE_PATTERN = /(?:\+?1[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}\b/g;
+const CREDIT_CARD_PATTERN = /\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12}|(?:[0-9]{4}[-\s]){3}[0-9]{4}|[0-9]{13,19})\b/g;
+const SSN_PATTERN = /\b[0-9]{3}-[0-9]{2}-[0-9]{4}\b/g;
+const IPV4_PATTERN = /\b(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/g;
+const REDACTED = "[REDACTED]";
+/**
+* Create a redaction function based on configuration.
+* Returns a function that applies all configured PII patterns.
+*/
+function createPiiRedactor(config) {
+	const patterns = [];
+	if (config.creditCard) patterns.push({
+		name: "creditCard",
+		pattern: CREDIT_CARD_PATTERN
+	});
+	if (config.email) patterns.push({
+		name: "email",
+		pattern: EMAIL_PATTERN
+	});
+	if (config.phone) patterns.push({
+		name: "phone",
+		pattern: PHONE_PATTERN
+	});
+	if (config.ssn) patterns.push({
+		name: "ssn",
+		pattern: SSN_PATTERN
+	});
+	if (config.ipAddress) patterns.push({
+		name: "ipAddress",
+		pattern: IPV4_PATTERN
+	});
+	if (config.customPatterns) for (let i = 0; i < config.customPatterns.length; i++) {
+		const customPattern = config.customPatterns[i];
+		if (customPattern) patterns.push({
+			name: `custom_${i}`,
+			pattern: customPattern
+		});
+	}
+	return (text) => {
+		let redactedText = text;
+		let totalRedactions = 0;
+		const redactionsByType = {};
+		for (const { name, pattern } of patterns) {
+			pattern.lastIndex = 0;
+			const matchCount = text.match(pattern)?.length ?? 0;
+			if (matchCount > 0) {
+				redactedText = redactedText.replace(pattern, REDACTED);
+				totalRedactions += matchCount;
+				redactionsByType[name] = (redactionsByType[name] ?? 0) + matchCount;
+			}
+		}
+		return {
+			text: redactedText,
+			redacted: totalRedactions > 0,
+			redactionCount: totalRedactions,
+			redactionsByType
+		};
+	};
+}
+/**
+* Simple redaction that applies all default patterns.
+* Use createPiiRedactor() for fine-grained control.
+*/
+function redactPii(text) {
+	return createPiiRedactor({
+		email: true,
+		phone: true,
+		creditCard: true,
+		ssn: true,
+		ipAddress: true
+	})(text);
+}
+/**
+* Check if text contains any PII.
+* Useful for validation before sending to external APIs.
+*/
+function containsPii(text, config) {
+	const fullConfig = {
+		email: config?.email ?? true,
+		phone: config?.phone ?? true,
+		creditCard: config?.creditCard ?? true,
+		ssn: config?.ssn ?? true,
+		ipAddress: config?.ipAddress ?? true,
+		customPatterns: config?.customPatterns
+	};
+	const patterns = [];
+	if (fullConfig.email) patterns.push(EMAIL_PATTERN);
+	if (fullConfig.phone) patterns.push(PHONE_PATTERN);
+	if (fullConfig.creditCard) patterns.push(CREDIT_CARD_PATTERN);
+	if (fullConfig.ssn) patterns.push(SSN_PATTERN);
+	if (fullConfig.ipAddress) patterns.push(IPV4_PATTERN);
+	if (fullConfig.customPatterns) patterns.push(...fullConfig.customPatterns);
+	for (const pattern of patterns) {
+		pattern.lastIndex = 0;
+		if (pattern.test(text)) return true;
+	}
+	return false;
+}
+
+//#endregion
+//#region src/embeddings/pipeline.ts
+const DEFAULT_CHUNK_SIZE = 500;
+/**
+* Get the effective model for embedding.
+* Prioritizes: explicit options.model > provider config model
+*/
+function getEffectiveModel(providerConfig, explicitModel) {
+	if (explicitModel) return explicitModel;
+	if (providerConfig.type === "http") return providerConfig.config.model;
+}
+/**
+* Generate embeddings for scraped data.
+* This is the main entry point for the embedding pipeline.
+*/
+async function generateEmbeddings(data, options) {
+	const startTime = Date.now();
+	try {
+		const provider = createEmbeddingProvider(options.provider);
+		const model = getEffectiveModel(options.provider, options.model);
+		const validation = validateInput(selectInput(data, options.input), options.safety?.minTextLength ?? 10);
+		if (!validation.valid) return createSkippedResult(validation.reason, { model });
+		const originalInput = validation.text;
+		let inputText = validation.text;
+		let piiRedacted = false;
+		if (options.safety?.piiRedaction) {
+			const redactionResult = createPiiRedactor(options.safety.piiRedaction)(inputText);
+			inputText = redactionResult.text;
+			piiRedacted = redactionResult.redacted;
+		}
+		const effectiveChunking = applyMaxTokensToChunking(options.chunking, options.safety?.maxTokens);
+		const cacheKey = generateCacheKey({
+			providerKey: getProviderCacheKey(options.provider),
+			model,
+			dimensions: options.output?.dimensions,
+			aggregation: options.output?.aggregation,
+			input: options.input,
+			chunking: effectiveChunking,
+			safety: options.safety,
+			cacheKeySalt: options.cache?.cacheKeySalt,
+			content: inputText
+		});
+		const cache = options.cache?.store ?? getDefaultCache();
+		const cachedResult = await cache.get(cacheKey);
+		if (cachedResult && cachedResult.status === "success") {
+			if (options.onMetrics) options.onMetrics({
+				provider: provider.name,
+				model,
+				inputTokens: estimateTokens(inputText),
+				outputDimensions: getDimensions(cachedResult.aggregation === "all" ? cachedResult.vectors : cachedResult.vector),
+				chunks: cachedResult.source.chunks,
+				latencyMs: Date.now() - startTime,
+				cached: true,
+				retries: 0,
+				piiRedacted
+			});
+			return {
+				...cachedResult,
+				source: {
+					...cachedResult.source,
+					cached: true
+				}
+			};
+		}
+		const chunks = chunkText(inputText, effectiveChunking);
+		const callbackChunks = options.onChunk && options.safety?.allowSensitiveCallbacks ? chunkText(originalInput, effectiveChunking) : null;
+		if (chunks.length === 0) return createSkippedResult("No content after chunking", { model });
+		const sharedState = options.resilience?.state;
+		const rateLimiter = sharedState?.rateLimiter ?? (options.resilience?.rateLimit ? new RateLimiter(options.resilience.rateLimit) : null);
+		const circuitBreaker = sharedState?.circuitBreaker ?? (options.resilience?.circuitBreaker ? new CircuitBreaker(options.resilience.circuitBreaker) : null);
+		const concurrency = options.resilience?.concurrency ?? 1;
+		const semaphore = sharedState?.semaphore ?? new Semaphore(concurrency);
+		const embeddings = [];
+		let totalTokens = 0;
+		let retryCount = 0;
+		for (let i = 0; i < chunks.length; i++) {
+			const chunk = chunks[i];
+			if (!chunk) continue;
+			if (rateLimiter) await rateLimiter.acquire();
+			if (circuitBreaker?.isOpen()) return createSkippedResult("Circuit breaker is open", {
+				model,
+				chunks: i
+			});
+			await semaphore.execute(async () => {
+				const { result: result$1 } = await withResilience(async (signal) => {
+					return provider.embed([chunk.text], {
+						model,
+						dimensions: options.output?.dimensions,
+						signal
+					});
+				}, options.resilience, {
+					circuitBreaker: circuitBreaker ?? void 0,
+					rateLimiter: void 0,
+					semaphore: void 0
+				}, { onRetry: () => {
+					retryCount++;
+				} });
+				if (result$1.usage) totalTokens += result$1.usage.totalTokens;
+				else totalTokens += chunk.tokens;
+				const embedding = result$1.embeddings[0];
+				if (embedding) {
+					embeddings.push(embedding);
+					if (options.onChunk) {
+						const callbackText = callbackChunks?.[i]?.text ?? chunk.text;
+						options.onChunk(callbackText, embedding);
+					}
+				}
+			});
+		}
+		const aggregation = options.output?.aggregation ?? "average";
+		const aggregated = aggregateVectors(embeddings, aggregation);
+		const source = {
+			model,
+			chunks: chunks.length,
+			tokens: totalTokens || estimateTokens(inputText),
+			checksum: generateChecksum(inputText),
+			cached: false,
+			latencyMs: Date.now() - startTime
+		};
+		let result;
+		if (aggregated.type === "single") result = {
+			status: "success",
+			aggregation,
+			vector: aggregated.vector,
+			source
+		};
+		else result = {
+			status: "success",
+			aggregation: "all",
+			vectors: aggregated.vectors,
+			source
+		};
+		await cache.set(cacheKey, result, { ttlMs: options.cache?.ttlMs });
+		if (options.onMetrics) {
+			const metrics = {
+				provider: provider.name,
+				model,
+				inputTokens: source.tokens,
+				outputDimensions: aggregated.dimensions,
+				chunks: chunks.length,
+				latencyMs: source.latencyMs,
+				cached: false,
+				retries: retryCount,
+				piiRedacted
+			};
+			options.onMetrics(metrics);
+		}
+		return result;
+	} catch (error) {
+		const reason = error instanceof Error ? error.message : String(error);
+		if (error instanceof ScrapeError && ["INVALID_URL", "BLOCKED"].includes(error.code)) throw error;
+		return createSkippedResult(reason, { latencyMs: Date.now() - startTime });
+	}
+}
+function applyMaxTokensToChunking(chunking, maxTokens) {
+	if (!maxTokens || maxTokens <= 0) return chunking;
+	const baseSize = chunking?.size ?? DEFAULT_CHUNK_SIZE;
+	const baseOverlap = chunking?.overlap ?? 50;
+	const clampedSize = Math.min(baseSize, maxTokens);
+	const clampedOverlap = Math.min(baseOverlap, Math.max(0, clampedSize - 1));
+	return {
+		...chunking,
+		size: clampedSize,
+		overlap: clampedOverlap
+	};
+}
+/**
+* Embed arbitrary text directly.
+* Standalone function for embedding text outside of scrape().
+*/
+async function embed(text, options) {
+	return generateEmbeddings({ textContent: text }, {
+		...options,
+		input: {
+			...options.input,
+			type: "textContent"
+		}
+	});
+}
+/**
+* Embed from existing ScrapedData.
+* Useful when you've already scraped and want to add embeddings later.
+*/
+async function embedScrapedData(data, options) {
+	return generateEmbeddings(data, options);
+}
+/**
+* Create a skipped result with reason.
+*/
+function createSkippedResult(reason, partialSource) {
+	return {
+		status: "skipped",
+		reason,
+		source: partialSource ?? {}
+	};
+}
+
+//#endregion
+export { generateCacheKey as A, normalizeVector as B, createTokenizer as C, needsChunking as D, heuristicTokenCount as E, aggregateVectors as F, cosineSimilarity as I, dotProduct as L, getDefaultCache as M, resetDefaultCache as N, InMemoryEmbeddingCache as O, validateCachedResult as P, euclideanDistance as R, chunkText as S, getChunkingStats as T, createHttpEmbedding as _, createPiiRedactor as a, selectInput as b, isEmbeddingProvider as c, createCohereEmbedding as d, createHuggingFaceEmbedding as f, HttpEmbeddingProvider as g, createTransformersEmbedding as h, containsPii as i, generateChecksum as j, createNoOpCache as k, TRANSFORMERS_MODELS as l, createOpenAIEmbedding as m, embedScrapedData as n, redactPii as o, createOllamaEmbedding as p, generateEmbeddings as r, createEmbeddingProvider as s, embed as t, createAzureEmbedding as u, getDefaultModel as v, estimateTokens as w, validateInput as x, previewInput as y, getDimensions as z };
+//# sourceMappingURL=embeddings-Bsymy_jA.mjs.map