scrapex 1.0.0-beta.1 → 1.0.0-beta.2

package/dist/index.mjs

@@ -1,7 +1,8 @@
-import { c as BaseHttpProvider, d as Semaphore, f as withResilience, l as CircuitBreaker, n as enhance, p as ScrapeError, r as extract, u as RateLimiter } from "./enhancer-ByjRD-t5.mjs";
-import { t as RSSParser } from "./parsers-CwkYnyWY.mjs";
+import { d as ScrapeError } from "./http-base-DM7YNo6X.mjs";
+import { F as aggregateVectors, I as cosineSimilarity, O as InMemoryEmbeddingCache, S as chunkText, _ as createHttpEmbedding, a as createPiiRedactor, f as createHuggingFaceEmbedding, h as createTransformersEmbedding, l as TRANSFORMERS_MODELS, m as createOpenAIEmbedding, n as embedScrapedData, o as redactPii, p as createOllamaEmbedding, r as generateEmbeddings, s as createEmbeddingProvider, t as embed, u as createAzureEmbedding, w as estimateTokens } from "./embeddings-Bsymy_jA.mjs";
+import { n as enhance, r as extract } from "./enhancer-INx5NlgO.mjs";
+import { t as RSSParser } from "./parsers-DsawHeo0.mjs";
 import * as cheerio from "cheerio";
-import { createHash } from "node:crypto";
 import { Readability } from "@mozilla/readability";
 import TurndownService from "turndown";
 
@@ -54,1042 +55,6 @@ function mergeResults(context, extracted) {
 	};
 }
 
-//#endregion
-//#region src/embeddings/aggregation.ts
-/**
-* Aggregate multiple embedding vectors into a single vector or return all.
-*
-* @param vectors - Array of embedding vectors (must all have same dimensions)
-* @param strategy - Aggregation strategy
-* @returns Aggregated result based on strategy
-*/
-function aggregateVectors(vectors, strategy = "average") {
-	if (vectors.length === 0) throw new Error("Cannot aggregate empty vector array");
-	const firstVector = vectors[0];
-	if (!firstVector) throw new Error("Cannot aggregate empty vector array");
-	const dimensions = firstVector.length;
-	for (let i = 1; i < vectors.length; i++) {
-		const vec = vectors[i];
-		if (!vec || vec.length !== dimensions) throw new Error(`Vector dimension mismatch: expected ${dimensions}, got ${vec?.length ?? 0} at index ${i}`);
-	}
-	switch (strategy) {
-		case "average": return {
-			type: "single",
-			vector: averageVectors(vectors),
-			dimensions
-		};
-		case "max": return {
-			type: "single",
-			vector: maxPoolVectors(vectors),
-			dimensions
-		};
-		case "first": return {
-			type: "single",
-			vector: firstVector,
-			dimensions
-		};
-		case "all": return {
-			type: "multiple",
-			vectors,
-			dimensions
-		};
-		default: {
-			const _exhaustive = strategy;
-			throw new Error(`Unknown aggregation strategy: ${_exhaustive}`);
-		}
-	}
-}
-/**
-* Compute element-wise average of vectors.
-*/
-function averageVectors(vectors) {
-	const first = vectors[0];
-	if (!first || vectors.length === 1) return first ?? [];
-	const dimensions = first.length;
-	const count = vectors.length;
-	const result = new Array(dimensions).fill(0);
-	for (const vector of vectors) for (let i = 0; i < dimensions; i++) {
-		const val = result[i];
-		if (val !== void 0) result[i] = val + (vector[i] ?? 0);
-	}
-	for (let i = 0; i < dimensions; i++) {
-		const val = result[i];
-		if (val !== void 0) result[i] = val / count;
-	}
-	return result;
-}
-/**
-* Compute element-wise maximum of vectors (max pooling).
-*/
-function maxPoolVectors(vectors) {
-	const first = vectors[0];
-	if (!first || vectors.length === 1) return first ?? [];
-	const dimensions = first.length;
-	const result = [...first];
-	for (let v = 1; v < vectors.length; v++) {
-		const vec = vectors[v];
-		if (!vec) continue;
-		for (let i = 0; i < dimensions; i++) {
-			const val = vec[i] ?? 0;
-			if (val > (result[i] ?? 0)) result[i] = val;
-		}
-	}
-	return result;
-}
-/**
-* Compute cosine similarity between two vectors.
-* Both vectors should be normalized for accurate results.
-*/
-function cosineSimilarity(a, b) {
-	if (a.length !== b.length) throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
-	let dot = 0;
-	let magnitudeA = 0;
-	let magnitudeB = 0;
-	for (let i = 0; i < a.length; i++) {
-		const aVal = a[i] ?? 0;
-		const bVal = b[i] ?? 0;
-		dot += aVal * bVal;
-		magnitudeA += aVal * aVal;
-		magnitudeB += bVal * bVal;
-	}
-	const magnitude = Math.sqrt(magnitudeA) * Math.sqrt(magnitudeB);
-	if (magnitude === 0) return 0;
-	return dot / magnitude;
-}
-/**
-* Get the dimensions of a vector or set of vectors.
-*/
-function getDimensions(vectors) {
-	if (vectors.length === 0) return 0;
-	const first = vectors[0];
-	if (typeof first === "number") return vectors.length;
-	return first?.length ?? 0;
-}
-
-//#endregion
-//#region src/embeddings/cache.ts
-/**
-* Default maximum cache entries.
-*/
-const DEFAULT_MAX_ENTRIES = 1e3;
-/**
-* Default TTL in milliseconds (1 hour).
-*/
-const DEFAULT_TTL_MS = 3600 * 1e3;
-/**
-* Generate a content-addressable cache key.
-* Key is based on content hash and embedding configuration.
-* Note: custom RegExp patterns are serialized by source+flags; different
-* constructions can yield different cache keys even if equivalent.
-*/
-function generateCacheKey(params) {
-	const hash = createHash("sha256");
-	const fingerprint = stableStringify({
-		providerKey: params.providerKey,
-		model: params.model ?? "provider-default",
-		dimensions: params.dimensions ?? "default",
-		aggregation: params.aggregation ?? "average",
-		input: serializeInputConfig(params.input),
-		chunking: serializeChunkingConfig(params.chunking),
-		safety: serializeSafetyConfig(params.safety),
-		cacheKeySalt: params.cacheKeySalt
-	});
-	hash.update(fingerprint);
-	hash.update("\0");
-	hash.update(params.content);
-	return hash.digest("hex");
-}
-/**
-* Generate a checksum for content verification.
-*/
-function generateChecksum(content) {
-	return createHash("sha256").update(content).digest("hex").slice(0, 16);
-}
-function serializeInputConfig(config) {
-	if (!config) return void 0;
-	return normalizeObject({
-		type: config.type ?? "textContent",
-		hasTransform: Boolean(config.transform),
-		hasCustomText: Boolean(config.customText)
-	});
-}
-function serializeChunkingConfig(config) {
-	if (!config) return void 0;
-	return normalizeObject({
-		size: config.size,
-		overlap: config.overlap,
-		tokenizer: getTokenizerId(config.tokenizer),
-		maxInputLength: config.maxInputLength
-	});
-}
-function serializeSafetyConfig(config) {
-	if (!config) return void 0;
-	return normalizeObject({
-		piiRedaction: serializePiiConfig(config.piiRedaction),
-		minTextLength: config.minTextLength,
-		maxTokens: config.maxTokens
-	});
-}
-function serializePiiConfig(config) {
-	if (!config) return void 0;
-	return normalizeObject({
-		email: config.email ?? false,
-		phone: config.phone ?? false,
-		creditCard: config.creditCard ?? false,
-		ssn: config.ssn ?? false,
-		ipAddress: config.ipAddress ?? false,
-		customPatterns: config.customPatterns?.map((pattern) => `${pattern.source}/${pattern.flags}`)
-	});
-}
-function getTokenizerId(tokenizer) {
-	if (!tokenizer || tokenizer === "heuristic") return "heuristic";
-	if (tokenizer === "tiktoken") return "tiktoken";
-	return "custom";
-}
-function stableStringify(value) {
-	return stringifyNormalized(normalizeValue(value));
-}
-function normalizeValue(value) {
-	if (value === void 0) return void 0;
-	if (value === null) return null;
-	if (Array.isArray(value)) return value.map((entry) => normalizeValue(entry)).filter((entry) => entry !== void 0);
-	if (typeof value === "object") return normalizeObject(value);
-	return value;
-}
-function normalizeObject(value) {
-	const normalized = {};
-	for (const key of Object.keys(value).sort()) {
-		const entry = normalizeValue(value[key]);
-		if (entry !== void 0) normalized[key] = entry;
-	}
-	return normalized;
-}
-function stringifyNormalized(value) {
-	if (value === void 0) return "undefined";
-	if (value === null) return "null";
-	if (typeof value === "string") return JSON.stringify(value);
-	if (typeof value === "number" || typeof value === "boolean") return String(value);
-	if (Array.isArray(value)) return `[${value.map((entry) => stringifyNormalized(entry)).join(",")}]`;
-	if (typeof value === "object") {
-		const obj = value;
-		return `{${Object.keys(obj).sort().map((key) => `${JSON.stringify(key)}:${stringifyNormalized(obj[key])}`).join(",")}}`;
-	}
-	return JSON.stringify(value);
-}
-/**
-* In-memory LRU cache with TTL support.
-* Content-addressable: uses content hash as key, not URL.
-*/
-var InMemoryEmbeddingCache = class {
-	cache;
-	maxEntries;
-	defaultTtlMs;
-	constructor(options) {
-		this.cache = /* @__PURE__ */ new Map();
-		this.maxEntries = options?.maxEntries ?? DEFAULT_MAX_ENTRIES;
-		this.defaultTtlMs = options?.ttlMs ?? DEFAULT_TTL_MS;
-	}
-	async get(key) {
-		const entry = this.cache.get(key);
-		if (!entry) return;
-		const now = Date.now();
-		if (now > entry.expiresAt) {
-			this.cache.delete(key);
-			return;
-		}
-		entry.accessedAt = now;
-		return entry.value;
-	}
-	async set(key, value, options) {
-		const now = Date.now();
-		const ttl = options?.ttlMs ?? this.defaultTtlMs;
-		if (this.cache.size >= this.maxEntries && !this.cache.has(key)) this.evictLRU();
-		this.cache.set(key, {
-			value,
-			createdAt: now,
-			expiresAt: now + ttl,
-			accessedAt: now
-		});
-	}
-	async delete(key) {
-		return this.cache.delete(key);
-	}
-	async clear() {
-		this.cache.clear();
-	}
-	/**
-	* Get cache statistics.
-	*/
-	getStats() {
-		const now = Date.now();
-		let expired = 0;
-		for (const entry of this.cache.values()) if (now > entry.expiresAt) expired++;
-		return {
-			size: this.cache.size,
-			maxEntries: this.maxEntries,
-			expired,
-			utilization: this.cache.size / this.maxEntries
-		};
-	}
-	/**
-	* Evict expired entries.
-	*/
-	cleanup() {
-		const now = Date.now();
-		let evicted = 0;
-		for (const [key, entry] of this.cache.entries()) if (now > entry.expiresAt) {
-			this.cache.delete(key);
-			evicted++;
-		}
-		return evicted;
-	}
-	/**
-	* Evict least recently used entry.
-	*/
-	evictLRU() {
-		let oldestKey = null;
-		let oldestAccess = Number.POSITIVE_INFINITY;
-		for (const [key, entry] of this.cache.entries()) if (entry.accessedAt < oldestAccess) {
-			oldestAccess = entry.accessedAt;
-			oldestKey = key;
-		}
-		if (oldestKey) this.cache.delete(oldestKey);
-	}
-};
-/**
-* Default in-memory cache instance.
-* Optimized for moderate cache sizes (default 1000 entries).
-*/
-let defaultCache = null;
-/**
-* Get or create the default cache instance.
-*/
-function getDefaultCache() {
-	if (!defaultCache) defaultCache = new InMemoryEmbeddingCache();
-	return defaultCache;
-}
-
-//#endregion
-//#region src/embeddings/chunking.ts
-/**
-* Default chunk size in tokens.
-*/
-const DEFAULT_CHUNK_SIZE$1 = 500;
-/**
-* Default overlap in tokens.
-*/
-const DEFAULT_OVERLAP = 50;
-/**
-* Default maximum input length in characters.
-*/
-const DEFAULT_MAX_INPUT_LENGTH = 1e5;
-/**
-* Heuristic token counting: approximately 4 characters per token.
-* This is a reasonable approximation for English text.
-*/
-function heuristicTokenCount(text) {
-	return Math.ceil(text.length / 4);
-}
-/**
-* Convert token count to approximate character count.
-*/
-function tokensToChars(tokens) {
-	return tokens * 4;
-}
-/**
-* Create a tokenizer function based on configuration.
-*/
-function createTokenizer(config) {
-	if (!config || config === "heuristic") return heuristicTokenCount;
-	if (config === "tiktoken") return heuristicTokenCount;
-	return config;
-}
-/**
-* Find a natural break point in text (sentence or word boundary).
-* Prefers common sentence boundaries (Latin + CJK), falls back to word boundaries.
-*/
-function findBreakPoint(text, targetIndex) {
-	const searchStart = Math.max(0, targetIndex - Math.floor(targetIndex * .2));
-	const searchEnd = Math.min(text.length, targetIndex + Math.floor(targetIndex * .2));
-	const searchText = text.slice(searchStart, searchEnd);
-	const sentenceMatch = /[.!?。！？]\s*/g;
-	let lastSentenceEnd = -1;
-	for (const match of searchText.matchAll(sentenceMatch)) {
-		const absolutePos = searchStart + match.index + match[0].length;
-		if (absolutePos <= targetIndex) lastSentenceEnd = absolutePos;
-	}
-	if (lastSentenceEnd !== -1) return lastSentenceEnd;
-	const wordBoundary = text.lastIndexOf(" ", targetIndex);
-	if (wordBoundary > searchStart) return wordBoundary + 1;
-	return targetIndex;
-}
-/**
-* Split text into overlapping chunks optimized for embedding.
-* Respects sentence boundaries when possible.
-*/
-function chunkText(text, config) {
-	const chunkSize = config?.size ?? DEFAULT_CHUNK_SIZE$1;
-	const rawOverlap = config?.overlap ?? DEFAULT_OVERLAP;
-	const safeOverlap = Math.max(0, rawOverlap);
-	const overlap = Math.min(safeOverlap, Math.max(0, chunkSize - 1));
-	const maxInputLength = config?.maxInputLength ?? DEFAULT_MAX_INPUT_LENGTH;
-	const tokenizer = createTokenizer(config?.tokenizer);
-	const normalizedText = (text.length > maxInputLength ? text.slice(0, maxInputLength) : text).replace(/\s+/g, " ").trim();
-	if (!normalizedText) return [];
-	const totalTokens = tokenizer(normalizedText);
-	if (totalTokens <= chunkSize) return [{
-		text: normalizedText,
-		startIndex: 0,
-		endIndex: normalizedText.length,
-		tokens: totalTokens
-	}];
-	const chunks = [];
-	const chunkSizeChars = tokensToChars(chunkSize);
-	const overlapChars = tokensToChars(overlap);
-	let startIndex = 0;
-	while (startIndex < normalizedText.length) {
-		const targetEnd = Math.min(startIndex + chunkSizeChars, normalizedText.length);
-		const endIndex = targetEnd < normalizedText.length ? findBreakPoint(normalizedText, targetEnd) : targetEnd;
-		const chunkText$1 = normalizedText.slice(startIndex, endIndex).trim();
-		if (chunkText$1) chunks.push({
-			text: chunkText$1,
-			startIndex,
-			endIndex,
-			tokens: tokenizer(chunkText$1)
-		});
-		if (endIndex >= normalizedText.length) break;
-		const nextStart = endIndex - overlapChars;
-		startIndex = Math.max(nextStart, startIndex + 1);
-		if (startIndex < normalizedText.length) {
-			const spaceIndex = normalizedText.indexOf(" ", startIndex);
-			if (spaceIndex !== -1 && spaceIndex < startIndex + overlapChars) startIndex = spaceIndex + 1;
-		}
-	}
-	return chunks;
-}
-/**
-* Estimate total tokens for a text without chunking.
-*/
-function estimateTokens(text, tokenizer) {
-	return createTokenizer(tokenizer)(text);
-}
-
-//#endregion
-//#region src/embeddings/input.ts
-/**
-* Select and prepare input text for embedding based on configuration.
-*
-* @param data - Scraped data to extract input from
-* @param config - Input configuration
-* @returns Selected and prepared text, or undefined if no valid input
-*/
-function selectInput(data, config) {
-	if (config?.transform) return normalizeText(config.transform(data));
-	if (config?.type === "custom" && config.customText) return normalizeText(config.customText);
-	const type = config?.type ?? "textContent";
-	switch (type) {
-		case "textContent": return selectTextContent(data);
-		case "title+summary": return selectTitleSummary(data);
-		case "custom": return selectTextContent(data);
-		default: {
-			const _exhaustive = type;
-			throw new Error(`Unknown input type: ${_exhaustive}`);
-		}
-	}
-}
-/**
-* Select textContent as input.
-*/
-function selectTextContent(data) {
-	if (data.textContent) return normalizeText(data.textContent);
-	if (data.content) return normalizeText(stripMarkdown(data.content));
-	if (data.excerpt) return normalizeText(data.excerpt);
-	if (data.description) return normalizeText(data.description);
-}
-/**
-* Select title + summary (or fallbacks) as input.
-* Optimized for semantic search and classification.
-*/
-function selectTitleSummary(data) {
-	const parts = [];
-	if (data.title) parts.push(data.title);
-	if (data.summary) parts.push(data.summary);
-	else if (data.excerpt) parts.push(data.excerpt);
-	else if (data.description) parts.push(data.description);
-	if (parts.length === 0) return;
-	return normalizeText(parts.join("\n\n"));
-}
-/**
-* Normalize text for embedding:
-* - Collapse whitespace
-* - Trim leading/trailing whitespace
-* - Remove control characters
-*/
-function normalizeText(text) {
-	if (!text) return "";
-	return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").split("\n").map((line) => line.trim()).join("\n").trim();
-}
-/**
-* Basic markdown stripping for when we need plain text from content.
-* Not comprehensive, but handles common cases.
-*/
-function stripMarkdown(markdown) {
-	return markdown.replace(/```[\s\S]*?```/g, "").replace(/`[^`]+`/g, "").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replace(/^#{1,6}\s+/gm, "").replace(/\*\*([^*]+)\*\*/g, "$1").replace(/\*([^*]+)\*/g, "$1").replace(/__([^_]+)__/g, "$1").replace(/_([^_]+)_/g, "$1").replace(/^>\s+/gm, "").replace(/^[-*_]{3,}$/gm, "").replace(/^[\s]*[-*+]\s+/gm, "").replace(/^[\s]*\d+\.\s+/gm, "");
-}
-/**
-* Check if the selected input meets minimum requirements.
-*/
-function validateInput(text, minLength = 10) {
-	if (!text) return {
-		valid: false,
-		reason: "No input text available"
-	};
-	if (text.length < minLength) return {
-		valid: false,
-		reason: `Input too short (${text.length} < ${minLength} characters)`
-	};
-	const wordCount = text.split(/\s+/).filter((w) => w.length > 1).length;
-	if (wordCount < 3) return {
-		valid: false,
-		reason: `Input has too few words (${wordCount} < 3)`
-	};
-	return {
-		valid: true,
-		text,
-		wordCount,
-		charCount: text.length
-	};
-}
-
-//#endregion
-//#region src/embeddings/providers/base.ts
-/**
-* Generate a stable cache key identifier for provider configuration.
-*/
-function getProviderCacheKey(config) {
-	switch (config.type) {
-		case "http": return `http:${config.config.baseUrl.replace(/\/$/, "")}:${config.config.model}`;
-		case "custom": return `custom:${config.provider.name}`;
-		default: {
-			const _exhaustive = config;
-			return String(_exhaustive);
-		}
-	}
-}
-
-//#endregion
-//#region src/embeddings/providers/http.ts
-/**
-* HTTP-based Embedding Provider using native fetch.
-* Provides a unified interface for any REST-based embedding API.
-*/
-/**
-* HTTP-based embedding provider.
-* Works with any REST API using native fetch.
-*/
-var HttpEmbeddingProvider = class extends BaseHttpProvider {
-	name = "http-embedding";
-	requestBuilder;
-	responseMapper;
-	constructor(config) {
-		super(config);
-		this.requestBuilder = config.requestBuilder ?? ((texts, model) => ({
-			input: texts,
-			model
-		}));
-		this.responseMapper = config.responseMapper ?? ((response) => {
-			const resp = response;
-			if (Array.isArray(resp.data)) return resp.data.map((item) => item.embedding);
-			if (Array.isArray(resp.embeddings)) return resp.embeddings;
-			if (Array.isArray(resp.embedding)) return [resp.embedding];
-			if (Array.isArray(response)) return response;
-			throw new ScrapeError("Unable to parse embedding response. Provide a custom responseMapper.", "VALIDATION_ERROR");
-		});
-	}
-	/**
-	* Generate embeddings for one or more texts.
-	*/
-	async embed(texts, options) {
-		const model = options.model || this.model;
-		const body = this.requestBuilder(texts, model);
-		const { data } = await this.fetch(this.baseUrl, {
-			body,
-			signal: options.signal
-		});
-		const embeddings = this.responseMapper(data);
-		if (embeddings.length !== texts.length) throw new ScrapeError(`Embedding count mismatch: expected ${texts.length}, got ${embeddings.length}`, "VALIDATION_ERROR");
-		return { embeddings };
-	}
-};
-/**
-* Create a generic HTTP embedding provider.
-*/
-function createHttpEmbedding(config) {
-	return new HttpEmbeddingProvider(config);
-}
-
-//#endregion
-//#region src/embeddings/providers/presets.ts
-/**
-* Create an OpenAI embedding provider.
-*
-* @example
-* ```ts
-* const provider = createOpenAIEmbedding({ apiKey: 'sk-...' });
-* const { embeddings } = await provider.embed(['Hello'], { model: 'text-embedding-3-small' });
-* ```
-*/
-function createOpenAIEmbedding(options) {
-	const apiKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
-	if (!apiKey) throw new Error("OpenAI API key required. Set OPENAI_API_KEY env var or pass apiKey option.");
-	const headers = { Authorization: `Bearer ${apiKey}` };
-	if (options?.organization) headers["OpenAI-Organization"] = options.organization;
-	return new HttpEmbeddingProvider({
-		baseUrl: options?.baseUrl ?? "https://api.openai.com/v1/embeddings",
-		model: options?.model ?? "text-embedding-3-small",
-		headers,
-		requestBuilder: (texts, model) => ({
-			input: texts,
-			model
-		}),
-		responseMapper: (res) => res.data.map((item) => item.embedding)
-	});
-}
-/**
-* Create an Azure OpenAI embedding provider.
-*
-* @example
-* ```ts
-* const provider = createAzureEmbedding({
-*   endpoint: 'https://my-resource.openai.azure.com',
-*   deploymentName: 'text-embedding-ada-002',
-*   apiVersion: '2023-05-15',
-* });
-* ```
-*/
-function createAzureEmbedding(options) {
-	const apiKey = options.apiKey ?? process.env.AZURE_OPENAI_API_KEY;
-	if (!apiKey) throw new Error("Azure OpenAI API key required. Set AZURE_OPENAI_API_KEY env var or pass apiKey option.");
-	return new HttpEmbeddingProvider({
-		baseUrl: `${options.endpoint.replace(/\/$/, "")}/openai/deployments/${options.deploymentName}/embeddings?api-version=${options.apiVersion}`,
-		model: options.deploymentName,
-		headers: { "api-key": apiKey },
-		requestBuilder: (texts) => ({ input: texts }),
-		responseMapper: (res) => res.data.map((item) => item.embedding)
-	});
-}
-/**
-* Create an Ollama embedding provider for local models.
-*
-* LIMITATION: Ollama's /api/embeddings endpoint processes one text at a time,
-* not batches. When multiple chunks are embedded, each chunk triggers a
-* separate HTTP request. This is handled transparently by the pipeline's
-* sequential chunk processing, but may be slower than batch-capable providers.
-* For high-throughput scenarios, consider using OpenAI, Cohere, or HuggingFace
-* which support batch embedding in a single request.
-*
-* @example
-* ```ts
-* const provider = createOllamaEmbedding({ model: 'nomic-embed-text' });
-* ```
-*/
-function createOllamaEmbedding(options) {
-	return new HttpEmbeddingProvider({
-		baseUrl: options?.baseUrl ?? "http://localhost:11434/api/embeddings",
-		model: options?.model ?? "nomic-embed-text",
-		requireHttps: false,
-		allowPrivate: true,
-		requestBuilder: (texts, model) => ({
-			model,
-			prompt: texts[0]
-		}),
-		responseMapper: (res) => [res.embedding]
-	});
-}
-/**
-* Create a HuggingFace Inference API embedding provider.
-*
-* @example
-* ```ts
-* const provider = createHuggingFaceEmbedding({
-*   model: 'sentence-transformers/all-MiniLM-L6-v2',
-* });
-* ```
-*/
-function createHuggingFaceEmbedding(options) {
-	const apiKey = options.apiKey ?? process.env.HF_TOKEN ?? process.env.HUGGINGFACE_API_KEY;
-	const headers = {};
-	if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
-	return new HttpEmbeddingProvider({
-		baseUrl: `https://api-inference.huggingface.co/models/${options.model}`,
-		model: options.model,
-		headers,
-		requestBuilder: (texts) => ({ inputs: texts }),
-		responseMapper: (response) => {
-			if (Array.isArray(response)) {
-				if (Array.isArray(response[0]) && typeof response[0][0] === "number") return response;
-				return [response];
-			}
-			throw new Error("Unexpected HuggingFace response format");
-		}
-	});
-}
-/**
-* Create a local Transformers.js embedding provider.
-* Uses dependency injection - user provides the imported transformers module.
-*
-* @example
-* ```typescript
-* import * as transformers from '@huggingface/transformers';
-* import { createTransformersEmbedding } from 'scrapex/embeddings';
-*
-* const provider = createTransformersEmbedding(transformers, {
-*   model: 'Xenova/all-MiniLM-L6-v2',
-* });
-* ```
-*
-* Required Node.js dependencies:
-* ```
-* npm install @huggingface/transformers onnxruntime-node
-* ```
-*/
-function createTransformersEmbedding(transformers, options) {
-	let pipeline = null;
-	let currentModel = null;
-	const config = {
-		model: options?.model ?? "Xenova/all-MiniLM-L6-v2",
-		quantized: options?.quantized ?? true,
-		pooling: options?.pooling ?? "mean",
-		normalize: options?.normalize ?? true
-	};
-	return {
-		name: "transformers",
-		async embed(texts, request) {
-			const model = request.model || config.model;
-			if (!pipeline || currentModel !== model) {
-				const cacheDir = options?.cacheDir;
-				const env = transformers.env;
-				const priorCacheDir = env?.cacheDir;
-				if (cacheDir && env) env.cacheDir = cacheDir;
-				try {
-					pipeline = await transformers.pipeline("feature-extraction", model, { quantized: config.quantized });
-				} finally {
-					if (cacheDir && env) if (priorCacheDir === void 0) delete env.cacheDir;
-					else env.cacheDir = priorCacheDir;
-				}
-				currentModel = model;
-			}
-			const embeddings = [];
-			for (const text of texts) {
-				const output = await pipeline(text, {
-					pooling: config.pooling,
-					normalize: config.normalize
-				});
-				embeddings.push(Array.from(output.data));
-			}
-			return { embeddings };
-		}
-	};
-}
-/** Recommended models for Transformers.js */
-const TRANSFORMERS_MODELS = {
-	DEFAULT: "Xenova/all-MiniLM-L6-v2",
-	QUALITY: "Xenova/all-mpnet-base-v2",
-	RETRIEVAL: "Xenova/bge-small-en-v1.5",
-	MULTILINGUAL: "Xenova/multilingual-e5-small"
-};
-
-//#endregion
-//#region src/embeddings/providers/index.ts
-/**
-* Create an embedding provider from configuration.
-* This is the main factory function for creating providers.
-*/
-function createEmbeddingProvider(config) {
-	switch (config.type) {
-		case "http": return createHttpEmbedding(config.config);
-		case "custom": return config.provider;
-		default: throw new ScrapeError(`Unknown embedding provider type: ${config.type}`, "VALIDATION_ERROR");
-	}
-}
-
-//#endregion
-//#region src/embeddings/safety.ts
-/**
-* PII redaction patterns with high precision to minimize false positives.
-* Patterns are designed to match common formats while avoiding over-matching.
-*/
-const EMAIL_PATTERN = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
-const PHONE_PATTERN = /(?:\+?1[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}\b/g;
-const CREDIT_CARD_PATTERN = /\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12}|(?:[0-9]{4}[-\s]){3}[0-9]{4}|[0-9]{13,19})\b/g;
-const SSN_PATTERN = /\b[0-9]{3}-[0-9]{2}-[0-9]{4}\b/g;
-const IPV4_PATTERN = /\b(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/g;
-const REDACTED = "[REDACTED]";
-/**
-* Create a redaction function based on configuration.
-* Returns a function that applies all configured PII patterns.
-*/
-function createPiiRedactor(config) {
-	const patterns = [];
-	if (config.creditCard) patterns.push({
-		name: "creditCard",
-		pattern: CREDIT_CARD_PATTERN
-	});
-	if (config.email) patterns.push({
-		name: "email",
-		pattern: EMAIL_PATTERN
-	});
-	if (config.phone) patterns.push({
-		name: "phone",
-		pattern: PHONE_PATTERN
-	});
-	if (config.ssn) patterns.push({
-		name: "ssn",
-		pattern: SSN_PATTERN
-	});
-	if (config.ipAddress) patterns.push({
-		name: "ipAddress",
-		pattern: IPV4_PATTERN
-	});
-	if (config.customPatterns) for (let i = 0; i < config.customPatterns.length; i++) {
-		const customPattern = config.customPatterns[i];
-		if (customPattern) patterns.push({
-			name: `custom_${i}`,
-			pattern: customPattern
-		});
-	}
-	return (text) => {
-		let redactedText = text;
-		let totalRedactions = 0;
-		const redactionsByType = {};
-		for (const { name, pattern } of patterns) {
-			pattern.lastIndex = 0;
-			const matchCount = text.match(pattern)?.length ?? 0;
-			if (matchCount > 0) {
-				redactedText = redactedText.replace(pattern, REDACTED);
-				totalRedactions += matchCount;
-				redactionsByType[name] = (redactionsByType[name] ?? 0) + matchCount;
-			}
-		}
-		return {
-			text: redactedText,
-			redacted: totalRedactions > 0,
-			redactionCount: totalRedactions,
-			redactionsByType
-		};
-	};
-}
-/**
-* Simple redaction that applies all default patterns.
-* Use createPiiRedactor() for fine-grained control.
-*/
-function redactPii(text) {
-	return createPiiRedactor({
-		email: true,
-		phone: true,
-		creditCard: true,
-		ssn: true,
-		ipAddress: true
-	})(text);
-}
-
-//#endregion
-//#region src/embeddings/pipeline.ts
-const DEFAULT_CHUNK_SIZE = 500;
-/**
-* Get the effective model for embedding.
-* Prioritizes: explicit options.model > provider config model
-*/
-function getEffectiveModel(providerConfig, explicitModel) {
-	if (explicitModel) return explicitModel;
-	if (providerConfig.type === "http") return providerConfig.config.model;
-}
-/**
-* Generate embeddings for scraped data.
-* This is the main entry point for the embedding pipeline.
-*/
-async function generateEmbeddings(data, options) {
-	const startTime = Date.now();
-	try {
-		const provider = createEmbeddingProvider(options.provider);
-		const model = getEffectiveModel(options.provider, options.model);
-		const validation = validateInput(selectInput(data, options.input), options.safety?.minTextLength ?? 10);
-		if (!validation.valid) return createSkippedResult(validation.reason, { model });
-		const originalInput = validation.text;
-		let inputText = validation.text;
-		let piiRedacted = false;
-		if (options.safety?.piiRedaction) {
-			const redactionResult = createPiiRedactor(options.safety.piiRedaction)(inputText);
-			inputText = redactionResult.text;
-			piiRedacted = redactionResult.redacted;
-		}
-		const effectiveChunking = applyMaxTokensToChunking(options.chunking, options.safety?.maxTokens);
-		const cacheKey = generateCacheKey({
-			providerKey: getProviderCacheKey(options.provider),
-			model,
-			dimensions: options.output?.dimensions,
-			aggregation: options.output?.aggregation,
-			input: options.input,
-			chunking: effectiveChunking,
-			safety: options.safety,
-			cacheKeySalt: options.cache?.cacheKeySalt,
-			content: inputText
-		});
-		const cache = options.cache?.store ?? getDefaultCache();
-		const cachedResult = await cache.get(cacheKey);
-		if (cachedResult && cachedResult.status === "success") {
-			if (options.onMetrics) options.onMetrics({
-				provider: provider.name,
-				model,
-				inputTokens: estimateTokens(inputText),
-				outputDimensions: getDimensions(cachedResult.aggregation === "all" ? cachedResult.vectors : cachedResult.vector),
-				chunks: cachedResult.source.chunks,
-				latencyMs: Date.now() - startTime,
-				cached: true,
-				retries: 0,
-				piiRedacted
-			});
-			return {
-				...cachedResult,
-				source: {
-					...cachedResult.source,
-					cached: true
-				}
-			};
-		}
-		const chunks = chunkText(inputText, effectiveChunking);
-		const callbackChunks = options.onChunk && options.safety?.allowSensitiveCallbacks ? chunkText(originalInput, effectiveChunking) : null;
-		if (chunks.length === 0) return createSkippedResult("No content after chunking", { model });
-		const sharedState = options.resilience?.state;
-		const rateLimiter = sharedState?.rateLimiter ?? (options.resilience?.rateLimit ? new RateLimiter(options.resilience.rateLimit) : null);
-		const circuitBreaker = sharedState?.circuitBreaker ?? (options.resilience?.circuitBreaker ? new CircuitBreaker(options.resilience.circuitBreaker) : null);
-		const concurrency = options.resilience?.concurrency ?? 1;
-		const semaphore = sharedState?.semaphore ?? new Semaphore(concurrency);
-		const embeddings = [];
-		let totalTokens = 0;
-		let retryCount = 0;
-		for (let i = 0; i < chunks.length; i++) {
-			const chunk = chunks[i];
-			if (!chunk) continue;
-			if (rateLimiter) await rateLimiter.acquire();
-			if (circuitBreaker?.isOpen()) return createSkippedResult("Circuit breaker is open", {
-				model,
-				chunks: i
-			});
-			await semaphore.execute(async () => {
-				const { result: result$1 } = await withResilience(async (signal) => {
-					return provider.embed([chunk.text], {
-						model,
-						dimensions: options.output?.dimensions,
-						signal
-					});
-				}, options.resilience, {
-					circuitBreaker: circuitBreaker ?? void 0,
-					rateLimiter: void 0,
-					semaphore: void 0
-				}, { onRetry: () => {
-					retryCount++;
-				} });
-				if (result$1.usage) totalTokens += result$1.usage.totalTokens;
-				else totalTokens += chunk.tokens;
-				const embedding = result$1.embeddings[0];
-				if (embedding) {
-					embeddings.push(embedding);
-					if (options.onChunk) {
-						const callbackText = callbackChunks?.[i]?.text ?? chunk.text;
-						options.onChunk(callbackText, embedding);
-					}
-				}
-			});
-		}
-		const aggregation = options.output?.aggregation ?? "average";
-		const aggregated = aggregateVectors(embeddings, aggregation);
-		const source = {
-			model,
-			chunks: chunks.length,
-			tokens: totalTokens || estimateTokens(inputText),
-			checksum: generateChecksum(inputText),
-			cached: false,
-			latencyMs: Date.now() - startTime
-		};
-		let result;
-		if (aggregated.type === "single") result = {
-			status: "success",
-			aggregation,
-			vector: aggregated.vector,
-			source
-		};
-		else result = {
-			status: "success",
-			aggregation: "all",
-			vectors: aggregated.vectors,
-			source
-		};
-		await cache.set(cacheKey, result, { ttlMs: options.cache?.ttlMs });
-		if (options.onMetrics) {
-			const metrics = {
-				provider: provider.name,
-				model,
-				inputTokens: source.tokens,
-				outputDimensions: aggregated.dimensions,
-				chunks: chunks.length,
-				latencyMs: source.latencyMs,
-				cached: false,
-				retries: retryCount,
-				piiRedacted
-			};
-			options.onMetrics(metrics);
-		}
-		return result;
-	} catch (error) {
-		const reason = error instanceof Error ? error.message : String(error);
-		if (error instanceof ScrapeError && ["INVALID_URL", "BLOCKED"].includes(error.code)) throw error;
-		return createSkippedResult(reason, { latencyMs: Date.now() - startTime });
-	}
-}
-function applyMaxTokensToChunking(chunking, maxTokens) {
-	if (!maxTokens || maxTokens <= 0) return chunking;
-	const baseSize = chunking?.size ?? DEFAULT_CHUNK_SIZE;
-	const baseOverlap = chunking?.overlap ?? 50;
-	const clampedSize = Math.min(baseSize, maxTokens);
-	const clampedOverlap = Math.min(baseOverlap, Math.max(0, clampedSize - 1));
-	return {
-		...chunking,
-		size: clampedSize,
-		overlap: clampedOverlap
-	};
-}
-/**
-* Embed arbitrary text directly.
-* Standalone function for embedding text outside of scrape().
-*/
-async function embed(text, options) {
-	return generateEmbeddings({ textContent: text }, {
-		...options,
-		input: {
-			...options.input,
-			type: "textContent"
-		}
-	});
-}
-/**
-* Embed from existing ScrapedData.
-* Useful when you've already scraped and want to add embeddings later.
-*/
-async function embedScrapedData(data, options) {
-	return generateEmbeddings(data, options);
-}
-/**
-* Create a skipped result with reason.
-*/
-function createSkippedResult(reason, partialSource) {
-	return {
-		status: "skipped",
-		reason,
-		source: partialSource ?? {}
-	};
-}
-
 //#endregion
 //#region src/extractors/content.ts
 const turndown = new TurndownService({