scrapex 1.0.0-alpha.1 → 1.0.0-beta.1

package/dist/enhancer-j0xqKDJm.cjs

@@ -0,0 +1,847 @@
+const require_parsers = require('./parsers-Bneuws8x.cjs');
+let node_dns = require("node:dns");
+let node_net = require("node:net");
+let zod = require("zod");
+
+//#region src/core/errors.ts
+/**
+* Custom error class for scraping failures with structured error codes
+*/
+var ScrapeError = class ScrapeError extends Error {
+	code;
+	statusCode;
+	constructor(message, code, statusCode, cause) {
+		super(message, { cause });
+		this.name = "ScrapeError";
+		this.code = code;
+		this.statusCode = statusCode;
+		if (Error.captureStackTrace) Error.captureStackTrace(this, ScrapeError);
+	}
+	/**
+	* Create a ScrapeError from an unknown error
+	*/
+	static from(error, code = "FETCH_FAILED") {
+		if (error instanceof ScrapeError) return error;
+		if (error instanceof Error) return new ScrapeError(error.message, code, void 0, error);
+		return new ScrapeError(String(error), code);
+	}
+	/**
+	* Check if error is retryable (network issues, timeouts)
+	*/
+	isRetryable() {
+		return this.code === "FETCH_FAILED" || this.code === "TIMEOUT";
+	}
+	/**
+	* Convert to a plain object for serialization
+	*/
+	toJSON() {
+		return {
+			name: this.name,
+			message: this.message,
+			code: this.code,
+			statusCode: this.statusCode,
+			stack: this.stack
+		};
+	}
+};
+
+//#endregion
+//#region src/common/errors.ts
+/**
+* Error normalization utilities for HTTP providers.
+* Maps HTTP status codes to consistent ScrapeError codes.
+*/
+/**
+* HTTP status code to ScrapeError code mapping.
+*/
+function getErrorCodeFromStatus(status) {
+	if (status === 401 || status === 403) return "BLOCKED";
+	if (status === 404) return "NOT_FOUND";
+	if (status === 429) return "BLOCKED";
+	if (status === 408) return "TIMEOUT";
+	if (status >= 500) return "LLM_ERROR";
+	return "FETCH_FAILED";
+}
+/**
+* Parse error message from API response body.
+*/
+async function parseErrorBody(response) {
+	try {
+		const text = await response.text();
+		try {
+			const json = JSON.parse(text);
+			if (typeof json.error === "object" && json.error !== null) {
+				const error = json.error;
+				return String(error.message ?? error.msg ?? JSON.stringify(error));
+			}
+			if (typeof json.error === "string") return json.error;
+			if (typeof json.message === "string") return json.message;
+			if (typeof json.detail === "string") return json.detail;
+			return text;
+		} catch {
+			return text || `HTTP ${response.status} ${response.statusText}`;
+		}
+	} catch {
+		return `HTTP ${response.status} ${response.statusText}`;
+	}
+}
+/**
+* Create a ScrapeError from an HTTP response.
+*/
+async function createHttpError(response, providerName, errorMapper) {
+	const code = getErrorCodeFromStatus(response.status);
+	let message;
+	if (errorMapper) try {
+		message = errorMapper(await response.json());
+	} catch {
+		message = await parseErrorBody(response);
+	}
+	else message = await parseErrorBody(response);
+	return new ScrapeError(`${providerName} API error (${response.status}): ${message}`, code, response.status);
+}
+
+//#endregion
+//#region src/common/resilience.ts
+/**
+* Default retry configuration.
+*/
+const DEFAULT_RETRY = {
+	maxAttempts: 3,
+	backoffMs: 1e3,
+	backoffMultiplier: 2,
+	retryableStatuses: [
+		408,
+		429,
+		500,
+		502,
+		503,
+		504
+	]
+};
+/**
+* Errors that should be retried (transient failures).
+*/
+const RETRYABLE_ERROR_CODES = [
+	"ECONNRESET",
+	"ETIMEDOUT",
+	"ECONNREFUSED",
+	"EPIPE",
+	"ENOTFOUND",
+	"ENETUNREACH",
+	"EAI_AGAIN"
+];
+/**
+* Check if an error is retryable.
+*/
+function isRetryableError(error, retryableStatuses = DEFAULT_RETRY.retryableStatuses) {
+	if (error instanceof Error) {
+		const code = error.code;
+		if (code && RETRYABLE_ERROR_CODES.includes(code)) return true;
+		if ("statusCode" in error && typeof error.statusCode === "number") return retryableStatuses.includes(error.statusCode);
+		if ("status" in error && typeof error.status === "number") return retryableStatuses.includes(error.status);
+		if ("code" in error) {
+			const errCode = error.code;
+			if (errCode === "TIMEOUT" || errCode === "FETCH_FAILED") return true;
+		}
+		const message = error.message.toLowerCase();
+		if (message.includes("timeout") || message.includes("rate limit") || message.includes("too many requests") || message.includes("temporarily unavailable")) return true;
+	}
+	return false;
+}
+/**
+* Sleep for specified milliseconds.
+*/
+function sleep(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+* Execute a function with retry logic.
+*/
+async function withRetry(fn, config, onRetry) {
+	const maxAttempts = config?.maxAttempts ?? DEFAULT_RETRY.maxAttempts;
+	const backoffMs = config?.backoffMs ?? DEFAULT_RETRY.backoffMs;
+	const multiplier = config?.backoffMultiplier ?? DEFAULT_RETRY.backoffMultiplier;
+	const retryableStatuses = config?.retryableStatuses ?? DEFAULT_RETRY.retryableStatuses;
+	let lastError;
+	for (let attempt = 1; attempt <= maxAttempts; attempt++) try {
+		return {
+			result: await fn(),
+			attempts: attempt
+		};
+	} catch (error) {
+		lastError = error instanceof Error ? error : new Error(String(error));
+		if (attempt === maxAttempts || !isRetryableError(error, retryableStatuses)) throw lastError;
+		const jitter = backoffMs * multiplier ** (attempt - 1) * (.9 + Math.random() * .2);
+		onRetry?.(attempt, lastError, jitter);
+		await sleep(jitter);
+	}
+	throw lastError ?? /* @__PURE__ */ new Error("Retry failed");
+}
+/**
+* Execute a function with timeout.
+*/
+async function withTimeout(fn, timeoutMs) {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+	try {
+		return await fn(controller.signal);
+	} finally {
+		clearTimeout(timeoutId);
+	}
+}
+/**
+* Default circuit breaker configuration.
+*/
+const DEFAULT_CIRCUIT_BREAKER = {
+	failureThreshold: 5,
+	resetTimeoutMs: 3e4
+};
+/**
+* Error thrown when circuit breaker is open.
+*/
+var CircuitOpenError = class extends Error {
+	isCircuitOpen = true;
+	constructor(message) {
+		super(message);
+		this.name = "CircuitOpenError";
+	}
+};
+/**
+* Circuit breaker implementation.
+* Prevents cascade failures by stopping requests when failure rate is high.
+*/
+var CircuitBreaker = class {
+	state;
+	failureThreshold;
+	resetTimeoutMs;
+	constructor(config) {
+		this.failureThreshold = config?.failureThreshold ?? DEFAULT_CIRCUIT_BREAKER.failureThreshold;
+		this.resetTimeoutMs = config?.resetTimeoutMs ?? DEFAULT_CIRCUIT_BREAKER.resetTimeoutMs;
+		this.state = {
+			state: "closed",
+			failures: 0
+		};
+	}
+	/**
+	* Check if requests are blocked.
+	*/
+	isOpen() {
+		this.updateState();
+		return this.state.state === "open";
+	}
+	/**
+	* Get current circuit state.
+	*/
+	getState() {
+		this.updateState();
+		return this.state.state;
+	}
+	/**
+	* Record a successful request.
+	*/
+	recordSuccess() {
+		this.state.failures = 0;
+		this.state.state = "closed";
+		this.state.lastFailureTime = void 0;
+		this.state.nextAttemptTime = void 0;
+	}
+	/**
+	* Record a failed request.
+	*/
+	recordFailure() {
+		this.state.failures++;
+		this.state.lastFailureTime = Date.now();
+		if (this.state.failures >= this.failureThreshold) {
+			this.state.state = "open";
+			this.state.nextAttemptTime = Date.now() + this.resetTimeoutMs;
+		}
+	}
+	/**
+	* Execute a function with circuit breaker protection.
+	*/
+	async execute(fn) {
+		if (this.isOpen()) throw new CircuitOpenError(`Circuit breaker is open. Next attempt at ${this.state.nextAttemptTime ? new Date(this.state.nextAttemptTime).toISOString() : "unknown"}`);
+		try {
+			const result = await fn();
+			this.recordSuccess();
+			return result;
+		} catch (error) {
+			this.recordFailure();
+			throw error;
+		}
+	}
+	/**
+	* Reset the circuit breaker.
+	*/
+	reset() {
+		this.state = {
+			state: "closed",
+			failures: 0
+		};
+	}
+	/**
+	* Update state based on time (open -> half-open transition).
+	*/
+	updateState() {
+		if (this.state.state === "open" && this.state.nextAttemptTime && Date.now() >= this.state.nextAttemptTime) this.state.state = "half-open";
+	}
+};
+/**
+* Token bucket rate limiter.
+*/
+var RateLimiter = class {
+	tokens;
+	lastRefill;
+	maxTokens;
+	refillRate;
+	constructor(config) {
+		const requestsPerSecond = (config.requestsPerMinute ?? 60) / 60;
+		this.maxTokens = Math.max(1, Math.ceil(requestsPerSecond * 10));
+		this.refillRate = requestsPerSecond;
+		this.tokens = this.maxTokens;
+		this.lastRefill = Date.now();
+	}
+	/**
+	* Check if a request is allowed without consuming tokens.
+	*/
+	canProceed() {
+		this.refill();
+		return this.tokens >= 1;
+	}
+	/**
+	* Attempt to acquire tokens for a request.
+	* Returns true if allowed, false if rate limited.
+	*/
+	tryAcquire(tokens = 1) {
+		this.refill();
+		if (this.tokens >= tokens) {
+			this.tokens -= tokens;
+			return true;
+		}
+		return false;
+	}
+	/**
+	* Wait until tokens are available, then acquire.
+	*/
+	async acquire(tokens = 1) {
+		if (this.tryAcquire(tokens)) return;
+		this.refill();
+		const tokensNeeded = tokens - this.tokens;
+		const waitMs = Math.ceil(tokensNeeded / this.refillRate * 1e3);
+		if (waitMs > 0) await sleep(waitMs);
+		while (!this.tryAcquire(tokens)) await sleep(Math.ceil(1 / this.refillRate * 1e3));
+	}
+	/**
+	* Get time until next token is available (in milliseconds).
+	*/
+	getWaitTime() {
+		this.refill();
+		if (this.tokens >= 1) return 0;
+		return Math.ceil(1 / this.refillRate * 1e3);
+	}
+	/**
+	* Refill tokens based on elapsed time.
+	*/
+	refill() {
+		const now = Date.now();
+		const newTokens = (now - this.lastRefill) / 1e3 * this.refillRate;
+		this.tokens = Math.min(this.maxTokens, this.tokens + newTokens);
+		this.lastRefill = now;
+	}
+};
+/**
+* Semaphore for limiting concurrent operations.
+*/
+var Semaphore = class {
+	permits;
+	waiting = [];
+	constructor(permits) {
+		this.permits = permits;
+	}
+	/**
+	* Acquire a permit, waiting if necessary.
+	*/
+	async acquire() {
+		if (this.permits > 0) {
+			this.permits--;
+			return;
+		}
+		return new Promise((resolve) => {
+			this.waiting.push(resolve);
+		});
+	}
+	/**
+	* Release a permit.
+	*/
+	release() {
+		const next = this.waiting.shift();
+		if (next) next();
+		else this.permits++;
+	}
+	/**
+	* Execute function with semaphore protection.
+	*/
+	async execute(fn) {
+		await this.acquire();
+		try {
+			return await fn();
+		} finally {
+			this.release();
+		}
+	}
+};
+/**
+* Execute a function with all resilience features.
+*
+* @param fn - The async function to execute with resilience
+* @param config - Configuration for retry and timeout behavior
+* @param state - Pre-instantiated resilience primitives for stateful features.
+*   Circuit breaker, rate limiter, and semaphore must be instantiated by the caller
+*   and passed via state to enable those features. This allows sharing state across
+*   multiple calls for proper circuit breaker tracking and rate limiting.
+*   The config parameter is only used for retry and timeout settings.
+* @param callbacks - Optional callbacks for retry events
+*/
+async function withResilience(fn, config, state, callbacks) {
+	const timeoutMs = config?.timeoutMs ?? 3e4;
+	if (state?.circuitBreaker?.isOpen()) throw new CircuitOpenError("Circuit breaker is open");
+	if (state?.rateLimiter) await state.rateLimiter.acquire();
+	const executeWithConcurrency = async () => {
+		const withTimeoutFn = () => withTimeout(fn, timeoutMs);
+		try {
+			const retryResult = await withRetry(withTimeoutFn, config?.retry, callbacks?.onRetry);
+			state?.circuitBreaker?.recordSuccess();
+			return retryResult;
+		} catch (error) {
+			state?.circuitBreaker?.recordFailure();
+			throw error;
+		}
+	};
+	if (state?.semaphore) return state.semaphore.execute(executeWithConcurrency);
+	return executeWithConcurrency();
+}
+
+//#endregion
+//#region src/common/http-base.ts
+/**
+* Shared HTTP provider infrastructure for LLM and Embedding providers.
+* Provides SSRF protection, resilience, and error normalization.
+*/
+/**
+* Private IP ranges blocked for SSRF protection.
+*/
+const PRIVATE_IP_PATTERNS = [
+	/^10\./,
+	/^172\.(1[6-9]|2\d|3[01])\./,
+	/^192\.168\./,
+	/^127\./,
+	/^0\./,
+	/^169\.254\./,
+	/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./,
+	/^::1$/,
+	/^(fc|fd)[0-9a-f]{2}:/i,
+	/^fe80:/i,
+	/^fec0:/i,
+	/^::ffff:(10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.|127\.|0\.)/i,
+	/^localhost$/i
+];
+/**
+* Check if a hostname/IP is private.
+*/
+function isPrivateHost(hostname) {
+	return PRIVATE_IP_PATTERNS.some((pattern) => pattern.test(hostname));
+}
+/**
+* Validate a URL for security.
+*/
+function validateUrl(url, options = {}) {
+	const requireHttps = options.requireHttps ?? true;
+	const allowPrivate = options.allowPrivate ?? false;
+	let parsed;
+	try {
+		parsed = new URL(url);
+	} catch {
+		throw new ScrapeError(`Invalid URL: ${url}`, "INVALID_URL");
+	}
+	if (requireHttps && parsed.protocol !== "https:") throw new ScrapeError(`HTTPS required. Got: ${parsed.protocol}`, "VALIDATION_ERROR");
+	if (!allowPrivate && isPrivateHost(parsed.hostname)) throw new ScrapeError(`Private/internal addresses not allowed: ${parsed.hostname}`, "VALIDATION_ERROR");
+	return parsed;
+}
+/**
+* Validate URL and resolve DNS to check for private IPs.
+*/
+async function validateUrlWithDns(url, options = {}) {
+	const parsed = validateUrl(url, options);
+	const resolveDns = options.resolveDns ?? true;
+	const allowPrivate = options.allowPrivate ?? false;
+	if (!resolveDns || allowPrivate) return;
+	const host = parsed.hostname;
+	if ((0, node_net.isIP)(host)) return;
+	try {
+		const addresses = await node_dns.promises.lookup(host, { all: true });
+		for (const addr of addresses) if (isPrivateHost(addr.address)) throw new ScrapeError(`DNS resolved to private address: ${host} -> ${addr.address}`, "VALIDATION_ERROR");
+	} catch (error) {
+		if (error instanceof ScrapeError) throw error;
+		throw new ScrapeError(`Failed to resolve hostname: ${host} (${error instanceof Error ? error.message : String(error)})`, "FETCH_FAILED");
+	}
+}
+/**
+* Base HTTP provider with shared security and resilience.
+*/
+var BaseHttpProvider = class {
+	baseUrl;
+	model;
+	headers;
+	errorMapper;
+	requireHttps;
+	allowPrivate;
+	resolveDns;
+	allowRedirects;
+	timeoutMs;
+	retryConfig;
+	concurrency;
+	circuitBreaker;
+	rateLimiter;
+	semaphore;
+	constructor(config) {
+		this.baseUrl = config.baseUrl.replace(/\/$/, "");
+		this.model = config.model;
+		this.headers = {
+			"Content-Type": "application/json",
+			...config.headers
+		};
+		this.errorMapper = config.errorMapper;
+		this.requireHttps = config.requireHttps ?? true;
+		this.allowPrivate = config.allowPrivate ?? false;
+		this.resolveDns = config.resolveDns ?? true;
+		this.allowRedirects = config.allowRedirects ?? false;
+		this.timeoutMs = config.resilience?.timeoutMs ?? 3e4;
+		this.retryConfig = config.resilience?.retry;
+		this.concurrency = config.resilience?.concurrency ?? 1;
+		const sharedState = config.resilience?.state;
+		this.circuitBreaker = sharedState?.circuitBreaker ?? (config.resilience?.circuitBreaker ? new CircuitBreaker(config.resilience.circuitBreaker) : void 0);
+		this.rateLimiter = sharedState?.rateLimiter ?? (config.resilience?.rateLimit ? new RateLimiter(config.resilience.rateLimit) : void 0);
+		this.semaphore = sharedState?.semaphore ?? new Semaphore(this.concurrency);
+		validateUrl(this.baseUrl, {
+			requireHttps: this.requireHttps,
+			allowPrivate: this.allowPrivate
+		});
+	}
+	/**
+	* Get the current resilience state for persistence across calls.
+	*/
+	getResilienceState() {
+		return {
+			circuitBreaker: this.circuitBreaker,
+			rateLimiter: this.rateLimiter,
+			semaphore: this.semaphore
+		};
+	}
+	/**
+	* Make an HTTP request with security and resilience.
+	*/
+	async fetch(url, options = {}) {
+		const securityOptions = {
+			requireHttps: this.requireHttps,
+			allowPrivate: this.allowPrivate,
+			resolveDns: this.resolveDns,
+			allowRedirects: this.allowRedirects
+		};
+		await validateUrlWithDns(url, securityOptions);
+		if (this.circuitBreaker?.isOpen()) throw new CircuitOpenError("Circuit breaker is open. Too many recent failures.");
+		if (this.rateLimiter) await this.rateLimiter.acquire();
+		const doFetch = async (signal) => {
+			const composedSignal = options.signal ? AbortSignal.any([options.signal, signal]) : signal;
+			const response = await fetch(url, {
+				method: options.method ?? "POST",
+				headers: {
+					...this.headers,
+					...options.headers
+				},
+				body: options.body ? JSON.stringify(options.body) : void 0,
+				signal: composedSignal,
+				redirect: this.allowRedirects ? "follow" : "error"
+			});
+			if (this.allowRedirects && response.redirected) await validateUrlWithDns(response.url, securityOptions);
+			if (!response.ok) throw await createHttpError(response, this.constructor.name, this.errorMapper);
+			return {
+				data: await response.json(),
+				status: response.status,
+				headers: response.headers
+			};
+		};
+		const executeWithConcurrency = async () => {
+			if (!this.semaphore) throw new ScrapeError("Semaphore not initialized", "VALIDATION_ERROR");
+			return this.semaphore.execute(async () => {
+				const fetchWithTimeout = async () => {
+					return withTimeout((signal) => doFetch(signal), this.timeoutMs);
+				};
+				try {
+					let result;
+					if (this.retryConfig) result = (await withRetry(fetchWithTimeout, this.retryConfig)).result;
+					else result = await fetchWithTimeout();
+					this.circuitBreaker?.recordSuccess();
+					return result;
+				} catch (error) {
+					this.circuitBreaker?.recordFailure();
+					throw error;
+				}
+			});
+		};
+		return executeWithConcurrency();
+	}
+};
+
+//#endregion
+//#region src/llm/types.ts
+/**
+* Zod schemas for LLM outputs
+*/
+const SummarySchema = zod.z.object({ summary: zod.z.string().describe("A concise 2-3 sentence summary of the content") });
+const TagsSchema = zod.z.object({ tags: zod.z.array(zod.z.string()).describe("5-10 relevant tags/keywords") });
+const EntitiesSchema = zod.z.object({
+	people: zod.z.array(zod.z.string()).describe("People mentioned"),
+	organizations: zod.z.array(zod.z.string()).describe("Organizations/companies"),
+	technologies: zod.z.array(zod.z.string()).describe("Technologies/tools/frameworks"),
+	locations: zod.z.array(zod.z.string()).describe("Locations/places"),
+	concepts: zod.z.array(zod.z.string()).describe("Key concepts/topics")
+});
+const ClassifySchema = zod.z.object({
+	contentType: zod.z.enum([
+		"article",
+		"repo",
+		"docs",
+		"package",
+		"video",
+		"tool",
+		"product",
+		"unknown"
+	]).describe("The type of content"),
+	confidence: zod.z.number().min(0).max(1).describe("Confidence score 0-1")
+});
+
+//#endregion
+//#region src/llm/enhancer.ts
+/**
+* Enhance scraped data with LLM-powered features
+*/
+async function enhance(data, provider, types) {
+	const results = {};
+	const content = data.excerpt || data.textContent.slice(0, 1e4);
+	const context = `Title: ${data.title}\nURL: ${data.url}\n\nContent:\n${content}`;
+	const promises = [];
+	if (types.includes("summarize")) promises.push(summarize(context, provider).then((summary) => {
+		results.summary = summary;
+	}));
+	if (types.includes("tags")) promises.push(extractTags(context, provider).then((tags) => {
+		results.suggestedTags = tags;
+	}));
+	if (types.includes("entities")) promises.push(extractEntities(context, provider).then((entities) => {
+		results.entities = entities;
+	}));
+	if (types.includes("classify")) promises.push(classify(context, provider).then((classification) => {
+		if (classification.confidence > .7) results.contentType = classification.contentType;
+	}));
+	await Promise.all(promises);
+	return results;
+}
+/**
+* Ask a custom question about the scraped content
+* Results are stored in the `custom` field of ScrapedData
+*/
+async function ask(data, provider, prompt, options) {
+	const key = options?.key || "response";
+	const content = data.excerpt || data.textContent.slice(0, 1e4);
+	const processedPrompt = applyPlaceholders(prompt, data, content);
+	if (options?.schema) {
+		const result = await extract(data, provider, options.schema, processedPrompt);
+		return { custom: { [key]: result } };
+	}
+	const fullPrompt = prompt.includes("{{content}}") ? processedPrompt : `${processedPrompt}\n\nTitle: ${data.title}\nURL: ${data.url}\n\nContent:\n${content}`;
+	const response = await provider.complete(fullPrompt);
+	return { custom: { [key]: response } };
+}
+/**
+* Apply placeholder replacements to a prompt template
+*/
+function applyPlaceholders(prompt, data, content) {
+	const domain = (() => {
+		try {
+			return new URL(data.url).hostname;
+		} catch {
+			return "";
+		}
+	})();
+	return prompt.replace(/\{\{title\}\}/g, data.title).replace(/\{\{url\}\}/g, data.url).replace(/\{\{content\}\}/g, content).replace(/\{\{description\}\}/g, data.description || "").replace(/\{\{excerpt\}\}/g, data.excerpt || "").replace(/\{\{domain\}\}/g, domain);
+}
+/**
+* Extract structured data using LLM and a custom schema
+*/
+async function extract(data, provider, schema, promptTemplate) {
+	const zodShape = {};
+	for (const [key, type] of Object.entries(schema)) {
+		const isOptional = type.endsWith("?");
+		const baseType = isOptional ? type.slice(0, -1) : type;
+		let zodType;
+		switch (baseType) {
+			case "string":
+				zodType = zod.z.string();
+				break;
+			case "number":
+				zodType = zod.z.number();
+				break;
+			case "boolean":
+				zodType = zod.z.boolean();
+				break;
+			case "string[]":
+				zodType = zod.z.array(zod.z.string());
+				break;
+			case "number[]":
+				zodType = zod.z.array(zod.z.number());
+				break;
+			default: zodType = zod.z.string();
+		}
+		zodShape[key] = isOptional ? zodType.optional() : zodType;
+	}
+	const zodSchema = zod.z.object(zodShape);
+	const content = data.textContent.slice(0, 4e3);
+	let prompt;
+	if (promptTemplate) {
+		prompt = applyPlaceholders(promptTemplate, data, content);
+		if (!promptTemplate.includes("{{content}}")) prompt += `\n\nContext:\n${content}`;
+	} else prompt = `Extract the following information from this content:
+
+Title: ${data.title}
+URL: ${data.url}
+
+Content:
+${content}
+
+Extract these fields:
+${Object.entries(schema).map(([key, type]) => `- ${key} (${type})`).join("\n")}`;
+	return provider.completeJSON(prompt, zodSchema);
+}
+/**
+* Generate a summary of the content
+*/
+async function summarize(context, provider) {
+	const prompt = `Summarize the following content in 2-3 concise sentences:
+
+${context}`;
+	return (await provider.completeJSON(prompt, SummarySchema)).summary;
+}
+/**
+* Extract relevant tags/keywords
+*/
+async function extractTags(context, provider) {
+	const prompt = `Extract 5-10 relevant tags or keywords from the following content. Focus on technologies, concepts, and topics mentioned:
+
+${context}`;
+	return (await provider.completeJSON(prompt, TagsSchema)).tags;
+}
+/**
+* Extract named entities from content
+*/
+async function extractEntities(context, provider) {
+	const prompt = `Extract named entities from the following content. Identify people, organizations, technologies, locations, and key concepts:
+
+${context}`;
+	return provider.completeJSON(prompt, EntitiesSchema);
+}
+/**
+* Classify content type using LLM
+*/
+async function classify(context, provider) {
+	const prompt = `Classify the following content into one of these categories:
+- article: Blog post, news article, essay
+- repo: Code repository, open source project
+- docs: Documentation, API reference, guides
+- package: npm/pip package page
+- video: Video content, YouTube
+- tool: Software tool, web application
+- product: Commercial product, e-commerce
+
+${context}`;
+	return provider.completeJSON(prompt, ClassifySchema);
+}
+
+//#endregion
+Object.defineProperty(exports, 'BaseHttpProvider', {
+  enumerable: true,
+  get: function () {
+    return BaseHttpProvider;
+  }
+});
+Object.defineProperty(exports, 'CircuitBreaker', {
+  enumerable: true,
+  get: function () {
+    return CircuitBreaker;
+  }
+});
+Object.defineProperty(exports, 'ClassifySchema', {
+  enumerable: true,
+  get: function () {
+    return ClassifySchema;
+  }
+});
+Object.defineProperty(exports, 'EntitiesSchema', {
+  enumerable: true,
+  get: function () {
+    return EntitiesSchema;
+  }
+});
+Object.defineProperty(exports, 'RateLimiter', {
+  enumerable: true,
+  get: function () {
+    return RateLimiter;
+  }
+});
+Object.defineProperty(exports, 'ScrapeError', {
+  enumerable: true,
+  get: function () {
+    return ScrapeError;
+  }
+});
+Object.defineProperty(exports, 'Semaphore', {
+  enumerable: true,
+  get: function () {
+    return Semaphore;
+  }
+});
+Object.defineProperty(exports, 'SummarySchema', {
+  enumerable: true,
+  get: function () {
+    return SummarySchema;
+  }
+});
+Object.defineProperty(exports, 'TagsSchema', {
+  enumerable: true,
+  get: function () {
+    return TagsSchema;
+  }
+});
+Object.defineProperty(exports, 'ask', {
+  enumerable: true,
+  get: function () {
+    return ask;
+  }
+});
+Object.defineProperty(exports, 'enhance', {
+  enumerable: true,
+  get: function () {
+    return enhance;
+  }
+});
+Object.defineProperty(exports, 'extract', {
+  enumerable: true,
+  get: function () {
+    return extract;
+  }
+});
+Object.defineProperty(exports, 'withResilience', {
+  enumerable: true,
+  get: function () {
+    return withResilience;
+  }
+});
+//# sourceMappingURL=enhancer-j0xqKDJm.cjs.map