scrapex 1.0.0-alpha.1 → 1.0.0-beta.2

package/dist/http-base-DM7YNo6X.mjs

@@ -0,0 +1,618 @@
+import { promises } from "node:dns";
+import { isIP } from "node:net";
+
+//#region src/core/errors.ts
+/**
+* Custom error class for scraping failures with structured error codes
+*/
+var ScrapeError = class ScrapeError extends Error {
+	code;
+	statusCode;
+	constructor(message, code, statusCode, cause) {
+		super(message, { cause });
+		this.name = "ScrapeError";
+		this.code = code;
+		this.statusCode = statusCode;
+		if (Error.captureStackTrace) Error.captureStackTrace(this, ScrapeError);
+	}
+	/**
+	* Create a ScrapeError from an unknown error
+	*/
+	static from(error, code = "FETCH_FAILED") {
+		if (error instanceof ScrapeError) return error;
+		if (error instanceof Error) return new ScrapeError(error.message, code, void 0, error);
+		return new ScrapeError(String(error), code);
+	}
+	/**
+	* Check if error is retryable (network issues, timeouts)
+	*/
+	isRetryable() {
+		return this.code === "FETCH_FAILED" || this.code === "TIMEOUT";
+	}
+	/**
+	* Convert to a plain object for serialization
+	*/
+	toJSON() {
+		return {
+			name: this.name,
+			message: this.message,
+			code: this.code,
+			statusCode: this.statusCode,
+			stack: this.stack
+		};
+	}
+};
+
+//#endregion
+//#region src/common/errors.ts
+/**
+* Error normalization utilities for HTTP providers.
+* Maps HTTP status codes to consistent ScrapeError codes.
+*/
+/**
+* HTTP status code to ScrapeError code mapping.
+*/
+function getErrorCodeFromStatus(status) {
+	if (status === 401 || status === 403) return "BLOCKED";
+	if (status === 404) return "NOT_FOUND";
+	if (status === 429) return "BLOCKED";
+	if (status === 408) return "TIMEOUT";
+	if (status >= 500) return "LLM_ERROR";
+	return "FETCH_FAILED";
+}
+/**
+* Parse error message from API response body.
+*/
+async function parseErrorBody(response) {
+	try {
+		const text = await response.text();
+		try {
+			const json = JSON.parse(text);
+			if (typeof json.error === "object" && json.error !== null) {
+				const error = json.error;
+				return String(error.message ?? error.msg ?? JSON.stringify(error));
+			}
+			if (typeof json.error === "string") return json.error;
+			if (typeof json.message === "string") return json.message;
+			if (typeof json.detail === "string") return json.detail;
+			return text;
+		} catch {
+			return text || `HTTP ${response.status} ${response.statusText}`;
+		}
+	} catch {
+		return `HTTP ${response.status} ${response.statusText}`;
+	}
+}
+/**
+* Create a ScrapeError from an HTTP response.
+*/
+async function createHttpError(response, providerName, errorMapper) {
+	const code = getErrorCodeFromStatus(response.status);
+	let message;
+	if (errorMapper) try {
+		message = errorMapper(await response.json());
+	} catch {
+		message = await parseErrorBody(response);
+	}
+	else message = await parseErrorBody(response);
+	return new ScrapeError(`${providerName} API error (${response.status}): ${message}`, code, response.status);
+}
+
+//#endregion
+//#region src/common/resilience.ts
+/**
+* Default retry configuration.
+*/
+const DEFAULT_RETRY = {
+	maxAttempts: 3,
+	backoffMs: 1e3,
+	backoffMultiplier: 2,
+	retryableStatuses: [
+		408,
+		429,
+		500,
+		502,
+		503,
+		504
+	]
+};
+/**
+* Errors that should be retried (transient failures).
+*/
+const RETRYABLE_ERROR_CODES = [
+	"ECONNRESET",
+	"ETIMEDOUT",
+	"ECONNREFUSED",
+	"EPIPE",
+	"ENOTFOUND",
+	"ENETUNREACH",
+	"EAI_AGAIN"
+];
+/**
+* Check if an error is retryable.
+*/
+function isRetryableError(error, retryableStatuses = DEFAULT_RETRY.retryableStatuses) {
+	if (error instanceof Error) {
+		const code = error.code;
+		if (code && RETRYABLE_ERROR_CODES.includes(code)) return true;
+		if ("statusCode" in error && typeof error.statusCode === "number") return retryableStatuses.includes(error.statusCode);
+		if ("status" in error && typeof error.status === "number") return retryableStatuses.includes(error.status);
+		if ("code" in error) {
+			const errCode = error.code;
+			if (errCode === "TIMEOUT" || errCode === "FETCH_FAILED") return true;
+		}
+		const message = error.message.toLowerCase();
+		if (message.includes("timeout") || message.includes("rate limit") || message.includes("too many requests") || message.includes("temporarily unavailable")) return true;
+	}
+	return false;
+}
+/**
+* Sleep for specified milliseconds.
+*/
+function sleep(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+* Execute a function with retry logic.
+*/
+async function withRetry(fn, config, onRetry) {
+	const maxAttempts = config?.maxAttempts ?? DEFAULT_RETRY.maxAttempts;
+	const backoffMs = config?.backoffMs ?? DEFAULT_RETRY.backoffMs;
+	const multiplier = config?.backoffMultiplier ?? DEFAULT_RETRY.backoffMultiplier;
+	const retryableStatuses = config?.retryableStatuses ?? DEFAULT_RETRY.retryableStatuses;
+	let lastError;
+	for (let attempt = 1; attempt <= maxAttempts; attempt++) try {
+		return {
+			result: await fn(),
+			attempts: attempt
+		};
+	} catch (error) {
+		lastError = error instanceof Error ? error : new Error(String(error));
+		if (attempt === maxAttempts || !isRetryableError(error, retryableStatuses)) throw lastError;
+		const jitter = backoffMs * multiplier ** (attempt - 1) * (.9 + Math.random() * .2);
+		onRetry?.(attempt, lastError, jitter);
+		await sleep(jitter);
+	}
+	throw lastError ?? /* @__PURE__ */ new Error("Retry failed");
+}
+/**
+* Execute a function with timeout.
+*/
+async function withTimeout(fn, timeoutMs) {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+	try {
+		return await fn(controller.signal);
+	} finally {
+		clearTimeout(timeoutId);
+	}
+}
+/**
+* Create an AbortSignal that times out after specified milliseconds.
+* If parentSignal is provided, this signal will abort when the parent aborts.
+*/
+function createTimeoutSignal(timeoutMs, parentSignal) {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+	timeoutId.unref?.();
+	const clear = () => clearTimeout(timeoutId);
+	controller.signal.addEventListener("abort", clear, { once: true });
+	if (parentSignal) {
+		if (parentSignal.aborted) {
+			clear();
+			controller.abort(parentSignal.reason);
+			return controller.signal;
+		}
+		parentSignal.addEventListener("abort", () => {
+			clear();
+			controller.abort(parentSignal.reason);
+		}, { once: true });
+	}
+	return controller.signal;
+}
+/**
+* Default circuit breaker configuration.
+*/
+const DEFAULT_CIRCUIT_BREAKER = {
+	failureThreshold: 5,
+	resetTimeoutMs: 3e4
+};
+/**
+* Error thrown when circuit breaker is open.
+*/
+var CircuitOpenError = class extends Error {
+	isCircuitOpen = true;
+	constructor(message) {
+		super(message);
+		this.name = "CircuitOpenError";
+	}
+};
+/**
+* Circuit breaker implementation.
+* Prevents cascade failures by stopping requests when failure rate is high.
+*/
+var CircuitBreaker = class {
+	state;
+	failureThreshold;
+	resetTimeoutMs;
+	constructor(config) {
+		this.failureThreshold = config?.failureThreshold ?? DEFAULT_CIRCUIT_BREAKER.failureThreshold;
+		this.resetTimeoutMs = config?.resetTimeoutMs ?? DEFAULT_CIRCUIT_BREAKER.resetTimeoutMs;
+		this.state = {
+			state: "closed",
+			failures: 0
+		};
+	}
+	/**
+	* Check if requests are blocked.
+	*/
+	isOpen() {
+		this.updateState();
+		return this.state.state === "open";
+	}
+	/**
+	* Get current circuit state.
+	*/
+	getState() {
+		this.updateState();
+		return this.state.state;
+	}
+	/**
+	* Record a successful request.
+	*/
+	recordSuccess() {
+		this.state.failures = 0;
+		this.state.state = "closed";
+		this.state.lastFailureTime = void 0;
+		this.state.nextAttemptTime = void 0;
+	}
+	/**
+	* Record a failed request.
+	*/
+	recordFailure() {
+		this.state.failures++;
+		this.state.lastFailureTime = Date.now();
+		if (this.state.failures >= this.failureThreshold) {
+			this.state.state = "open";
+			this.state.nextAttemptTime = Date.now() + this.resetTimeoutMs;
+		}
+	}
+	/**
+	* Execute a function with circuit breaker protection.
+	*/
+	async execute(fn) {
+		if (this.isOpen()) throw new CircuitOpenError(`Circuit breaker is open. Next attempt at ${this.state.nextAttemptTime ? new Date(this.state.nextAttemptTime).toISOString() : "unknown"}`);
+		try {
+			const result = await fn();
+			this.recordSuccess();
+			return result;
+		} catch (error) {
+			this.recordFailure();
+			throw error;
+		}
+	}
+	/**
+	* Reset the circuit breaker.
+	*/
+	reset() {
+		this.state = {
+			state: "closed",
+			failures: 0
+		};
+	}
+	/**
+	* Update state based on time (open -> half-open transition).
+	*/
+	updateState() {
+		if (this.state.state === "open" && this.state.nextAttemptTime && Date.now() >= this.state.nextAttemptTime) this.state.state = "half-open";
+	}
+};
+/**
+* Token bucket rate limiter.
+*/
+var RateLimiter = class {
+	tokens;
+	lastRefill;
+	maxTokens;
+	refillRate;
+	constructor(config) {
+		const requestsPerSecond = (config.requestsPerMinute ?? 60) / 60;
+		this.maxTokens = Math.max(1, Math.ceil(requestsPerSecond * 10));
+		this.refillRate = requestsPerSecond;
+		this.tokens = this.maxTokens;
+		this.lastRefill = Date.now();
+	}
+	/**
+	* Check if a request is allowed without consuming tokens.
+	*/
+	canProceed() {
+		this.refill();
+		return this.tokens >= 1;
+	}
+	/**
+	* Attempt to acquire tokens for a request.
+	* Returns true if allowed, false if rate limited.
+	*/
+	tryAcquire(tokens = 1) {
+		this.refill();
+		if (this.tokens >= tokens) {
+			this.tokens -= tokens;
+			return true;
+		}
+		return false;
+	}
+	/**
+	* Wait until tokens are available, then acquire.
+	*/
+	async acquire(tokens = 1) {
+		if (this.tryAcquire(tokens)) return;
+		this.refill();
+		const tokensNeeded = tokens - this.tokens;
+		const waitMs = Math.ceil(tokensNeeded / this.refillRate * 1e3);
+		if (waitMs > 0) await sleep(waitMs);
+		while (!this.tryAcquire(tokens)) await sleep(Math.ceil(1 / this.refillRate * 1e3));
+	}
+	/**
+	* Get time until next token is available (in milliseconds).
+	*/
+	getWaitTime() {
+		this.refill();
+		if (this.tokens >= 1) return 0;
+		return Math.ceil(1 / this.refillRate * 1e3);
+	}
+	/**
+	* Refill tokens based on elapsed time.
+	*/
+	refill() {
+		const now = Date.now();
+		const newTokens = (now - this.lastRefill) / 1e3 * this.refillRate;
+		this.tokens = Math.min(this.maxTokens, this.tokens + newTokens);
+		this.lastRefill = now;
+	}
+};
+/**
+* Semaphore for limiting concurrent operations.
+*/
+var Semaphore = class {
+	permits;
+	waiting = [];
+	constructor(permits) {
+		this.permits = permits;
+	}
+	/**
+	* Acquire a permit, waiting if necessary.
+	*/
+	async acquire() {
+		if (this.permits > 0) {
+			this.permits--;
+			return;
+		}
+		return new Promise((resolve) => {
+			this.waiting.push(resolve);
+		});
+	}
+	/**
+	* Release a permit.
+	*/
+	release() {
+		const next = this.waiting.shift();
+		if (next) next();
+		else this.permits++;
+	}
+	/**
+	* Execute function with semaphore protection.
+	*/
+	async execute(fn) {
+		await this.acquire();
+		try {
+			return await fn();
+		} finally {
+			this.release();
+		}
+	}
+};
+/**
+* Execute a function with all resilience features.
+*
+* @param fn - The async function to execute with resilience
+* @param config - Configuration for retry and timeout behavior
+* @param state - Pre-instantiated resilience primitives for stateful features.
+*   Circuit breaker, rate limiter, and semaphore must be instantiated by the caller
+*   and passed via state to enable those features. This allows sharing state across
+*   multiple calls for proper circuit breaker tracking and rate limiting.
+*   The config parameter is only used for retry and timeout settings.
+* @param callbacks - Optional callbacks for retry events
+*/
+async function withResilience(fn, config, state, callbacks) {
+	const timeoutMs = config?.timeoutMs ?? 3e4;
+	if (state?.circuitBreaker?.isOpen()) throw new CircuitOpenError("Circuit breaker is open");
+	if (state?.rateLimiter) await state.rateLimiter.acquire();
+	const executeWithConcurrency = async () => {
+		const withTimeoutFn = () => withTimeout(fn, timeoutMs);
+		try {
+			const retryResult = await withRetry(withTimeoutFn, config?.retry, callbacks?.onRetry);
+			state?.circuitBreaker?.recordSuccess();
+			return retryResult;
+		} catch (error) {
+			state?.circuitBreaker?.recordFailure();
+			throw error;
+		}
+	};
+	if (state?.semaphore) return state.semaphore.execute(executeWithConcurrency);
+	return executeWithConcurrency();
+}
+
+//#endregion
+//#region src/common/http-base.ts
+/**
+* Shared HTTP provider infrastructure for LLM and Embedding providers.
+* Provides SSRF protection, resilience, and error normalization.
+*/
+/**
+* Private IP ranges blocked for SSRF protection.
+*/
+const PRIVATE_IP_PATTERNS = [
+	/^10\./,
+	/^172\.(1[6-9]|2\d|3[01])\./,
+	/^192\.168\./,
+	/^127\./,
+	/^0\./,
+	/^169\.254\./,
+	/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./,
+	/^::1$/,
+	/^(fc|fd)[0-9a-f]{2}:/i,
+	/^fe80:/i,
+	/^fec0:/i,
+	/^::ffff:(10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.|127\.|0\.)/i,
+	/^localhost$/i
+];
+/**
+* Check if a hostname/IP is private.
+*/
+function isPrivateHost(hostname) {
+	return PRIVATE_IP_PATTERNS.some((pattern) => pattern.test(hostname));
+}
+/**
+* Validate a URL for security.
+*/
+function validateUrl(url, options = {}) {
+	const requireHttps = options.requireHttps ?? true;
+	const allowPrivate = options.allowPrivate ?? false;
+	let parsed;
+	try {
+		parsed = new URL(url);
+	} catch {
+		throw new ScrapeError(`Invalid URL: ${url}`, "INVALID_URL");
+	}
+	if (requireHttps && parsed.protocol !== "https:") throw new ScrapeError(`HTTPS required. Got: ${parsed.protocol}`, "VALIDATION_ERROR");
+	if (!allowPrivate && isPrivateHost(parsed.hostname)) throw new ScrapeError(`Private/internal addresses not allowed: ${parsed.hostname}`, "VALIDATION_ERROR");
+	return parsed;
+}
+/**
+* Validate URL and resolve DNS to check for private IPs.
+*/
+async function validateUrlWithDns(url, options = {}) {
+	const parsed = validateUrl(url, options);
+	const resolveDns = options.resolveDns ?? true;
+	const allowPrivate = options.allowPrivate ?? false;
+	if (!resolveDns || allowPrivate) return;
+	const host = parsed.hostname;
+	if (isIP(host)) return;
+	try {
+		const addresses = await promises.lookup(host, { all: true });
+		for (const addr of addresses) if (isPrivateHost(addr.address)) throw new ScrapeError(`DNS resolved to private address: ${host} -> ${addr.address}`, "VALIDATION_ERROR");
+	} catch (error) {
+		if (error instanceof ScrapeError) throw error;
+		throw new ScrapeError(`Failed to resolve hostname: ${host} (${error instanceof Error ? error.message : String(error)})`, "FETCH_FAILED");
+	}
+}
+/**
+* Base HTTP provider with shared security and resilience.
+*/
+var BaseHttpProvider = class {
+	baseUrl;
+	model;
+	headers;
+	errorMapper;
+	requireHttps;
+	allowPrivate;
+	resolveDns;
+	allowRedirects;
+	timeoutMs;
+	retryConfig;
+	concurrency;
+	circuitBreaker;
+	rateLimiter;
+	semaphore;
+	constructor(config) {
+		this.baseUrl = config.baseUrl.replace(/\/$/, "");
+		this.model = config.model;
+		this.headers = {
+			"Content-Type": "application/json",
+			...config.headers
+		};
+		this.errorMapper = config.errorMapper;
+		this.requireHttps = config.requireHttps ?? true;
+		this.allowPrivate = config.allowPrivate ?? false;
+		this.resolveDns = config.resolveDns ?? true;
+		this.allowRedirects = config.allowRedirects ?? false;
+		this.timeoutMs = config.resilience?.timeoutMs ?? 3e4;
+		this.retryConfig = config.resilience?.retry;
+		this.concurrency = config.resilience?.concurrency ?? 1;
+		const sharedState = config.resilience?.state;
+		this.circuitBreaker = sharedState?.circuitBreaker ?? (config.resilience?.circuitBreaker ? new CircuitBreaker(config.resilience.circuitBreaker) : void 0);
+		this.rateLimiter = sharedState?.rateLimiter ?? (config.resilience?.rateLimit ? new RateLimiter(config.resilience.rateLimit) : void 0);
+		this.semaphore = sharedState?.semaphore ?? new Semaphore(this.concurrency);
+		validateUrl(this.baseUrl, {
+			requireHttps: this.requireHttps,
+			allowPrivate: this.allowPrivate
+		});
+	}
+	/**
+	* Get the current resilience state for persistence across calls.
+	*/
+	getResilienceState() {
+		return {
+			circuitBreaker: this.circuitBreaker,
+			rateLimiter: this.rateLimiter,
+			semaphore: this.semaphore
+		};
+	}
+	/**
+	* Make an HTTP request with security and resilience.
+	*/
+	async fetch(url, options = {}) {
+		const securityOptions = {
+			requireHttps: this.requireHttps,
+			allowPrivate: this.allowPrivate,
+			resolveDns: this.resolveDns,
+			allowRedirects: this.allowRedirects
+		};
+		await validateUrlWithDns(url, securityOptions);
+		if (this.circuitBreaker?.isOpen()) throw new CircuitOpenError("Circuit breaker is open. Too many recent failures.");
+		if (this.rateLimiter) await this.rateLimiter.acquire();
+		const doFetch = async (signal) => {
+			const composedSignal = options.signal ? AbortSignal.any([options.signal, signal]) : signal;
+			const response = await fetch(url, {
+				method: options.method ?? "POST",
+				headers: {
+					...this.headers,
+					...options.headers
+				},
+				body: options.body ? JSON.stringify(options.body) : void 0,
+				signal: composedSignal,
+				redirect: this.allowRedirects ? "follow" : "error"
+			});
+			if (this.allowRedirects && response.redirected) await validateUrlWithDns(response.url, securityOptions);
+			if (!response.ok) throw await createHttpError(response, this.constructor.name, this.errorMapper);
+			return {
+				data: await response.json(),
+				status: response.status,
+				headers: response.headers
+			};
+		};
+		const executeWithConcurrency = async () => {
+			if (!this.semaphore) throw new ScrapeError("Semaphore not initialized", "VALIDATION_ERROR");
+			return this.semaphore.execute(async () => {
+				const fetchWithTimeout = async () => {
+					return withTimeout((signal) => doFetch(signal), this.timeoutMs);
+				};
+				try {
+					let result;
+					if (this.retryConfig) result = (await withRetry(fetchWithTimeout, this.retryConfig)).result;
+					else result = await fetchWithTimeout();
+					this.circuitBreaker?.recordSuccess();
+					return result;
+				} catch (error) {
+					this.circuitBreaker?.recordFailure();
+					throw error;
+				}
+			});
+		};
+		return executeWithConcurrency();
+	}
+};
+
+//#endregion
+export { Semaphore as a, withResilience as c, ScrapeError as d, RateLimiter as i, withRetry as l, CircuitBreaker as n, createTimeoutSignal as o, CircuitOpenError as r, isRetryableError as s, BaseHttpProvider as t, withTimeout as u };
+//# sourceMappingURL=http-base-DM7YNo6X.mjs.map