scrapex 1.0.0-alpha.1 → 1.0.0-beta.2

package/dist/http-base-CHLf-Tco.cjs

@@ -0,0 +1,684 @@
+const require_parsers = require('./parsers-Bneuws8x.cjs');
+let node_dns = require("node:dns");
+let node_net = require("node:net");
+
+//#region src/core/errors.ts
+/**
+* Custom error class for scraping failures with structured error codes
+*/
+var ScrapeError = class ScrapeError extends Error {
+	code;
+	statusCode;
+	constructor(message, code, statusCode, cause) {
+		super(message, { cause });
+		this.name = "ScrapeError";
+		this.code = code;
+		this.statusCode = statusCode;
+		if (Error.captureStackTrace) Error.captureStackTrace(this, ScrapeError);
+	}
+	/**
+	* Create a ScrapeError from an unknown error
+	*/
+	static from(error, code = "FETCH_FAILED") {
+		if (error instanceof ScrapeError) return error;
+		if (error instanceof Error) return new ScrapeError(error.message, code, void 0, error);
+		return new ScrapeError(String(error), code);
+	}
+	/**
+	* Check if error is retryable (network issues, timeouts)
+	*/
+	isRetryable() {
+		return this.code === "FETCH_FAILED" || this.code === "TIMEOUT";
+	}
+	/**
+	* Convert to a plain object for serialization
+	*/
+	toJSON() {
+		return {
+			name: this.name,
+			message: this.message,
+			code: this.code,
+			statusCode: this.statusCode,
+			stack: this.stack
+		};
+	}
+};
+
+//#endregion
+//#region src/common/errors.ts
+/**
+* Error normalization utilities for HTTP providers.
+* Maps HTTP status codes to consistent ScrapeError codes.
+*/
+/**
+* HTTP status code to ScrapeError code mapping.
+*/
+function getErrorCodeFromStatus(status) {
+	if (status === 401 || status === 403) return "BLOCKED";
+	if (status === 404) return "NOT_FOUND";
+	if (status === 429) return "BLOCKED";
+	if (status === 408) return "TIMEOUT";
+	if (status >= 500) return "LLM_ERROR";
+	return "FETCH_FAILED";
+}
+/**
+* Parse error message from API response body.
+*/
+async function parseErrorBody(response) {
+	try {
+		const text = await response.text();
+		try {
+			const json = JSON.parse(text);
+			if (typeof json.error === "object" && json.error !== null) {
+				const error = json.error;
+				return String(error.message ?? error.msg ?? JSON.stringify(error));
+			}
+			if (typeof json.error === "string") return json.error;
+			if (typeof json.message === "string") return json.message;
+			if (typeof json.detail === "string") return json.detail;
+			return text;
+		} catch {
+			return text || `HTTP ${response.status} ${response.statusText}`;
+		}
+	} catch {
+		return `HTTP ${response.status} ${response.statusText}`;
+	}
+}
+/**
+* Create a ScrapeError from an HTTP response.
+*/
+async function createHttpError(response, providerName, errorMapper) {
+	const code = getErrorCodeFromStatus(response.status);
+	let message;
+	if (errorMapper) try {
+		message = errorMapper(await response.json());
+	} catch {
+		message = await parseErrorBody(response);
+	}
+	else message = await parseErrorBody(response);
+	return new ScrapeError(`${providerName} API error (${response.status}): ${message}`, code, response.status);
+}
+
+//#endregion
+//#region src/common/resilience.ts
+/**
+* Default retry configuration.
+*/
+const DEFAULT_RETRY = {
+	maxAttempts: 3,
+	backoffMs: 1e3,
+	backoffMultiplier: 2,
+	retryableStatuses: [
+		408,
+		429,
+		500,
+		502,
+		503,
+		504
+	]
+};
+/**
+* Errors that should be retried (transient failures).
+*/
+const RETRYABLE_ERROR_CODES = [
+	"ECONNRESET",
+	"ETIMEDOUT",
+	"ECONNREFUSED",
+	"EPIPE",
+	"ENOTFOUND",
+	"ENETUNREACH",
+	"EAI_AGAIN"
+];
+/**
+* Check if an error is retryable.
+*/
+function isRetryableError(error, retryableStatuses = DEFAULT_RETRY.retryableStatuses) {
+	if (error instanceof Error) {
+		const code = error.code;
+		if (code && RETRYABLE_ERROR_CODES.includes(code)) return true;
+		if ("statusCode" in error && typeof error.statusCode === "number") return retryableStatuses.includes(error.statusCode);
+		if ("status" in error && typeof error.status === "number") return retryableStatuses.includes(error.status);
+		if ("code" in error) {
+			const errCode = error.code;
+			if (errCode === "TIMEOUT" || errCode === "FETCH_FAILED") return true;
+		}
+		const message = error.message.toLowerCase();
+		if (message.includes("timeout") || message.includes("rate limit") || message.includes("too many requests") || message.includes("temporarily unavailable")) return true;
+	}
+	return false;
+}
+/**
+* Sleep for specified milliseconds.
+*/
+function sleep(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+* Execute a function with retry logic.
+*/
+async function withRetry(fn, config, onRetry) {
+	const maxAttempts = config?.maxAttempts ?? DEFAULT_RETRY.maxAttempts;
+	const backoffMs = config?.backoffMs ?? DEFAULT_RETRY.backoffMs;
+	const multiplier = config?.backoffMultiplier ?? DEFAULT_RETRY.backoffMultiplier;
+	const retryableStatuses = config?.retryableStatuses ?? DEFAULT_RETRY.retryableStatuses;
+	let lastError;
+	for (let attempt = 1; attempt <= maxAttempts; attempt++) try {
+		return {
+			result: await fn(),
+			attempts: attempt
+		};
+	} catch (error) {
+		lastError = error instanceof Error ? error : new Error(String(error));
+		if (attempt === maxAttempts || !isRetryableError(error, retryableStatuses)) throw lastError;
+		const jitter = backoffMs * multiplier ** (attempt - 1) * (.9 + Math.random() * .2);
+		onRetry?.(attempt, lastError, jitter);
+		await sleep(jitter);
+	}
+	throw lastError ?? /* @__PURE__ */ new Error("Retry failed");
+}
+/**
+* Execute a function with timeout.
+*/
+async function withTimeout(fn, timeoutMs) {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+	try {
+		return await fn(controller.signal);
+	} finally {
+		clearTimeout(timeoutId);
+	}
+}
+/**
+* Create an AbortSignal that times out after specified milliseconds.
+* If parentSignal is provided, this signal will abort when the parent aborts.
+*/
+function createTimeoutSignal(timeoutMs, parentSignal) {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+	timeoutId.unref?.();
+	const clear = () => clearTimeout(timeoutId);
+	controller.signal.addEventListener("abort", clear, { once: true });
+	if (parentSignal) {
+		if (parentSignal.aborted) {
+			clear();
+			controller.abort(parentSignal.reason);
+			return controller.signal;
+		}
+		parentSignal.addEventListener("abort", () => {
+			clear();
+			controller.abort(parentSignal.reason);
+		}, { once: true });
+	}
+	return controller.signal;
+}
+/**
+* Default circuit breaker configuration.
+*/
+const DEFAULT_CIRCUIT_BREAKER = {
+	failureThreshold: 5,
+	resetTimeoutMs: 3e4
+};
+/**
+* Error thrown when circuit breaker is open.
+*/
+var CircuitOpenError = class extends Error {
+	isCircuitOpen = true;
+	constructor(message) {
+		super(message);
+		this.name = "CircuitOpenError";
+	}
+};
+/**
+* Circuit breaker implementation.
+* Prevents cascade failures by stopping requests when failure rate is high.
+*/
+var CircuitBreaker = class {
+	state;
+	failureThreshold;
+	resetTimeoutMs;
+	constructor(config) {
+		this.failureThreshold = config?.failureThreshold ?? DEFAULT_CIRCUIT_BREAKER.failureThreshold;
+		this.resetTimeoutMs = config?.resetTimeoutMs ?? DEFAULT_CIRCUIT_BREAKER.resetTimeoutMs;
+		this.state = {
+			state: "closed",
+			failures: 0
+		};
+	}
+	/**
+	* Check if requests are blocked.
+	*/
+	isOpen() {
+		this.updateState();
+		return this.state.state === "open";
+	}
+	/**
+	* Get current circuit state.
+	*/
+	getState() {
+		this.updateState();
+		return this.state.state;
+	}
+	/**
+	* Record a successful request.
+	*/
+	recordSuccess() {
+		this.state.failures = 0;
+		this.state.state = "closed";
+		this.state.lastFailureTime = void 0;
+		this.state.nextAttemptTime = void 0;
+	}
+	/**
+	* Record a failed request.
+	*/
+	recordFailure() {
+		this.state.failures++;
+		this.state.lastFailureTime = Date.now();
+		if (this.state.failures >= this.failureThreshold) {
+			this.state.state = "open";
+			this.state.nextAttemptTime = Date.now() + this.resetTimeoutMs;
+		}
+	}
+	/**
+	* Execute a function with circuit breaker protection.
+	*/
+	async execute(fn) {
+		if (this.isOpen()) throw new CircuitOpenError(`Circuit breaker is open. Next attempt at ${this.state.nextAttemptTime ? new Date(this.state.nextAttemptTime).toISOString() : "unknown"}`);
+		try {
+			const result = await fn();
+			this.recordSuccess();
+			return result;
+		} catch (error) {
+			this.recordFailure();
+			throw error;
+		}
+	}
+	/**
+	* Reset the circuit breaker.
+	*/
+	reset() {
+		this.state = {
+			state: "closed",
+			failures: 0
+		};
+	}
+	/**
+	* Update state based on time (open -> half-open transition).
+	*/
+	updateState() {
+		if (this.state.state === "open" && this.state.nextAttemptTime && Date.now() >= this.state.nextAttemptTime) this.state.state = "half-open";
+	}
+};
+/**
+* Token bucket rate limiter.
+*/
+var RateLimiter = class {
+	tokens;
+	lastRefill;
+	maxTokens;
+	refillRate;
+	constructor(config) {
+		const requestsPerSecond = (config.requestsPerMinute ?? 60) / 60;
+		this.maxTokens = Math.max(1, Math.ceil(requestsPerSecond * 10));
+		this.refillRate = requestsPerSecond;
+		this.tokens = this.maxTokens;
+		this.lastRefill = Date.now();
+	}
+	/**
+	* Check if a request is allowed without consuming tokens.
+	*/
+	canProceed() {
+		this.refill();
+		return this.tokens >= 1;
+	}
+	/**
+	* Attempt to acquire tokens for a request.
+	* Returns true if allowed, false if rate limited.
+	*/
+	tryAcquire(tokens = 1) {
+		this.refill();
+		if (this.tokens >= tokens) {
+			this.tokens -= tokens;
+			return true;
+		}
+		return false;
+	}
+	/**
+	* Wait until tokens are available, then acquire.
+	*/
+	async acquire(tokens = 1) {
+		if (this.tryAcquire(tokens)) return;
+		this.refill();
+		const tokensNeeded = tokens - this.tokens;
+		const waitMs = Math.ceil(tokensNeeded / this.refillRate * 1e3);
+		if (waitMs > 0) await sleep(waitMs);
+		while (!this.tryAcquire(tokens)) await sleep(Math.ceil(1 / this.refillRate * 1e3));
+	}
+	/**
+	* Get time until next token is available (in milliseconds).
+	*/
+	getWaitTime() {
+		this.refill();
+		if (this.tokens >= 1) return 0;
+		return Math.ceil(1 / this.refillRate * 1e3);
+	}
+	/**
+	* Refill tokens based on elapsed time.
+	*/
+	refill() {
+		const now = Date.now();
+		const newTokens = (now - this.lastRefill) / 1e3 * this.refillRate;
+		this.tokens = Math.min(this.maxTokens, this.tokens + newTokens);
+		this.lastRefill = now;
+	}
+};
+/**
+* Semaphore for limiting concurrent operations.
+*/
+var Semaphore = class {
+	permits;
+	waiting = [];
+	constructor(permits) {
+		this.permits = permits;
+	}
+	/**
+	* Acquire a permit, waiting if necessary.
+	*/
+	async acquire() {
+		if (this.permits > 0) {
+			this.permits--;
+			return;
+		}
+		return new Promise((resolve) => {
+			this.waiting.push(resolve);
+		});
+	}
+	/**
+	* Release a permit.
+	*/
+	release() {
+		const next = this.waiting.shift();
+		if (next) next();
+		else this.permits++;
+	}
+	/**
+	* Execute function with semaphore protection.
+	*/
+	async execute(fn) {
+		await this.acquire();
+		try {
+			return await fn();
+		} finally {
+			this.release();
+		}
+	}
+};
+/**
+* Execute a function with all resilience features.
+*
+* @param fn - The async function to execute with resilience
+* @param config - Configuration for retry and timeout behavior
+* @param state - Pre-instantiated resilience primitives for stateful features.
+*   Circuit breaker, rate limiter, and semaphore must be instantiated by the caller
+*   and passed via state to enable those features. This allows sharing state across
+*   multiple calls for proper circuit breaker tracking and rate limiting.
+*   The config parameter is only used for retry and timeout settings.
+* @param callbacks - Optional callbacks for retry events
+*/
+async function withResilience(fn, config, state, callbacks) {
+	const timeoutMs = config?.timeoutMs ?? 3e4;
+	if (state?.circuitBreaker?.isOpen()) throw new CircuitOpenError("Circuit breaker is open");
+	if (state?.rateLimiter) await state.rateLimiter.acquire();
+	const executeWithConcurrency = async () => {
+		const withTimeoutFn = () => withTimeout(fn, timeoutMs);
+		try {
+			const retryResult = await withRetry(withTimeoutFn, config?.retry, callbacks?.onRetry);
+			state?.circuitBreaker?.recordSuccess();
+			return retryResult;
+		} catch (error) {
+			state?.circuitBreaker?.recordFailure();
+			throw error;
+		}
+	};
+	if (state?.semaphore) return state.semaphore.execute(executeWithConcurrency);
+	return executeWithConcurrency();
+}
+
+//#endregion
+//#region src/common/http-base.ts
+/**
+* Shared HTTP provider infrastructure for LLM and Embedding providers.
+* Provides SSRF protection, resilience, and error normalization.
+*/
+/**
+* Private IP ranges blocked for SSRF protection.
+*/
+const PRIVATE_IP_PATTERNS = [
+	/^10\./,
+	/^172\.(1[6-9]|2\d|3[01])\./,
+	/^192\.168\./,
+	/^127\./,
+	/^0\./,
+	/^169\.254\./,
+	/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./,
+	/^::1$/,
+	/^(fc|fd)[0-9a-f]{2}:/i,
+	/^fe80:/i,
+	/^fec0:/i,
+	/^::ffff:(10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.|127\.|0\.)/i,
+	/^localhost$/i
+];
+/**
+* Check if a hostname/IP is private.
+*/
+function isPrivateHost(hostname) {
+	return PRIVATE_IP_PATTERNS.some((pattern) => pattern.test(hostname));
+}
+/**
+* Validate a URL for security.
+*/
+function validateUrl(url, options = {}) {
+	const requireHttps = options.requireHttps ?? true;
+	const allowPrivate = options.allowPrivate ?? false;
+	let parsed;
+	try {
+		parsed = new URL(url);
+	} catch {
+		throw new ScrapeError(`Invalid URL: ${url}`, "INVALID_URL");
+	}
+	if (requireHttps && parsed.protocol !== "https:") throw new ScrapeError(`HTTPS required. Got: ${parsed.protocol}`, "VALIDATION_ERROR");
+	if (!allowPrivate && isPrivateHost(parsed.hostname)) throw new ScrapeError(`Private/internal addresses not allowed: ${parsed.hostname}`, "VALIDATION_ERROR");
+	return parsed;
+}
+/**
+* Validate URL and resolve DNS to check for private IPs.
+*/
+async function validateUrlWithDns(url, options = {}) {
+	const parsed = validateUrl(url, options);
+	const resolveDns = options.resolveDns ?? true;
+	const allowPrivate = options.allowPrivate ?? false;
+	if (!resolveDns || allowPrivate) return;
+	const host = parsed.hostname;
+	if ((0, node_net.isIP)(host)) return;
+	try {
+		const addresses = await node_dns.promises.lookup(host, { all: true });
+		for (const addr of addresses) if (isPrivateHost(addr.address)) throw new ScrapeError(`DNS resolved to private address: ${host} -> ${addr.address}`, "VALIDATION_ERROR");
+	} catch (error) {
+		if (error instanceof ScrapeError) throw error;
+		throw new ScrapeError(`Failed to resolve hostname: ${host} (${error instanceof Error ? error.message : String(error)})`, "FETCH_FAILED");
+	}
+}
+/**
+* Base HTTP provider with shared security and resilience.
+*/
+var BaseHttpProvider = class {
+	baseUrl;
+	model;
+	headers;
+	errorMapper;
+	requireHttps;
+	allowPrivate;
+	resolveDns;
+	allowRedirects;
+	timeoutMs;
+	retryConfig;
+	concurrency;
+	circuitBreaker;
+	rateLimiter;
+	semaphore;
+	constructor(config) {
+		this.baseUrl = config.baseUrl.replace(/\/$/, "");
+		this.model = config.model;
+		this.headers = {
+			"Content-Type": "application/json",
+			...config.headers
+		};
+		this.errorMapper = config.errorMapper;
+		this.requireHttps = config.requireHttps ?? true;
+		this.allowPrivate = config.allowPrivate ?? false;
+		this.resolveDns = config.resolveDns ?? true;
+		this.allowRedirects = config.allowRedirects ?? false;
+		this.timeoutMs = config.resilience?.timeoutMs ?? 3e4;
+		this.retryConfig = config.resilience?.retry;
+		this.concurrency = config.resilience?.concurrency ?? 1;
+		const sharedState = config.resilience?.state;
+		this.circuitBreaker = sharedState?.circuitBreaker ?? (config.resilience?.circuitBreaker ? new CircuitBreaker(config.resilience.circuitBreaker) : void 0);
+		this.rateLimiter = sharedState?.rateLimiter ?? (config.resilience?.rateLimit ? new RateLimiter(config.resilience.rateLimit) : void 0);
+		this.semaphore = sharedState?.semaphore ?? new Semaphore(this.concurrency);
+		validateUrl(this.baseUrl, {
+			requireHttps: this.requireHttps,
+			allowPrivate: this.allowPrivate
+		});
+	}
+	/**
+	* Get the current resilience state for persistence across calls.
+	*/
+	getResilienceState() {
+		return {
+			circuitBreaker: this.circuitBreaker,
+			rateLimiter: this.rateLimiter,
+			semaphore: this.semaphore
+		};
+	}
+	/**
+	* Make an HTTP request with security and resilience.
+	*/
+	async fetch(url, options = {}) {
+		const securityOptions = {
+			requireHttps: this.requireHttps,
+			allowPrivate: this.allowPrivate,
+			resolveDns: this.resolveDns,
+			allowRedirects: this.allowRedirects
+		};
+		await validateUrlWithDns(url, securityOptions);
+		if (this.circuitBreaker?.isOpen()) throw new CircuitOpenError("Circuit breaker is open. Too many recent failures.");
+		if (this.rateLimiter) await this.rateLimiter.acquire();
+		const doFetch = async (signal) => {
+			const composedSignal = options.signal ? AbortSignal.any([options.signal, signal]) : signal;
+			const response = await fetch(url, {
+				method: options.method ?? "POST",
+				headers: {
+					...this.headers,
+					...options.headers
+				},
+				body: options.body ? JSON.stringify(options.body) : void 0,
+				signal: composedSignal,
+				redirect: this.allowRedirects ? "follow" : "error"
+			});
+			if (this.allowRedirects && response.redirected) await validateUrlWithDns(response.url, securityOptions);
+			if (!response.ok) throw await createHttpError(response, this.constructor.name, this.errorMapper);
+			return {
+				data: await response.json(),
+				status: response.status,
+				headers: response.headers
+			};
+		};
+		const executeWithConcurrency = async () => {
+			if (!this.semaphore) throw new ScrapeError("Semaphore not initialized", "VALIDATION_ERROR");
+			return this.semaphore.execute(async () => {
+				const fetchWithTimeout = async () => {
+					return withTimeout((signal) => doFetch(signal), this.timeoutMs);
+				};
+				try {
+					let result;
+					if (this.retryConfig) result = (await withRetry(fetchWithTimeout, this.retryConfig)).result;
+					else result = await fetchWithTimeout();
+					this.circuitBreaker?.recordSuccess();
+					return result;
+				} catch (error) {
+					this.circuitBreaker?.recordFailure();
+					throw error;
+				}
+			});
+		};
+		return executeWithConcurrency();
+	}
+};
+
+//#endregion
+Object.defineProperty(exports, 'BaseHttpProvider', {
+  enumerable: true,
+  get: function () {
+    return BaseHttpProvider;
+  }
+});
+Object.defineProperty(exports, 'CircuitBreaker', {
+  enumerable: true,
+  get: function () {
+    return CircuitBreaker;
+  }
+});
+Object.defineProperty(exports, 'CircuitOpenError', {
+  enumerable: true,
+  get: function () {
+    return CircuitOpenError;
+  }
+});
+Object.defineProperty(exports, 'RateLimiter', {
+  enumerable: true,
+  get: function () {
+    return RateLimiter;
+  }
+});
+Object.defineProperty(exports, 'ScrapeError', {
+  enumerable: true,
+  get: function () {
+    return ScrapeError;
+  }
+});
+Object.defineProperty(exports, 'Semaphore', {
+  enumerable: true,
+  get: function () {
+    return Semaphore;
+  }
+});
+Object.defineProperty(exports, 'createTimeoutSignal', {
+  enumerable: true,
+  get: function () {
+    return createTimeoutSignal;
+  }
+});
+Object.defineProperty(exports, 'isRetryableError', {
+  enumerable: true,
+  get: function () {
+    return isRetryableError;
+  }
+});
+Object.defineProperty(exports, 'withResilience', {
+  enumerable: true,
+  get: function () {
+    return withResilience;
+  }
+});
+Object.defineProperty(exports, 'withRetry', {
+  enumerable: true,
+  get: function () {
+    return withRetry;
+  }
+});
+Object.defineProperty(exports, 'withTimeout', {
+  enumerable: true,
+  get: function () {
+    return withTimeout;
+  }
+});
+//# sourceMappingURL=http-base-CHLf-Tco.cjs.map