scrapex 1.0.0-alpha.1 → 1.0.0-beta.2

package/dist/types-BOcHQU9s.d.mts

@@ -0,0 +1,831 @@
+import { CheerioAPI } from "cheerio";
+
+//#region src/core/types.d.ts
+
+/**
+ * Content type classification for scraped URLs
+ */
+type ContentType = 'article' | 'repo' | 'docs' | 'package' | 'video' | 'tool' | 'product' | 'unknown';
+/**
+ * Extracted link from content
+ */
+interface ExtractedLink {
+  url: string;
+  text: string;
+  isExternal: boolean;
+}
+/**
+ * Extracted entities from LLM enhancement
+ */
+interface ExtractedEntities {
+  people: string[];
+  organizations: string[];
+  technologies: string[];
+  locations: string[];
+  concepts: string[];
+}
+/**
+ * Main result of metadata scraping - optimized for LLM consumption
+ */
+interface ScrapedData {
+  url: string;
+  canonicalUrl: string;
+  domain: string;
+  title: string;
+  description: string;
+  image?: string;
+  favicon?: string;
+  content: string;
+  textContent: string;
+  excerpt: string;
+  wordCount: number;
+  author?: string;
+  publishedAt?: string;
+  modifiedAt?: string;
+  siteName?: string;
+  language?: string;
+  contentType: ContentType;
+  keywords: string[];
+  jsonLd?: Record<string, unknown>[];
+  links?: ExtractedLink[];
+  summary?: string;
+  suggestedTags?: string[];
+  entities?: ExtractedEntities;
+  extracted?: Record<string, unknown>;
+  custom?: Record<string, unknown>;
+  embeddings?: EmbeddingResult;
+  scrapedAt: string;
+  scrapeTimeMs: number;
+  error?: string;
+}
+/**
+ * LLM enhancement types
+ */
+type EnhancementType = 'summarize' | 'tags' | 'entities' | 'classify';
+/**
+ * Schema for structured LLM extraction
+ */
+type ExtractionSchemaType = 'string' | 'number' | 'boolean' | 'string[]' | 'number[]' | `${string}?`;
+type ExtractionSchema = Record<string, ExtractionSchemaType>;
+/**
+ * Forward declaration for LLM provider (defined in llm/types.ts)
+ */
+interface LLMProvider {
+  readonly name: string;
+  complete(prompt: string, options?: CompletionOptions): Promise<string>;
+  completeJSON<T>(prompt: string, schema: unknown): Promise<T>;
+}
+interface CompletionOptions {
+  maxTokens?: number;
+  temperature?: number;
+  systemPrompt?: string;
+}
+/**
+ * Forward declaration for Fetcher (defined in fetchers/types.ts)
+ */
+interface Fetcher {
+  readonly name: string;
+  fetch(url: string, options: FetchOptions$1): Promise<FetchResult$1>;
+}
+interface FetchOptions$1 {
+  timeout?: number;
+  userAgent?: string;
+  headers?: Record<string, string>;
+}
+interface FetchResult$1 {
+  html: string;
+  finalUrl: string;
+  statusCode: number;
+  contentType: string;
+  headers?: Record<string, string>;
+}
+/**
+ * Forward declaration for Extractor (defined in extractors/types.ts)
+ */
+interface Extractor {
+  readonly name: string;
+  readonly priority?: number;
+  extract(context: ExtractionContext): Promise<Partial<ScrapedData>>;
+}
+/**
+ * Shared context passed to all extractors
+ */
+interface ExtractionContext {
+  url: string;
+  finalUrl: string;
+  html: string;
+  $: CheerioAPI;
+  getDocument(): Document;
+  results: Partial<ScrapedData>;
+  options: ScrapeOptions;
+}
+/**
+ * Options for scraping
+ */
+interface ScrapeOptions {
+  /** Timeout in milliseconds (default: 10000) */
+  timeout?: number;
+  /** User agent string */
+  userAgent?: string;
+  /** Whether to extract full content (default: true) */
+  extractContent?: boolean;
+  /** Maximum content length in characters (default: 50000) */
+  maxContentLength?: number;
+  /** Custom fetcher (for Puppeteer/Playwright) */
+  fetcher?: Fetcher;
+  /** Custom extractors to run */
+  extractors?: Extractor[];
+  /** If true, only run custom extractors (replace defaults) */
+  replaceDefaultExtractors?: boolean;
+  /** Check robots.txt before scraping (default: false) */
+  respectRobots?: boolean;
+  /** LLM provider for enhancements */
+  llm?: LLMProvider;
+  /** LLM enhancement types to run */
+  enhance?: EnhancementType[];
+  /** Schema for structured LLM extraction */
+  extract?: ExtractionSchema;
+  /** Embedding generation options */
+  embeddings?: EmbeddingOptions;
+}
+//#endregion
+//#region src/common/resilience.d.ts
+/**
+ * Shared resilience utilities for HTTP providers.
+ * Provides retry, circuit breaker, rate limiting, timeout, and concurrency control.
+ */
+/**
+ * Retry configuration.
+ */
+interface RetryConfig$1 {
+  /** Maximum retry attempts. @default 3 */
+  maxAttempts?: number;
+  /** Initial backoff delay in ms. @default 1000 */
+  backoffMs?: number;
+  /** Backoff multiplier. @default 2 */
+  backoffMultiplier?: number;
+  /** HTTP status codes to retry. @default [408, 429, 500, 502, 503, 504] */
+  retryableStatuses?: number[];
+}
+/**
+ * Circuit breaker configuration.
+ */
+interface CircuitBreakerConfig$1 {
+  /** Failures before opening circuit. @default 5 */
+  failureThreshold?: number;
+  /** Time before attempting to close circuit. @default 30000 */
+  resetTimeoutMs?: number;
+}
+/**
+ * Rate limit configuration.
+ */
+interface RateLimitConfig$1 {
+  /** Max requests per minute */
+  requestsPerMinute?: number;
+  /** Max tokens per minute (for LLM providers) */
+  tokensPerMinute?: number;
+}
+/**
+ * Circuit breaker state.
+ */
+type CircuitState$1 = 'closed' | 'open' | 'half-open';
+/**
+ * Resilience configuration for HTTP providers.
+ */
+interface ResilienceConfig$1 {
+  retry?: RetryConfig$1;
+  circuitBreaker?: CircuitBreakerConfig$1;
+  rateLimit?: RateLimitConfig$1;
+  /** Request timeout in ms. @default 30000 */
+  timeoutMs?: number;
+  /** Max concurrent requests. @default 1 */
+  concurrency?: number;
+  /** Optional shared state for circuit breaker / rate limiter / semaphore */
+  state?: ResilienceState$1;
+}
+/**
+ * Shared resilience state for persistence across calls.
+ */
+interface ResilienceState$1 {
+  circuitBreaker?: {
+    isOpen(): boolean;
+    recordSuccess(): void;
+    recordFailure(): void;
+    getState?(): CircuitState$1;
+  };
+  rateLimiter?: {
+    acquire(): Promise<void>;
+  };
+  semaphore?: {
+    execute<T>(fn: () => Promise<T>): Promise<T>;
+  };
+}
+/**
+ * Check if an error is retryable.
+ */
+declare function isRetryableError(error: unknown, retryableStatuses?: number[]): boolean;
+/**
+ * Execute a function with retry logic.
+ */
+declare function withRetry<T>(fn: () => Promise<T>, config?: RetryConfig$1, onRetry?: (attempt: number, error: Error, delayMs: number) => void): Promise<{
+  result: T;
+  attempts: number;
+}>;
+/**
+ * Execute a function with timeout.
+ */
+declare function withTimeout<T>(fn: (signal: AbortSignal) => Promise<T>, timeoutMs: number): Promise<T>;
+/**
+ * Create an AbortSignal that times out after specified milliseconds.
+ * If parentSignal is provided, this signal will abort when the parent aborts.
+ */
+declare function createTimeoutSignal(timeoutMs: number, parentSignal?: AbortSignal): AbortSignal;
+/**
+ * Error thrown when circuit breaker is open.
+ */
+declare class CircuitOpenError extends Error {
+  readonly isCircuitOpen = true;
+  constructor(message: string);
+}
+/**
+ * Circuit breaker implementation.
+ * Prevents cascade failures by stopping requests when failure rate is high.
+ */
+declare class CircuitBreaker {
+  private state;
+  private readonly failureThreshold;
+  private readonly resetTimeoutMs;
+  constructor(config?: CircuitBreakerConfig$1);
+  /**
+   * Check if requests are blocked.
+   */
+  isOpen(): boolean;
+  /**
+   * Get current circuit state.
+   */
+  getState(): CircuitState$1;
+  /**
+   * Record a successful request.
+   */
+  recordSuccess(): void;
+  /**
+   * Record a failed request.
+   */
+  recordFailure(): void;
+  /**
+   * Execute a function with circuit breaker protection.
+   */
+  execute<T>(fn: () => Promise<T>): Promise<T>;
+  /**
+   * Reset the circuit breaker.
+   */
+  reset(): void;
+  /**
+   * Update state based on time (open -> half-open transition).
+   */
+  private updateState;
+}
+/**
+ * Token bucket rate limiter.
+ */
+declare class RateLimiter {
+  private tokens;
+  private lastRefill;
+  private readonly maxTokens;
+  private readonly refillRate;
+  constructor(config: RateLimitConfig$1);
+  /**
+   * Check if a request is allowed without consuming tokens.
+   */
+  canProceed(): boolean;
+  /**
+   * Attempt to acquire tokens for a request.
+   * Returns true if allowed, false if rate limited.
+   */
+  tryAcquire(tokens?: number): boolean;
+  /**
+   * Wait until tokens are available, then acquire.
+   */
+  acquire(tokens?: number): Promise<void>;
+  /**
+   * Get time until next token is available (in milliseconds).
+   */
+  getWaitTime(): number;
+  /**
+   * Refill tokens based on elapsed time.
+   */
+  private refill;
+}
+/**
+ * Semaphore for limiting concurrent operations.
+ */
+declare class Semaphore {
+  private permits;
+  private waiting;
+  constructor(permits: number);
+  /**
+   * Acquire a permit, waiting if necessary.
+   */
+  acquire(): Promise<void>;
+  /**
+   * Release a permit.
+   */
+  release(): void;
+  /**
+   * Execute function with semaphore protection.
+   */
+  execute<T>(fn: () => Promise<T>): Promise<T>;
+}
+/**
+ * Execute a function with all resilience features.
+ *
+ * @param fn - The async function to execute with resilience
+ * @param config - Configuration for retry and timeout behavior
+ * @param state - Pre-instantiated resilience primitives for stateful features.
+ *   Circuit breaker, rate limiter, and semaphore must be instantiated by the caller
+ *   and passed via state to enable those features. This allows sharing state across
+ *   multiple calls for proper circuit breaker tracking and rate limiting.
+ *   The config parameter is only used for retry and timeout settings.
+ * @param callbacks - Optional callbacks for retry events
+ */
+declare function withResilience<T>(fn: (signal: AbortSignal) => Promise<T>, config?: ResilienceConfig$1, state?: ResilienceState$1, callbacks?: {
+  onRetry?: (attempt: number, error: Error, delayMs: number) => void;
+}): Promise<{
+  result: T;
+  attempts: number;
+}>;
+//#endregion
+//#region src/common/http-base.d.ts
+/**
+ * Base configuration for HTTP providers.
+ */
+interface BaseHttpConfig<TError = unknown> {
+  /** Base URL for the API endpoint */
+  baseUrl: string;
+  /** Model identifier */
+  model: string;
+  /** Additional headers */
+  headers?: Record<string, string>;
+  /** Extract error message from failed response */
+  errorMapper?: (response: TError) => string;
+  /** Security options */
+  requireHttps?: boolean;
+  allowPrivate?: boolean;
+  resolveDns?: boolean;
+  allowRedirects?: boolean;
+  /** Resilience options */
+  resilience?: ResilienceConfig$1;
+}
+/**
+ * Fetch request options for base provider.
+ */
+interface FetchOptions {
+  method?: 'GET' | 'POST';
+  body?: unknown;
+  headers?: Record<string, string>;
+  signal?: AbortSignal;
+}
+/**
+ * Result of a fetch request.
+ */
+interface FetchResult<T> {
+  data: T;
+  status: number;
+  headers: Headers;
+}
+/**
+ * Base HTTP provider with shared security and resilience.
+ */
+declare abstract class BaseHttpProvider<TError = unknown> {
+  protected readonly baseUrl: string;
+  protected readonly model: string;
+  protected readonly headers: Record<string, string>;
+  protected readonly errorMapper?: (response: TError) => string;
+  protected readonly requireHttps: boolean;
+  protected readonly allowPrivate: boolean;
+  protected readonly resolveDns: boolean;
+  protected readonly allowRedirects: boolean;
+  protected readonly timeoutMs: number;
+  protected readonly retryConfig?: RetryConfig$1;
+  protected readonly concurrency: number;
+  private circuitBreaker?;
+  private rateLimiter?;
+  private semaphore?;
+  constructor(config: BaseHttpConfig<TError>);
+  /**
+   * Get the current resilience state for persistence across calls.
+   */
+  getResilienceState(): ResilienceState$1;
+  /**
+   * Make an HTTP request with security and resilience.
+   */
+  protected fetch<T>(url: string, options?: FetchOptions): Promise<FetchResult<T>>;
+}
+//#endregion
+//#region src/embeddings/providers/http.d.ts
+/**
+ * HTTP embedding provider configuration.
+ */
+interface HttpEmbeddingConfig<TRequest = unknown, TResponse = unknown, TError = unknown> extends BaseHttpConfig<TError> {
+  /**
+   * Build request body from input texts.
+   * @default { input: texts, model }
+   */
+  requestBuilder?: (texts: string[], model: string) => TRequest;
+  /**
+   * Extract embeddings array from response.
+   * @default (res) => res.data.map(d => d.embedding)
+   */
+  responseMapper?: (response: TResponse) => number[][];
+}
+/**
+ * HTTP-based embedding provider.
+ * Works with any REST API using native fetch.
+ */
+declare class HttpEmbeddingProvider<TRequest = unknown, TResponse = unknown, TError = unknown> extends BaseHttpProvider<TError> implements EmbeddingProvider {
+  readonly name = "http-embedding";
+  private readonly requestBuilder;
+  private readonly responseMapper;
+  constructor(config: HttpEmbeddingConfig<TRequest, TResponse, TError>);
+  /**
+   * Generate embeddings for one or more texts.
+   */
+  embed(texts: string[], options: EmbedRequest): Promise<EmbedResponse>;
+}
+/**
+ * Create a generic HTTP embedding provider.
+ */
+declare function createHttpEmbedding<TRequest = unknown, TResponse = unknown, TError = unknown>(config: HttpEmbeddingConfig<TRequest, TResponse, TError>): EmbeddingProvider;
+//#endregion
+//#region src/embeddings/types.d.ts
+/**
+ * Embedding provider configuration - discriminated union for type safety.
+ *
+ * Use preset factory functions to create providers:
+ * - `createOpenAIEmbedding()` - OpenAI API
+ * - `createAzureEmbedding()` - Azure OpenAI
+ * - `createOllamaEmbedding()` - Local Ollama
+ * - `createHuggingFaceEmbedding()` - HuggingFace Inference API
+ * - `createCohereEmbedding()` - Cohere API
+ * - `createTransformersEmbedding()` - Local Transformers.js
+ *
+ * @example Using a preset
+ * ```ts
+ * import { createOpenAIEmbedding } from 'scrapex/embeddings';
+ *
+ * const result = await scrape(url, {
+ *   embeddings: {
+ *     provider: { type: 'custom', provider: createOpenAIEmbedding() },
+ *   },
+ * });
+ * ```
+ *
+ * @example Using inline HTTP config
+ * ```ts
+ * const result = await scrape(url, {
+ *   embeddings: {
+ *     provider: {
+ *       type: 'http',
+ *       config: {
+ *         baseUrl: 'https://api.example.com/embed',
+ *         model: 'custom-model',
+ *         headers: { Authorization: 'Bearer ...' },
+ *       },
+ *     },
+ *   },
+ * });
+ * ```
+ */
+type EmbeddingProviderConfig = {
+  type: 'http';
+  config: HttpEmbeddingConfig;
+} | {
+  type: 'custom';
+  provider: EmbeddingProvider;
+};
+/**
+ * Embedding provider interface - mirrors LLMProvider pattern.
+ */
+interface EmbeddingProvider {
+  readonly name: string;
+  /**
+   * Generate embeddings for one or more texts.
+   */
+  embed(texts: string[], options: EmbedRequest): Promise<EmbedResponse>;
+}
+interface EmbedRequest {
+  /** Model to use. If undefined, provider uses its configured default. */
+  model?: string;
+  dimensions?: number;
+  signal?: AbortSignal;
+}
+interface EmbedResponse {
+  embeddings: number[][];
+  usage?: {
+    promptTokens: number;
+    totalTokens: number;
+  };
+}
+type EmbeddingInputType = 'textContent' | 'title+summary' | 'custom';
+interface EmbeddingInputConfig {
+  /**
+   * Predefined input source. Ignored if `transform` is provided.
+   * @default 'textContent'
+   */
+  type?: EmbeddingInputType;
+  /**
+   * Custom function to generate input text from scraped data.
+   * Enables dynamic construction (e.g., "Combine price + title").
+   */
+  transform?: (data: Partial<ScrapedData>) => string;
+  /**
+   * Static custom input string. Used when type is 'custom'.
+   */
+  customText?: string;
+}
+interface ChunkingConfig {
+  /**
+   * Target chunk size in tokens.
+   * @default 500
+   */
+  size?: number;
+  /**
+   * Overlap between chunks in tokens.
+   * @default 50
+   */
+  overlap?: number;
+  /**
+   * Token counting strategy.
+   * - 'heuristic': chars / 4 (fast, approximate)
+   * - 'tiktoken': accurate for OpenAI models (lazy-loaded)
+   * - function: custom tokenizer
+   */
+  tokenizer?: 'heuristic' | 'tiktoken' | ((text: string) => number);
+  /**
+   * Hard cap on input length (characters) to prevent memory exhaustion.
+   * @default 100000 (100KB)
+   */
+  maxInputLength?: number;
+}
+type EmbeddingAggregation = 'average' | 'max' | 'first' | 'all';
+interface OutputConfig {
+  /**
+   * Aggregation strategy for chunk vectors.
+   * @default 'average'
+   */
+  aggregation?: EmbeddingAggregation;
+  /** Model-specific dimension override */
+  dimensions?: number;
+}
+interface PiiRedactionConfig {
+  /** Redact email addresses */
+  email?: boolean;
+  /** Redact phone numbers */
+  phone?: boolean;
+  /** Redact credit card numbers */
+  creditCard?: boolean;
+  /** Redact SSN patterns */
+  ssn?: boolean;
+  /** Redact IP addresses */
+  ipAddress?: boolean;
+  /** Additional patterns to redact */
+  customPatterns?: RegExp[];
+}
+interface SafetyConfig {
+  /**
+   * PII redaction patterns to apply before embedding.
+   * Critical for GDPR/CCPA compliance with third-party APIs.
+   */
+  piiRedaction?: PiiRedactionConfig;
+  /**
+   * Minimum text length to proceed with embedding.
+   * Skips with reason if below threshold.
+   */
+  minTextLength?: number;
+  /**
+   * Maximum tokens per API request to prevent billing DoS.
+   * @default 8192
+   */
+  maxTokens?: number;
+  /**
+   * Explicitly opt-in to receive sensitive data in callbacks.
+   * When false (default), onChunk receives redacted content.
+   */
+  allowSensitiveCallbacks?: boolean;
+}
+interface EmbeddingCacheConfig {
+  /** Cache implementation */
+  store?: EmbeddingCache;
+  /** Time-to-live in milliseconds */
+  ttlMs?: number;
+  /** Maximum entries for in-memory cache */
+  maxEntries?: number;
+  /**
+   * Extra salt to disambiguate cache keys for custom providers/transforms.
+   */
+  cacheKeySalt?: string;
+}
+/**
+ * Content-addressable cache interface for embeddings.
+ * Keys are based on content hash, not URL.
+ */
+interface EmbeddingCache {
+  get(key: string): Promise<EmbeddingResult | undefined>;
+  set(key: string, value: EmbeddingResult, options?: {
+    ttlMs?: number;
+  }): Promise<void>;
+  delete(key: string): Promise<boolean>;
+  clear(): Promise<void>;
+}
+interface RetryConfig {
+  /**
+   * Maximum retry attempts.
+   * @default 3
+   */
+  maxAttempts?: number;
+  /**
+   * Initial backoff delay in milliseconds.
+   * @default 1000
+   */
+  backoffMs?: number;
+  /**
+   * Backoff multiplier for exponential delay.
+   * @default 2
+   */
+  backoffMultiplier?: number;
+}
+interface CircuitBreakerConfig {
+  /**
+   * Number of failures before opening the circuit.
+   * @default 5
+   */
+  failureThreshold?: number;
+  /**
+   * Time to wait before attempting to close the circuit.
+   * @default 30000
+   */
+  resetTimeoutMs?: number;
+}
+interface RateLimitConfig {
+  /** Maximum requests per minute */
+  requestsPerMinute?: number;
+  /** Maximum tokens per minute */
+  tokensPerMinute?: number;
+}
+interface ResilienceState {
+  circuitBreaker?: {
+    isOpen(): boolean;
+    recordSuccess(): void;
+    recordFailure(): void;
+    getState?(): CircuitState;
+  };
+  rateLimiter?: {
+    acquire(): Promise<void>;
+  };
+  semaphore?: {
+    execute<T>(fn: () => Promise<T>): Promise<T>;
+  };
+}
+interface ResilienceConfig {
+  /** Retry configuration for transient failures */
+  retry?: RetryConfig;
+  /** Circuit breaker to prevent cascade failures */
+  circuitBreaker?: CircuitBreakerConfig;
+  /** Rate limiting per provider */
+  rateLimit?: RateLimitConfig;
+  /**
+   * Optional shared state for circuit breaker and rate limiter.
+   * Use to persist state across multiple calls.
+   */
+  state?: ResilienceState;
+  /**
+   * Request timeout in milliseconds.
+   * @default 30000
+   */
+  timeoutMs?: number;
+  /**
+   * Concurrent chunk processing.
+   * @default 1
+   */
+  concurrency?: number;
+}
+interface EmbeddingOptions {
+  /** Embedding provider configuration */
+  provider: EmbeddingProviderConfig;
+  /** Model identifier (overrides provider default) */
+  model?: string;
+  /** Input text configuration */
+  input?: EmbeddingInputConfig;
+  /** Chunking and tokenization settings */
+  chunking?: ChunkingConfig;
+  /** Output format and aggregation */
+  output?: OutputConfig;
+  /** Safety and compliance settings */
+  safety?: SafetyConfig;
+  /** Caching configuration */
+  cache?: EmbeddingCacheConfig;
+  /** Resilience and rate limiting */
+  resilience?: ResilienceConfig;
+  /**
+   * Callback for each chunk (receives redacted content by default).
+   */
+  onChunk?: (chunk: Readonly<string>, embedding: Readonly<number[]>) => void;
+  /**
+   * Metrics callback for observability.
+   */
+  onMetrics?: (metrics: EmbeddingMetrics) => void;
+}
+interface EmbeddingSource {
+  /** Model used for embedding (may be undefined for custom providers) */
+  model?: string;
+  /** Number of chunks processed */
+  chunks: number;
+  /** Total tokens processed */
+  tokens: number;
+  /** Content checksum for cache validation */
+  checksum: string;
+  /** Whether result was from cache */
+  cached: boolean;
+  /** Total latency in milliseconds */
+  latencyMs: number;
+}
+/**
+ * Successful embedding result with single aggregated vector.
+ */
+interface EmbeddingSuccessSingle {
+  status: 'success';
+  aggregation: 'average' | 'max' | 'first';
+  vector: number[];
+  source: EmbeddingSource;
+}
+/**
+ * Successful embedding result with all chunk vectors.
+ */
+interface EmbeddingSuccessMultiple {
+  status: 'success';
+  aggregation: 'all';
+  vectors: number[][];
+  source: EmbeddingSource;
+}
+/**
+ * Skipped embedding with reason.
+ */
+interface EmbeddingSkipped {
+  status: 'skipped';
+  reason: string;
+  source: Partial<EmbeddingSource>;
+}
+/**
+ * Embedding result - discriminated union for type safety.
+ * Use `result.status` to narrow the type.
+ */
+type EmbeddingResult = EmbeddingSuccessSingle | EmbeddingSuccessMultiple | EmbeddingSkipped;
+interface EmbeddingMetrics {
+  /** Provider name */
+  provider: string;
+  /** Model used (may be undefined for custom providers) */
+  model?: string;
+  /** Input tokens processed */
+  inputTokens: number;
+  /** Output embedding dimensions */
+  outputDimensions: number;
+  /** Number of chunks processed */
+  chunks: number;
+  /** Total latency in milliseconds */
+  latencyMs: number;
+  /** Whether result was from cache */
+  cached: boolean;
+  /** Number of retry attempts */
+  retries: number;
+  /** Whether PII was redacted */
+  piiRedacted: boolean;
+}
+/**
+ * Internal chunk representation during processing.
+ */
+interface TextChunk {
+  /** Chunk text content */
+  text: string;
+  /** Start position in original text */
+  startIndex: number;
+  /** End position in original text */
+  endIndex: number;
+  /** Estimated token count */
+  tokens: number;
+}
+/**
+ * Circuit breaker state.
+ */
+type CircuitState = 'closed' | 'open' | 'half-open';
+/**
+ * Circuit breaker internal state tracking.
+ */
+interface CircuitBreakerState {
+  state: CircuitState;
+  failures: number;
+  lastFailureTime?: number;
+  nextAttemptTime?: number;
+}
+//#endregion
+export { FetchResult$1 as $, HttpEmbeddingProvider as A, withResilience as B, RateLimitConfig as C, SafetyConfig as D, RetryConfig as E, CircuitOpenError as F, EnhancementType as G, withTimeout as H, RateLimiter as I, ExtractionContext as J, ExtractedEntities as K, Semaphore as L, BaseHttpConfig as M, BaseHttpProvider as N, TextChunk as O, CircuitBreaker as P, FetchOptions$1 as Q, createTimeoutSignal as R, PiiRedactionConfig as S, ResilienceState as T, CompletionOptions as U, withRetry as V, ContentType as W, ExtractionSchemaType as X, ExtractionSchema as Y, Extractor as Z, EmbeddingSkipped as _, EmbedRequest as a, EmbeddingSuccessSingle as b, EmbeddingCache as c, EmbeddingInputType as d, Fetcher as et, EmbeddingMetrics as f, EmbeddingResult as g, EmbeddingProviderConfig as h, CircuitState as i, createHttpEmbedding as j, HttpEmbeddingConfig as k, EmbeddingCacheConfig as l, EmbeddingProvider as m, CircuitBreakerConfig as n, ScrapeOptions as nt, EmbedResponse as o, EmbeddingOptions as p, ExtractedLink as q, CircuitBreakerState as r, ScrapedData as rt, EmbeddingAggregation as s, ChunkingConfig as t, LLMProvider as tt, EmbeddingInputConfig as u, EmbeddingSource as v, ResilienceConfig as w, OutputConfig as x, EmbeddingSuccessMultiple as y, isRetryableError as z };
+//# sourceMappingURL=types-BOcHQU9s.d.mts.map