scrapex 1.0.0-beta.4 → 1.0.0

package/README.md

@@ -2,7 +2,7 @@
 
 Modern web scraper with LLM-enhanced extraction, extensible pipeline, and pluggable parsers.
 
-> **Beta Release**: v1.0.0 is currently in beta. The API is stable but minor changes may occur before the stable release.
+> **Release**: v1.0.0 (API stable; breaking changes documented below).
 
 ## Features
 
@@ -13,13 +13,14 @@ Modern web scraper with LLM-enhanced extraction, extensible pipeline, and plugga
 - **Smart Extraction** - Uses Mozilla Readability for content, Cheerio for metadata
 - **Markdown Parsing** - Parse markdown content, awesome lists, and GitHub repos
 - **RSS/Atom Feeds** - Parse RSS 2.0, RSS 1.0 (RDF), and Atom feeds with pagination support
+- **Content Normalization** - Clean, embedding-ready text with boilerplate removal
 - **TypeScript First** - Full type safety with comprehensive type exports
 - **Dual Format** - ESM and CommonJS builds
 
 ## Installation
 
 ```bash
-npm install scrapex@beta
+npm install scrapex
 ```
 
 ### Optional Peer Dependencies
@@ -29,6 +30,9 @@ npm install scrapex@beta
 npm install openai           # OpenAI/Ollama/LM Studio
 npm install @anthropic-ai/sdk  # Anthropic Claude
 
+# For local embeddings (zero API cost)
+npm install @huggingface/transformers onnxruntime-node
+
 # For JavaScript-rendered pages
 npm install puppeteer
 ```
@@ -60,6 +64,10 @@ const result = await scrape('https://example.com', {
   userAgent: 'MyBot/1.0',
   extractContent: true,
   maxContentLength: 50000,
+  normalize: {
+    mode: 'full',
+    removeBoilerplate: true,
+  },
   respectRobots: false,
 });
 ```
@@ -117,6 +125,17 @@ interface ScrapedData {
   entities?: ExtractedEntities;
   extracted?: Record<string, unknown>;
 
+  // Custom extractor results
+  custom?: Record<string, unknown>;
+
+  // Embeddings (when enabled)
+  embeddings?: EmbeddingResult;
+
+  // Normalized text (when enabled)
+  normalizedText?: string;
+  normalizationMeta?: NormalizationMeta;
+  normalizedBlocks?: ContentBlock[];
+
   // Meta
   scrapedAt: string;
   scrapeTimeMs: number;
@@ -173,7 +192,84 @@ Features include:
 
 See the [Embeddings Guide](https://scrapex.dev/guides/embeddings) for full documentation.
 
-## Breaking Changes (Beta)
+## Content Normalization
+
+Clean, embedding-ready text with boilerplate removal and block classification:
+
+```typescript
+const result = await scrape(url, {
+  normalize: {
+    mode: 'full',            // or 'summary' for score-ranked blocks
+    removeBoilerplate: true, // filter nav, footer, promos
+    maxChars: 5000,          // truncate at sentence boundary
+  },
+});
+
+console.log(result.normalizedText);    // Clean text ready for embedding
+console.log(result.normalizationMeta); // { charCount, tokenEstimate, hash, ... }
+```
+
+### Standalone Normalization
+
+Use normalization without scraping:
+
+```typescript
+import { parseBlocks, normalizeText } from 'scrapex';
+import { load } from 'cheerio';
+
+const $ = load(html);
+const blocks = parseBlocks($);
+const result = await normalizeText(blocks, { mode: 'full' });
+
+console.log(result.text);
+console.log(result.meta);
+```
+
+### Custom Classifiers
+
+Filter blocks with custom logic:
+
+```typescript
+import { combineClassifiers, defaultBlockClassifier } from 'scrapex';
+
+const myClassifier = combineClassifiers(
+  defaultBlockClassifier,
+  (block) => {
+    if (block.text.includes('Advertisement')) {
+      return { accept: false, label: 'ad' };
+    }
+    return { accept: true };
+  }
+);
+
+const result = await scrape(url, {
+  normalize: { blockClassifier: myClassifier },
+});
+```
+
+### Normalization Options
+
+```typescript
+interface NormalizeOptions {
+  mode?: 'summary' | 'full';       // Output mode (default: 'full')
+  maxChars?: number;               // Max characters in output
+  minChars?: number;               // Minimum characters required
+  maxBlocks?: number;              // Max blocks to process (default: 2000)
+  truncate?: 'sentence' | 'word' | 'char';  // Truncation strategy
+  dropSelectors?: string[];        // Selectors to drop before parsing
+  removeBoilerplate?: boolean;     // Filter nav/footer/promos (default: true)
+  decodeEntities?: boolean;        // Decode HTML entities (default: true)
+  normalizeUnicode?: boolean;      // Normalize Unicode to NFC (default: true)
+  preserveLineBreaks?: boolean;    // Preserve paragraph breaks (default: true)
+  stripLinks?: boolean;            // Strip Markdown links (default: true)
+  includeHtml?: boolean;           // Include raw HTML in blocks (default: false)
+  languageHint?: string;           // Language hint for metadata
+  blockClassifier?: ContentBlockClassifier; // Custom classifier
+  debug?: boolean;                 // Include blocks in output
+}
+```
+
+## Breaking Changes
 
 - LLM provider classes (e.g., `AnthropicProvider`) were removed. Use preset factories like
   `createOpenAI`, `createAnthropic`, `createOllama`, and `createLMStudio` instead.
@@ -409,6 +505,27 @@ const markdown = feedToMarkdown(result.data, { maxItems: 10 });
 const text = feedToText(result.data);
 ```
 
+### Normalizing Feed Items
+
+Convert feed item content to clean, embedding-ready text:
+
+```typescript
+import { RSSParser, normalizeFeedItem } from 'scrapex';
+
+const parser = new RSSParser();
+const result = parser.parse(feedXml);
+
+for (const item of result.data.items) {
+  const normalized = await normalizeFeedItem(item, {
+    mode: 'full',
+    removeBoilerplate: true,
+  });
+
+  console.log(normalized.text); // Clean text from content/description
+  console.log(normalized.meta); // { charCount, tokenEstimate, hash, ... }
+}
+```
+
 ### Custom Fields (Podcast/Media)
 
 Extract custom namespace fields like iTunes podcast tags:
@@ -429,6 +546,25 @@ console.log(item.customFields?.duration);  // '10:00'
 console.log(item.customFields?.explicit);  // 'no'
 ```
 
+#### Attribute Extraction
+
+Use `selector@attr` syntax to extract XML attribute values:
+
+```typescript
+const parser = new RSSParser({
+  customFields: {
+    // Extract url attribute from media:thumbnail element
+    thumbnail: 'media\\:thumbnail@url',
+    // Extract url attribute from media:content element
+    mediaUrl: 'media\\:content@url',
+  },
+});
+
+const result = parser.parse(mediaRssFeed);
+console.log(result.data.items[0]?.customFields?.thumbnail);
+// => "https://example.com/images/thumbnail.jpg"
+```
+
 ### Security
 
 The RSS parser enforces strict URL security:
@@ -536,6 +672,8 @@ interface ScrapeOptions {
   llm?: LLMProvider;             // LLM provider
   enhance?: EnhancementType[];   // LLM enhancements
   extract?: ExtractionSchema;    // Structured extraction
+  embeddings?: EmbeddingOptions; // Vector embeddings
+  normalize?: NormalizeOptions;  // Content normalization
 }
 ```
 
@@ -554,6 +692,29 @@ type EnhancementType =
 - Node.js 20+
 - TypeScript 5.0+ (for type imports)
 
+## Get Help
+
+- [Documentation](https://scrapex.dev) - Guides and API reference
+- [GitHub Issues](https://github.com/developer-rakeshpaul/scrapex/issues) - Bug reports and feature requests
+- [GitHub Discussions](https://github.com/developer-rakeshpaul/scrapex/discussions) - Questions and ideas
+- [Stack Overflow](https://stackoverflow.com/questions/tagged/scrapex) - Community Q&A
+
+## Contributing
+
+Contributions are welcome! Whether it's bug reports, feature requests, or pull requests.
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on:
+- Reporting bugs
+- Suggesting features
+- Submitting pull requests
+- Development setup
+
+## Support
+
+If you find scrapex useful, consider supporting its development:
+
+[![Buy Me A Coffee](https://img.shields.io/badge/Buy%20Me%20A%20Coffee-support-yellow?style=flat&logo=buy-me-a-coffee)](https://buymeacoffee.com/binaryroute)
+
 ## License
 
 MIT

package/dist/embeddings/index.cjs

@@ -1,5 +1,5 @@
-const require_http_base = require('../http-base-CHLf-Tco.cjs');
-const require_embeddings = require('../embeddings-DHVu52VF.cjs');
+const require_http_base = require('../http-base-B5EXbNwR.cjs');
+const require_embeddings = require('../embeddings-BzRSRX9t.cjs');
 
 exports.CircuitBreaker = require_http_base.CircuitBreaker;
 exports.CircuitOpenError = require_http_base.CircuitOpenError;

package/dist/embeddings/index.d.cts

@@ -1,3 +1,3 @@
-import { A as HttpEmbeddingProvider, B as withResilience, C as RateLimitConfig, D as SafetyConfig, E as RetryConfig, F as CircuitOpenError, H as withTimeout, I as RateLimiter, L as Semaphore, O as TextChunk, P as CircuitBreaker, R as createTimeoutSignal, S as PiiRedactionConfig, T as ResilienceState, V as withRetry, _ as EmbeddingSkipped, a as EmbedRequest, b as EmbeddingSuccessSingle, c as EmbeddingCache, d as EmbeddingInputType, f as EmbeddingMetrics, g as EmbeddingResult, h as EmbeddingProviderConfig, i as CircuitState, j as createHttpEmbedding, k as HttpEmbeddingConfig, l as EmbeddingCacheConfig, m as EmbeddingProvider, n as CircuitBreakerConfig, o as EmbedResponse, p as EmbeddingOptions, r as CircuitBreakerState, s as EmbeddingAggregation, t as ChunkingConfig, u as EmbeddingInputConfig, v as EmbeddingSource, w as ResilienceConfig, x as OutputConfig, y as EmbeddingSuccessMultiple, z as isRetryableError } from "../types-BYsD9u71.cjs";
-import { A as createNoOpCache, B as euclideanDistance, C as createTokenizer, D as needsChunking, E as heuristicTokenCount, F as validateCachedResult, H as normalizeVector, I as AggregationResult, L as aggregateVectors, M as generateChecksum, N as getDefaultCache, O as CacheStats, P as resetDefaultCache, R as cosineSimilarity, S as chunkText, T as getChunkingStats, V as getDimensions, _ as generateEmbeddings, a as createEmbeddingProvider, b as selectInput, c as createAzureEmbedding, d as createOllamaEmbedding, f as createOpenAIEmbedding, g as embedScrapedData, h as embed, i as redactPii, j as generateCacheKey, k as InMemoryEmbeddingCache, l as createCohereEmbedding, m as getDefaultModel, n as containsPii, o as isEmbeddingProvider, p as createTransformersEmbedding, r as createPiiRedactor, s as TRANSFORMERS_MODELS, t as RedactionResult, u as createHuggingFaceEmbedding, v as InputValidation, w as estimateTokens, x as validateInput, y as previewInput, z as dotProduct } from "../index-CliKPY5N.cjs";
+import { A as HttpEmbeddingProvider, B as withResilience, C as RateLimitConfig, D as SafetyConfig, E as RetryConfig, F as CircuitOpenError, H as withTimeout, I as RateLimiter, L as Semaphore, O as TextChunk, P as CircuitBreaker, R as createTimeoutSignal, S as PiiRedactionConfig, T as ResilienceState, V as withRetry, _ as EmbeddingSkipped, a as EmbedRequest, b as EmbeddingSuccessSingle, c as EmbeddingCache, d as EmbeddingInputType, f as EmbeddingMetrics, g as EmbeddingResult, h as EmbeddingProviderConfig, i as CircuitState, j as createHttpEmbedding, k as HttpEmbeddingConfig, l as EmbeddingCacheConfig, m as EmbeddingProvider, n as CircuitBreakerConfig, o as EmbedResponse, p as EmbeddingOptions, r as CircuitBreakerState, s as EmbeddingAggregation, t as ChunkingConfig, u as EmbeddingInputConfig, v as EmbeddingSource, w as ResilienceConfig, x as OutputConfig, y as EmbeddingSuccessMultiple, z as isRetryableError } from "../types--8EgNAtb.cjs";
+import { A as createNoOpCache, B as euclideanDistance, C as createTokenizer, D as needsChunking, E as heuristicTokenCount, F as validateCachedResult, H as normalizeVector, I as AggregationResult, L as aggregateVectors, M as generateChecksum, N as getDefaultCache, O as CacheStats, P as resetDefaultCache, R as cosineSimilarity, S as chunkText, T as getChunkingStats, V as getDimensions, _ as generateEmbeddings, a as createEmbeddingProvider, b as selectInput, c as createAzureEmbedding, d as createOllamaEmbedding, f as createOpenAIEmbedding, g as embedScrapedData, h as embed, i as redactPii, j as generateCacheKey, k as InMemoryEmbeddingCache, l as createCohereEmbedding, m as getDefaultModel, n as containsPii, o as isEmbeddingProvider, p as createTransformersEmbedding, r as createPiiRedactor, s as TRANSFORMERS_MODELS, t as RedactionResult, u as createHuggingFaceEmbedding, v as InputValidation, w as estimateTokens, x as validateInput, y as previewInput, z as dotProduct } from "../index-BKGgfF5C.cjs";
 export { AggregationResult, CacheStats, ChunkingConfig, CircuitBreaker, CircuitBreakerConfig, CircuitBreakerState, CircuitOpenError, CircuitState, EmbedRequest, EmbedResponse, EmbeddingAggregation, EmbeddingCache, EmbeddingCacheConfig, EmbeddingInputConfig, EmbeddingInputType, EmbeddingMetrics, EmbeddingOptions, EmbeddingProvider, EmbeddingProviderConfig, EmbeddingResult, EmbeddingSkipped, EmbeddingSource, EmbeddingSuccessMultiple, EmbeddingSuccessSingle, HttpEmbeddingConfig, HttpEmbeddingProvider, InMemoryEmbeddingCache, InputValidation, OutputConfig, PiiRedactionConfig, RateLimitConfig, RateLimiter, RedactionResult, ResilienceConfig, ResilienceState, RetryConfig, SafetyConfig, Semaphore, TRANSFORMERS_MODELS, TextChunk, aggregateVectors, chunkText, containsPii, cosineSimilarity, createAzureEmbedding, createCohereEmbedding, createEmbeddingProvider, createHttpEmbedding, createHuggingFaceEmbedding, createNoOpCache, createOllamaEmbedding, createOpenAIEmbedding, createPiiRedactor, createTimeoutSignal, createTokenizer, createTransformersEmbedding, dotProduct, embed, embedScrapedData, estimateTokens, euclideanDistance, generateCacheKey, generateChecksum, generateEmbeddings, getChunkingStats, getDefaultCache, getDefaultModel, getDimensions, heuristicTokenCount, isEmbeddingProvider, isRetryableError, needsChunking, normalizeVector, previewInput, redactPii, resetDefaultCache, selectInput, validateCachedResult, validateInput, withResilience, withRetry, withTimeout };

package/dist/embeddings/index.d.mts

@@ -1,3 +1,3 @@
-import { A as HttpEmbeddingProvider, B as withResilience, C as RateLimitConfig, D as SafetyConfig, E as RetryConfig, F as CircuitOpenError, H as withTimeout, I as RateLimiter, L as Semaphore, O as TextChunk, P as CircuitBreaker, R as createTimeoutSignal, S as PiiRedactionConfig, T as ResilienceState, V as withRetry, _ as EmbeddingSkipped, a as EmbedRequest, b as EmbeddingSuccessSingle, c as EmbeddingCache, d as EmbeddingInputType, f as EmbeddingMetrics, g as EmbeddingResult, h as EmbeddingProviderConfig, i as CircuitState, j as createHttpEmbedding, k as HttpEmbeddingConfig, l as EmbeddingCacheConfig, m as EmbeddingProvider, n as CircuitBreakerConfig, o as EmbedResponse, p as EmbeddingOptions, r as CircuitBreakerState, s as EmbeddingAggregation, t as ChunkingConfig, u as EmbeddingInputConfig, v as EmbeddingSource, w as ResilienceConfig, x as OutputConfig, y as EmbeddingSuccessMultiple, z as isRetryableError } from "../types-BdlvuOtp.mjs";
-import { A as createNoOpCache, B as euclideanDistance, C as createTokenizer, D as needsChunking, E as heuristicTokenCount, F as validateCachedResult, H as normalizeVector, I as AggregationResult, L as aggregateVectors, M as generateChecksum, N as getDefaultCache, O as CacheStats, P as resetDefaultCache, R as cosineSimilarity, S as chunkText, T as getChunkingStats, V as getDimensions, _ as generateEmbeddings, a as createEmbeddingProvider, b as selectInput, c as createAzureEmbedding, d as createOllamaEmbedding, f as createOpenAIEmbedding, g as embedScrapedData, h as embed, i as redactPii, j as generateCacheKey, k as InMemoryEmbeddingCache, l as createCohereEmbedding, m as getDefaultModel, n as containsPii, o as isEmbeddingProvider, p as createTransformersEmbedding, r as createPiiRedactor, s as TRANSFORMERS_MODELS, t as RedactionResult, u as createHuggingFaceEmbedding, v as InputValidation, w as estimateTokens, x as validateInput, y as previewInput, z as dotProduct } from "../index-REk4CMxV.mjs";
+import { A as HttpEmbeddingProvider, B as withResilience, C as RateLimitConfig, D as SafetyConfig, E as RetryConfig, F as CircuitOpenError, H as withTimeout, I as RateLimiter, L as Semaphore, O as TextChunk, P as CircuitBreaker, R as createTimeoutSignal, S as PiiRedactionConfig, T as ResilienceState, V as withRetry, _ as EmbeddingSkipped, a as EmbedRequest, b as EmbeddingSuccessSingle, c as EmbeddingCache, d as EmbeddingInputType, f as EmbeddingMetrics, g as EmbeddingResult, h as EmbeddingProviderConfig, i as CircuitState, j as createHttpEmbedding, k as HttpEmbeddingConfig, l as EmbeddingCacheConfig, m as EmbeddingProvider, n as CircuitBreakerConfig, o as EmbedResponse, p as EmbeddingOptions, r as CircuitBreakerState, s as EmbeddingAggregation, t as ChunkingConfig, u as EmbeddingInputConfig, v as EmbeddingSource, w as ResilienceConfig, x as OutputConfig, y as EmbeddingSuccessMultiple, z as isRetryableError } from "../types-dyXBh3Xy.mjs";
+import { A as createNoOpCache, B as euclideanDistance, C as createTokenizer, D as needsChunking, E as heuristicTokenCount, F as validateCachedResult, H as normalizeVector, I as AggregationResult, L as aggregateVectors, M as generateChecksum, N as getDefaultCache, O as CacheStats, P as resetDefaultCache, R as cosineSimilarity, S as chunkText, T as getChunkingStats, V as getDimensions, _ as generateEmbeddings, a as createEmbeddingProvider, b as selectInput, c as createAzureEmbedding, d as createOllamaEmbedding, f as createOpenAIEmbedding, g as embedScrapedData, h as embed, i as redactPii, j as generateCacheKey, k as InMemoryEmbeddingCache, l as createCohereEmbedding, m as getDefaultModel, n as containsPii, o as isEmbeddingProvider, p as createTransformersEmbedding, r as createPiiRedactor, s as TRANSFORMERS_MODELS, t as RedactionResult, u as createHuggingFaceEmbedding, v as InputValidation, w as estimateTokens, x as validateInput, y as previewInput, z as dotProduct } from "../index-DoA7xuDF.mjs";
 export { AggregationResult, CacheStats, ChunkingConfig, CircuitBreaker, CircuitBreakerConfig, CircuitBreakerState, CircuitOpenError, CircuitState, EmbedRequest, EmbedResponse, EmbeddingAggregation, EmbeddingCache, EmbeddingCacheConfig, EmbeddingInputConfig, EmbeddingInputType, EmbeddingMetrics, EmbeddingOptions, EmbeddingProvider, EmbeddingProviderConfig, EmbeddingResult, EmbeddingSkipped, EmbeddingSource, EmbeddingSuccessMultiple, EmbeddingSuccessSingle, HttpEmbeddingConfig, HttpEmbeddingProvider, InMemoryEmbeddingCache, InputValidation, OutputConfig, PiiRedactionConfig, RateLimitConfig, RateLimiter, RedactionResult, ResilienceConfig, ResilienceState, RetryConfig, SafetyConfig, Semaphore, TRANSFORMERS_MODELS, TextChunk, aggregateVectors, chunkText, containsPii, cosineSimilarity, createAzureEmbedding, createCohereEmbedding, createEmbeddingProvider, createHttpEmbedding, createHuggingFaceEmbedding, createNoOpCache, createOllamaEmbedding, createOpenAIEmbedding, createPiiRedactor, createTimeoutSignal, createTokenizer, createTransformersEmbedding, dotProduct, embed, embedScrapedData, estimateTokens, euclideanDistance, generateCacheKey, generateChecksum, generateEmbeddings, getChunkingStats, getDefaultCache, getDefaultModel, getDimensions, heuristicTokenCount, isEmbeddingProvider, isRetryableError, needsChunking, normalizeVector, previewInput, redactPii, resetDefaultCache, selectInput, validateCachedResult, validateInput, withResilience, withRetry, withTimeout };

package/dist/embeddings/index.mjs

@@ -1,4 +1,4 @@
-import { a as Semaphore, c as withResilience, i as RateLimiter, l as withRetry, n as CircuitBreaker, o as createTimeoutSignal, r as CircuitOpenError, s as isRetryableError, u as withTimeout } from "../http-base-DM7YNo6X.mjs";
-import { A as generateCacheKey, B as normalizeVector, C as createTokenizer, D as needsChunking, E as heuristicTokenCount, F as aggregateVectors, I as cosineSimilarity, L as dotProduct, M as getDefaultCache, N as resetDefaultCache, O as InMemoryEmbeddingCache, P as validateCachedResult, R as euclideanDistance, S as chunkText, T as getChunkingStats, _ as createHttpEmbedding, a as createPiiRedactor, b as selectInput, c as isEmbeddingProvider, d as createCohereEmbedding, f as createHuggingFaceEmbedding, g as HttpEmbeddingProvider, h as createTransformersEmbedding, i as containsPii, j as generateChecksum, k as createNoOpCache, l as TRANSFORMERS_MODELS, m as createOpenAIEmbedding, n as embedScrapedData, o as redactPii, p as createOllamaEmbedding, r as generateEmbeddings, s as createEmbeddingProvider, t as embed, u as createAzureEmbedding, v as getDefaultModel, w as estimateTokens, x as validateInput, y as previewInput, z as getDimensions } from "../embeddings-Dama_IQl.mjs";
+import { a as Semaphore, c as withResilience, i as RateLimiter, l as withRetry, n as CircuitBreaker, o as createTimeoutSignal, r as CircuitOpenError, s as isRetryableError, u as withTimeout } from "../http-base-B8qhcRit.mjs";
+import { A as generateCacheKey, B as normalizeVector, C as createTokenizer, D as needsChunking, E as heuristicTokenCount, F as aggregateVectors, I as cosineSimilarity, L as dotProduct, M as getDefaultCache, N as resetDefaultCache, O as InMemoryEmbeddingCache, P as validateCachedResult, R as euclideanDistance, S as chunkText, T as getChunkingStats, _ as createHttpEmbedding, a as createPiiRedactor, b as selectInput, c as isEmbeddingProvider, d as createCohereEmbedding, f as createHuggingFaceEmbedding, g as HttpEmbeddingProvider, h as createTransformersEmbedding, i as containsPii, j as generateChecksum, k as createNoOpCache, l as TRANSFORMERS_MODELS, m as createOpenAIEmbedding, n as embedScrapedData, o as redactPii, p as createOllamaEmbedding, r as generateEmbeddings, s as createEmbeddingProvider, t as embed, u as createAzureEmbedding, v as getDefaultModel, w as estimateTokens, x as validateInput, y as previewInput, z as getDimensions } from "../embeddings-CukTWZVJ.mjs";
 
 export { CircuitBreaker, CircuitOpenError, HttpEmbeddingProvider, InMemoryEmbeddingCache, RateLimiter, Semaphore, TRANSFORMERS_MODELS, aggregateVectors, chunkText, containsPii, cosineSimilarity, createAzureEmbedding, createCohereEmbedding, createEmbeddingProvider, createHttpEmbedding, createHuggingFaceEmbedding, createNoOpCache, createOllamaEmbedding, createOpenAIEmbedding, createPiiRedactor, createTimeoutSignal, createTokenizer, createTransformersEmbedding, dotProduct, embed, embedScrapedData, estimateTokens, euclideanDistance, generateCacheKey, generateChecksum, generateEmbeddings, getChunkingStats, getDefaultCache, getDefaultModel, getDimensions, heuristicTokenCount, isEmbeddingProvider, isRetryableError, needsChunking, normalizeVector, previewInput, redactPii, resetDefaultCache, selectInput, validateCachedResult, validateInput, withResilience, withRetry, withTimeout };

package/dist/{embeddings-DHVu52VF.cjs → embeddings-BzRSRX9t.cjs}

@@ -1,5 +1,5 @@
-const require_parsers = require('./parsers-Bneuws8x.cjs');
-const require_http_base = require('./http-base-CHLf-Tco.cjs');
+const require_parsers = require('./parsers-DsqevSf6.cjs');
+const require_http_base = require('./http-base-B5EXbNwR.cjs');
 let node_crypto = require("node:crypto");
 
 //#region src/embeddings/aggregation.ts
@@ -1453,4 +1453,4 @@ Object.defineProperty(exports, 'validateInput', {
     return validateInput;
   }
 });
-//# sourceMappingURL=embeddings-DHVu52VF.cjs.map
+//# sourceMappingURL=embeddings-BzRSRX9t.cjs.map