rag-lite-ts 2.1.1 → 2.2.0

package/dist/esm/text/chunker.js

@@ -0,0 +1,279 @@
+/**
+ * Text-specific chunking implementation
+ * Implements the ChunkingStrategy interface for text content
+ */
+import '../dom-polyfills.js';
+import { DEFAULT_CHUNK_CONFIG } from '../core/chunker.js';
+import { countTokens } from './tokenizer.js';
+/**
+ * Split text at paragraph boundaries (double newlines)
+ * This is the first tier of the chunking strategy
+ */
+function splitIntoParagraphs(text) {
+    // Split on double newlines, filter out empty strings
+    return text
+        .split(/\n\s*\n/)
+        .map(p => p.trim())
+        .filter(p => p.length > 0);
+}
+/**
+ * Split text at sentence boundaries using punctuation marks
+ * This is the second tier of the chunking strategy
+ */
+function splitIntoSentences(text) {
+    // Split on sentence-ending punctuation followed by whitespace or end of string
+    // Handle common abbreviations and edge cases
+    const sentences = text
+        .split(/(?<=[.!?])\s+/)
+        .map(s => s.trim())
+        .filter(s => s.length > 0);
+    return sentences;
+}
+/**
+ * Split text into fixed-size chunks based on character count
+ * This is the fallback tier when semantic splitting fails
+ */
+async function splitIntoFixedSizeChunks(text, maxTokens, overlapTokens) {
+    const chunks = [];
+    const words = text.split(/\s+/);
+    let currentChunk = '';
+    let currentTokens = 0;
+    let i = 0;
+    while (i < words.length) {
+        const word = words[i];
+        const testChunk = currentChunk ? `${currentChunk} ${word}` : word;
+        const testTokens = await countTokens(testChunk);
+        if (testTokens <= maxTokens) {
+            currentChunk = testChunk;
+            currentTokens = testTokens;
+            i++;
+        }
+        else {
+            // Current chunk is full, save it
+            if (currentChunk) {
+                chunks.push(currentChunk);
+                // Create overlap for next chunk
+                if (overlapTokens > 0 && chunks.length > 0) {
+                    const overlapText = await createOverlapFromWords(currentChunk, overlapTokens);
+                    currentChunk = overlapText;
+                    currentTokens = await countTokens(currentChunk);
+                }
+                else {
+                    currentChunk = '';
+                    currentTokens = 0;
+                }
+            }
+            else {
+                // Single word exceeds limit, add it anyway
+                chunks.push(word);
+                i++;
+            }
+        }
+    }
+    // Add final chunk if it has content
+    if (currentChunk.trim()) {
+        chunks.push(currentChunk.trim());
+    }
+    return chunks;
+}
+/**
+ * Create overlap text from words at the end of a chunk
+ */
+async function createOverlapFromWords(text, overlapTokens) {
+    const words = text.split(/\s+/);
+    let overlapText = '';
+    let tokens = 0;
+    // Work backwards from the end
+    for (let i = words.length - 1; i >= 0; i--) {
+        const word = words[i];
+        const testText = word + (overlapText ? ' ' + overlapText : '');
+        const testTokens = await countTokens(testText);
+        if (testTokens <= overlapTokens) {
+            overlapText = testText;
+            tokens = testTokens;
+        }
+        else {
+            break;
+        }
+    }
+    return overlapText;
+}
+/**
+ * Create chunks from a list of text segments, respecting token limits
+ */
+async function createChunksFromSegments(segments, config) {
+    const chunks = [];
+    let currentChunk = '';
+    let currentTokens = 0;
+    for (const segment of segments) {
+        const segmentTokens = await countTokens(segment);
+        // If this single segment exceeds our limit, we need to split it further
+        if (segmentTokens > config.chunkSize) {
+            // Save current chunk if it has content
+            if (currentChunk.trim()) {
+                chunks.push(currentChunk.trim());
+                currentChunk = '';
+                currentTokens = 0;
+            }
+            // Split the large segment using fixed-size chunking based on tokens
+            const subChunks = await splitIntoFixedSizeChunks(segment, config.chunkSize, config.chunkOverlap);
+            chunks.push(...subChunks);
+            continue;
+        }
+        // Check if adding this segment would exceed our token limit
+        const potentialChunk = currentChunk ? `${currentChunk}\n\n${segment}` : segment;
+        const potentialTokens = await countTokens(potentialChunk);
+        if (potentialTokens <= config.chunkSize) {
+            // Add to current chunk
+            currentChunk = potentialChunk;
+            currentTokens = potentialTokens;
+        }
+        else {
+            // Save current chunk and start a new one
+            if (currentChunk.trim()) {
+                chunks.push(currentChunk.trim());
+            }
+            // Start new chunk with overlap if possible
+            if (config.chunkOverlap > 0 && currentChunk) {
+                const overlapText = await createOverlapText(currentChunk, config.chunkOverlap);
+                currentChunk = overlapText ? `${overlapText}\n\n${segment}` : segment;
+            }
+            else {
+                currentChunk = segment;
+            }
+            currentTokens = await countTokens(currentChunk);
+        }
+    }
+    // Add final chunk if it has content
+    if (currentChunk.trim()) {
+        chunks.push(currentChunk.trim());
+    }
+    return chunks;
+}
+/**
+ * Create overlap text from the end of a chunk
+ */
+async function createOverlapText(text, overlapTokens) {
+    // Split into sentences and work backwards to get approximately the right amount of overlap
+    const sentences = splitIntoSentences(text);
+    let overlapText = '';
+    let tokens = 0;
+    for (let i = sentences.length - 1; i >= 0; i--) {
+        const sentence = sentences[i];
+        const sentenceTokens = await countTokens(sentence);
+        if (tokens + sentenceTokens <= overlapTokens) {
+            overlapText = sentence + (overlapText ? ' ' + overlapText : '');
+            tokens += sentenceTokens;
+        }
+        else {
+            break;
+        }
+    }
+    return overlapText;
+}
+/**
+ * Text chunking strategy implementation
+ */
+export class TextChunkingStrategy {
+    appliesTo(contentType) {
+        return contentType === 'text';
+    }
+    async chunk(document, config) {
+        console.log(`📝 Chunking document "${document.title}" with config: chunkSize=${config.chunkSize}, chunkOverlap=${config.chunkOverlap}`);
+        if (!document.content || document.content.trim().length === 0) {
+            return [];
+        }
+        // Tier 1: Split into paragraphs
+        const paragraphs = splitIntoParagraphs(document.content);
+        // Tier 2: For large paragraphs, split into sentences
+        const segments = [];
+        for (const paragraph of paragraphs) {
+            const paragraphTokens = await countTokens(paragraph);
+            if (paragraphTokens <= config.chunkSize) {
+                // Paragraph is small enough, use as-is
+                segments.push(paragraph);
+            }
+            else {
+                // Paragraph is too large, split into sentences
+                const sentences = splitIntoSentences(paragraph);
+                // Group sentences that fit within token limits
+                let currentGroup = '';
+                let currentTokens = 0;
+                for (const sentence of sentences) {
+                    const sentenceTokens = await countTokens(sentence);
+                    // If single sentence exceeds limit, it will be handled in createChunksFromSegments
+                    if (sentenceTokens > config.chunkSize) {
+                        // Save current group if it has content
+                        if (currentGroup.trim()) {
+                            segments.push(currentGroup.trim());
+                            currentGroup = '';
+                            currentTokens = 0;
+                        }
+                        // Add the large sentence as its own segment (will be split later)
+                        segments.push(sentence);
+                        continue;
+                    }
+                    const potentialGroup = currentGroup ? `${currentGroup} ${sentence}` : sentence;
+                    const potentialTokens = await countTokens(potentialGroup);
+                    if (potentialTokens <= config.chunkSize) {
+                        currentGroup = potentialGroup;
+                        currentTokens = potentialTokens;
+                    }
+                    else {
+                        // Save current group and start new one
+                        if (currentGroup.trim()) {
+                            segments.push(currentGroup.trim());
+                        }
+                        currentGroup = sentence;
+                        currentTokens = sentenceTokens;
+                    }
+                }
+                // Add final group if it has content
+                if (currentGroup.trim()) {
+                    segments.push(currentGroup.trim());
+                }
+            }
+        }
+        // Tier 3: Create final chunks with overlap handling
+        const chunkTexts = await createChunksFromSegments(segments, config);
+        // Convert to GenericChunk objects
+        const chunks = [];
+        for (let i = 0; i < chunkTexts.length; i++) {
+            const content = chunkTexts[i];
+            chunks.push({
+                content,
+                contentType: document.contentType,
+                chunkIndex: i,
+                metadata: {
+                    tokenCount: await countTokens(content),
+                    ...document.metadata
+                }
+            });
+        }
+        return chunks;
+    }
+}
+/**
+ * Text document chunking function
+ * Converts between text-specific and generic interfaces
+ */
+export async function chunkDocument(document, config = DEFAULT_CHUNK_CONFIG) {
+    const strategy = new TextChunkingStrategy();
+    // Convert Document to GenericDocument
+    const genericDocument = {
+        source: document.source,
+        title: document.title,
+        content: document.content,
+        contentType: 'text'
+    };
+    // Use the strategy to chunk
+    const genericChunks = await strategy.chunk(genericDocument, config);
+    // Convert GenericChunk back to Chunk format
+    const chunks = genericChunks.map(chunk => ({
+        text: chunk.content,
+        chunkIndex: chunk.chunkIndex,
+        tokenCount: chunk.metadata?.tokenCount || 0
+    }));
+    return chunks;
+}
+//# sourceMappingURL=chunker.js.map

package/dist/esm/text/embedder.d.ts

@@ -0,0 +1,111 @@
+import '../dom-polyfills.js';
+import type { EmbeddingResult, EmbedFunction } from '../core/types.js';
+/**
+ * Embedding engine using transformers.js for generating embeddings
+ */
+export declare class EmbeddingEngine {
+    private model;
+    private modelVersion;
+    private readonly modelName;
+    private readonly batchSize;
+    constructor(modelName?: string, batchSize?: number);
+    /**
+     * Load the embedding model
+     * @throws {Error} If model loading fails
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Generate embeddings for a batch of texts
+     * @param texts - Array of text strings to embed
+     * @returns Promise resolving to array of embedding results
+     */
+    embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
+    /**
+     * Process a single batch with error handling for individual chunks
+     * @param batch - Array of text strings in this batch
+     * @param startIndex - Starting index for this batch in the original array
+     * @returns Promise resolving to array of embedding results
+     */
+    private processBatchWithErrorHandling;
+    /**
+     * Fallback to individual chunk processing when batch fails
+     */
+    private fallbackToIndividualProcessing;
+    /**
+     * Process a single chunk with error handling
+     * @param text - Text to embed
+     * @param index - Index of this chunk
+     * @returns Promise resolving to embedding result or null if failed
+     */
+    private processSingleChunk;
+    /**
+     * Generate embedding for a single text
+     * @param text - Text string to embed
+     * @returns Promise resolving to embedding result
+     */
+    embedSingle(text: string): Promise<EmbeddingResult>;
+    /**
+     * Generate embeddings for document chunks with progress logging
+     * Optimized for large document ingestion with batch processing
+     * @param chunks - Array of text chunks from documents
+     * @returns Promise resolving to array of embedding results
+     */
+    embedDocumentBatch(chunks: string[]): Promise<EmbeddingResult[]>;
+    /**
+     * Get the current model version identifier
+     * @returns Model version string
+     */
+    getModelVersion(): string;
+    /**
+     * Check if the model is loaded
+     * @returns True if model is loaded
+     */
+    isLoaded(): boolean;
+    /**
+     * Get the model name
+     * @returns Model name string
+     */
+    getModelName(): string;
+    /**
+     * Get the batch size
+     * @returns Batch size number
+     */
+    getBatchSize(): number;
+    /**
+     * Generate a deterministic model version identifier
+     * Uses model name and configuration for consistent versioning
+     * @returns Model version string
+     */
+    private generateModelVersion;
+    /**
+     * Generate a deterministic embedding ID for a text chunk
+     * @param text - The text content
+     * @param index - Index in the batch
+     * @returns Deterministic embedding ID
+     */
+    private generateEmbeddingId;
+}
+/**
+ * Get the singleton embedding engine instance
+ * @param modelName - Optional model name override
+ * @param batchSize - Optional batch size override
+ * @returns EmbeddingEngine instance
+ */
+export declare function getEmbeddingEngine(modelName?: string, batchSize?: number): EmbeddingEngine;
+/**
+ * Initialize the embedding engine and load the model
+ * @param modelName - Optional model name override
+ * @param batchSize - Optional batch size override
+ * @returns Promise resolving to the loaded embedding engine
+ */
+export declare function initializeEmbeddingEngine(modelName?: string, batchSize?: number): Promise<EmbeddingEngine>;
+/**
+ 
+* Create an EmbedFunction implementation using the text embedding engine
+ * This function implements the core EmbedFunction interface for dependency injection
+ * @param modelName - Optional model name override
+ * @param batchSize - Optional batch size override
+ * @returns EmbedFunction that can be injected into core components
+ */
+export declare function createTextEmbedFunction(modelName?: string, batchSize?: number): EmbedFunction;
+//# sourceMappingURL=embedder.d.ts.map