rag-lite-ts 1.0.1 → 1.0.2

package/dist/{chunker.js → text/chunker.js}

@@ -1,11 +1,10 @@
-import { countTokens } from './tokenizer.js';
 /**
- * Default chunking configuration
+ * Text-specific chunking implementation
+ * Implements the ChunkingStrategy interface for text content
  */
-export const DEFAULT_CHUNK_CONFIG = {
-    chunkSize: 250, // Target 200-300 tokens
-    chunkOverlap: 50
-};
+import '../dom-polyfills.js';
+import { DEFAULT_CHUNK_CONFIG } from '../core/chunker.js';
+import { countTokens } from './tokenizer.js';
 /**
  * Split text at paragraph boundaries (double newlines)
  * This is the first tier of the chunking strategy
@@ -173,84 +172,108 @@ async function createOverlapText(text, overlapTokens) {
     return overlapText;
 }
 /**
- * Main chunking function implementing the three-tier strategy:
- * 1. Split at paragraph boundaries (double newlines)
- * 2. If paragraphs are too large, split at sentence boundaries
- * 3. If sentences are still too large, use fixed-size chunking
- *
- * @param document - Document to chunk
- * @param config - Chunking configuration
- * @returns Array of chunks with metadata
+ * Text chunking strategy implementation
  */
-export async function chunkDocument(document, config = DEFAULT_CHUNK_CONFIG) {
-    console.log(`📝 Chunking document "${document.title}" with config: chunkSize=${config.chunkSize}, chunkOverlap=${config.chunkOverlap}`);
-    if (!document.content || document.content.trim().length === 0) {
-        return [];
+export class TextChunkingStrategy {
+    appliesTo(contentType) {
+        return contentType === 'text';
     }
-    // Tier 1: Split into paragraphs
-    const paragraphs = splitIntoParagraphs(document.content);
-    // Tier 2: For large paragraphs, split into sentences
-    const segments = [];
-    for (const paragraph of paragraphs) {
-        const paragraphTokens = await countTokens(paragraph);
-        if (paragraphTokens <= config.chunkSize) {
-            // Paragraph is small enough, use as-is
-            segments.push(paragraph);
+    async chunk(document, config) {
+        console.log(`📝 Chunking document "${document.title}" with config: chunkSize=${config.chunkSize}, chunkOverlap=${config.chunkOverlap}`);
+        if (!document.content || document.content.trim().length === 0) {
+            return [];
         }
-        else {
-            // Paragraph is too large, split into sentences
-            const sentences = splitIntoSentences(paragraph);
-            // Group sentences that fit within token limits
-            let currentGroup = '';
-            let currentTokens = 0;
-            for (const sentence of sentences) {
-                const sentenceTokens = await countTokens(sentence);
-                // If single sentence exceeds limit, it will be handled in createChunksFromSegments
-                if (sentenceTokens > config.chunkSize) {
-                    // Save current group if it has content
-                    if (currentGroup.trim()) {
-                        segments.push(currentGroup.trim());
-                        currentGroup = '';
-                        currentTokens = 0;
+        // Tier 1: Split into paragraphs
+        const paragraphs = splitIntoParagraphs(document.content);
+        // Tier 2: For large paragraphs, split into sentences
+        const segments = [];
+        for (const paragraph of paragraphs) {
+            const paragraphTokens = await countTokens(paragraph);
+            if (paragraphTokens <= config.chunkSize) {
+                // Paragraph is small enough, use as-is
+                segments.push(paragraph);
+            }
+            else {
+                // Paragraph is too large, split into sentences
+                const sentences = splitIntoSentences(paragraph);
+                // Group sentences that fit within token limits
+                let currentGroup = '';
+                let currentTokens = 0;
+                for (const sentence of sentences) {
+                    const sentenceTokens = await countTokens(sentence);
+                    // If single sentence exceeds limit, it will be handled in createChunksFromSegments
+                    if (sentenceTokens > config.chunkSize) {
+                        // Save current group if it has content
+                        if (currentGroup.trim()) {
+                            segments.push(currentGroup.trim());
+                            currentGroup = '';
+                            currentTokens = 0;
+                        }
+                        // Add the large sentence as its own segment (will be split later)
+                        segments.push(sentence);
+                        continue;
                     }
-                    // Add the large sentence as its own segment (will be split later)
-                    segments.push(sentence);
-                    continue;
-                }
-                const potentialGroup = currentGroup ? `${currentGroup} ${sentence}` : sentence;
-                const potentialTokens = await countTokens(potentialGroup);
-                if (potentialTokens <= config.chunkSize) {
-                    currentGroup = potentialGroup;
-                    currentTokens = potentialTokens;
-                }
-                else {
-                    // Save current group and start new one
-                    if (currentGroup.trim()) {
-                        segments.push(currentGroup.trim());
+                    const potentialGroup = currentGroup ? `${currentGroup} ${sentence}` : sentence;
+                    const potentialTokens = await countTokens(potentialGroup);
+                    if (potentialTokens <= config.chunkSize) {
+                        currentGroup = potentialGroup;
+                        currentTokens = potentialTokens;
+                    }
+                    else {
+                        // Save current group and start new one
+                        if (currentGroup.trim()) {
+                            segments.push(currentGroup.trim());
+                        }
+                        currentGroup = sentence;
+                        currentTokens = sentenceTokens;
                     }
-                    currentGroup = sentence;
-                    currentTokens = sentenceTokens;
                 }
-            }
-            // Add final group if it has content
-            if (currentGroup.trim()) {
-                segments.push(currentGroup.trim());
+                // Add final group if it has content
+                if (currentGroup.trim()) {
+                    segments.push(currentGroup.trim());
+                }
             }
         }
+        // Tier 3: Create final chunks with overlap handling
+        const chunkTexts = await createChunksFromSegments(segments, config);
+        // Convert to GenericChunk objects
+        const chunks = [];
+        for (let i = 0; i < chunkTexts.length; i++) {
+            const content = chunkTexts[i];
+            chunks.push({
+                content,
+                contentType: document.contentType,
+                chunkIndex: i,
+                metadata: {
+                    tokenCount: await countTokens(content),
+                    ...document.metadata
+                }
+            });
+        }
+        return chunks;
     }
-    // Tier 3: Create final chunks with overlap handling
-    const chunkTexts = await createChunksFromSegments(segments, config);
-    // Convert to Chunk objects with metadata
-    const chunks = [];
-    for (let i = 0; i < chunkTexts.length; i++) {
-        const text = chunkTexts[i];
-        const tokenCount = await countTokens(text);
-        chunks.push({
-            text,
-            chunkIndex: i,
-            tokenCount
-        });
-    }
+}
+/**
+ * Text document chunking function
+ * Converts between text-specific and generic interfaces
+ */
+export async function chunkDocument(document, config = DEFAULT_CHUNK_CONFIG) {
+    const strategy = new TextChunkingStrategy();
+    // Convert Document to GenericDocument
+    const genericDocument = {
+        source: document.source,
+        title: document.title,
+        content: document.content,
+        contentType: 'text'
+    };
+    // Use the strategy to chunk
+    const genericChunks = await strategy.chunk(genericDocument, config);
+    // Convert GenericChunk back to Chunk format
+    const chunks = genericChunks.map(chunk => ({
+        text: chunk.content,
+        chunkIndex: chunk.chunkIndex,
+        tokenCount: chunk.metadata?.tokenCount || 0
+    }));
     return chunks;
 }
 //# sourceMappingURL=chunker.js.map

package/dist/text/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/text/chunker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,qBAAqB,CAAC;AAC7B,OAAO,EAKL,oBAAoB,EACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAiB7C;;;GAGG;AACH,SAAS,mBAAmB,CAAC,IAAY;IACvC,qDAAqD;IACrD,OAAO,IAAI;SACR,KAAK,CAAC,SAAS,CAAC;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,IAAY;IACtC,+EAA+E;IAC/E,6CAA6C;IAC7C,MAAM,SAAS,GAAG,IAAI;SACnB,KAAK,CAAC,eAAe,CAAC;SACtB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE7B,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,wBAAwB,CAAC,IAAY,EAAE,SAAiB,EAAE,aAAqB;IAC5F,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEhC,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,CAAC,GAAG,CAAC,CAAC;IAEV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAClE,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,CAAC;QAEhD,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;YAC5B,YAAY,GAAG,SAAS,CAAC;YACzB,aAAa,GAAG,UAAU,CAAC;YAC3B,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,CAAC;YACN,iCAAiC;YACjC,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAE1B,gCAAgC;gBAChC,IAAI,aAAa,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3C,MAAM,WAAW,GAAG,MAAM,sBAAsB,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;oBAC9E,YAAY,GAAG,WAAW,CAAC;oBAC3B,aAAa,GAAG,MAAM,WAAW,CAAC,YAAY,CAAC,CAAC;gBAClD,CAAC;qBAAM,CAAC;oBACN,YAAY,GAAG,EAAE,CAAC;oBAClB,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,2CAA2C;gBAC3C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClB,CAAC,EAAE,CAAC;YACN,CAAC;QACH,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,sBAAsB,CAAC,IAAY,EAAE,aAAqB;IACvE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,8BAA8B;IAC9B,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC;QAE/C,IAAI,UAAU,IAAI,aAAa,EAAE,CAAC;YAChC,WAAW,GAAG,QAAQ,CAAC;YACvB,MAAM,GAAG,UAAU,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,MAAM;QACR,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,wBAAwB,CACrC,QAAkB,EAClB,MAAmB;IAEnB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;QAEjD,wEAAwE;QACxE,IAAI,aAAa,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;YACrC,uCAAuC;YACvC,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjC,YAAY,GAAG,EAAE,CAAC;gBAClB,aAAa,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,oEAAoE;YACpE,MAAM,SAAS,GAAG,MAAM,wBAAwB,CAAC,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;YACjG,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,4DAA4D;QAC5D,MAAM,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,OAAO,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;QAChF,MAAM,eAAe,GAAG,MAAM,WAAW,CAAC,cAAc,CAAC,CAAC;QAE1D,IAAI,eAAe,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACxC,uBAAuB;YACvB,YAAY,GAAG,cAAc,CAAC;YAC9B,aAAa,GAAG,eAAe,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,yCAAyC;YACzC,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YACnC,CAAC;YAED,2CAA2C;YAC3C,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,IAAI,YAAY,EAAE,CAAC;gBAC5C,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC/E,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,WAAW,OAAO,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;YACxE,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,OAAO,CAAC;YACzB,CAAC;YACD,aAAa,GAAG,MAAM,WAAW,CAAC,YAAY,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,iBAAiB,CAAC,IAAY,EAAE,aAAqB;IAClE,2FAA2F;IAC3F,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,cAAc,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC;QAEnD,IAAI,MAAM,GAAG,cAAc,IAAI,aAAa,EAAE,CAAC;YAC7C,WAAW,GAAG,QAAQ,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,MAAM,IAAI,cAAc,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,MAAM;QACR,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,oBAAoB;IAC/B,SAAS,CAAC,WAAmB;QAC3B,OAAO,WAAW,KAAK,MAAM,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAyB,EAAE,MAAmB;QACxD,OAAO,CAAC,GAAG,CAAC,yBAAyB,QAAQ,CAAC,KAAK,4BAA4B,MAAM,CAAC,SAAS,kBAAkB,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QAExI,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9D,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,gCAAgC;QAChC,MAAM,UAAU,GAAG,mBAAmB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAEzD,qDAAqD;QACrD,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,eAAe,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,CAAC;YAErD,IAAI,eAAe,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACxC,uCAAuC;gBACvC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,+CAA+C;gBAC/C,MAAM,SAAS,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;gBAEhD,+CAA+C;gBAC/C,IAAI,YAAY,GAAG,EAAE,CAAC;gBACtB,IAAI,aAAa,GAAG,CAAC,CAAC;gBAEtB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,MAAM,cAAc,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC;oBAEnD,mFAAmF;oBACnF,IAAI,cAAc,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;wBACtC,uCAAuC;wBACvC,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;4BACxB,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;4BACnC,YAAY,GAAG,EAAE,CAAC;4BAClB,aAAa,GAAG,CAAC,CAAC;wBACpB,CAAC;wBACD,kEAAkE;wBAClE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;wBACxB,SAAS;oBACX,CAAC;oBAED,MAAM,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;oBAC/E,MAAM,eAAe,GAAG,MAAM,WAAW,CAAC,cAAc,CAAC,CAAC;oBAE1D,IAAI,eAAe,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;wBACxC,YAAY,GAAG,cAAc,CAAC;wBAC9B,aAAa,GAAG,eAAe,CAAC;oBAClC,CAAC;yBAAM,CAAC;wBACN,uCAAuC;wBACvC,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;4BACxB,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;wBACrC,CAAC;wBACD,YAAY,GAAG,QAAQ,CAAC;wBACxB,aAAa,GAAG,cAAc,CAAC;oBACjC,CAAC;gBACH,CAAC;gBAED,oCAAoC;gBACpC,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;oBACxB,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAED,oDAAoD;QACpD,MAAM,UAAU,GAAG,MAAM,wBAAwB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAEpE,kCAAkC;QAClC,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAE9B,MAAM,CAAC,IAAI,CAAC;gBACV,OAAO;gBACP,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,UAAU,EAAE,CAAC;gBACb,QAAQ,EAAE;oBACR,UAAU,EAAE,MAAM,WAAW,CAAC,OAAO,CAAC;oBACtC,GAAG,QAAQ,CAAC,QAAQ;iBACrB;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAkB,EAClB,SAAsB,oBAAoB;IAE1C,MAAM,QAAQ,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAE5C,sCAAsC;IACtC,MAAM,eAAe,GAAoB;QACvC,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,WAAW,EAAE,MAAM;KACpB,CAAC;IAEF,4BAA4B;IAC5B,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAEpE,4CAA4C;IAC5C,MAAM,MAAM,GAAY,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClD,IAAI,EAAE,KAAK,CAAC,OAAO;QACnB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,UAAU,EAAE,KAAK,CAAC,QAAQ,EAAE,UAAU,IAAI,CAAC;KAC5C,CAAC,CAAC,CAAC;IAEJ,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/{embedder.d.ts → text/embedder.d.ts}

@@ -1,4 +1,5 @@
-import type { EmbeddingResult } from './types.js';
+import '../dom-polyfills.js';
+import type { EmbeddingResult, EmbedFunction } from '../core/types.js';
 /**
  * Embedding engine using transformers.js for generating embeddings
  */
@@ -98,4 +99,24 @@ export declare function getEmbeddingEngine(modelName?: string, batchSize?: numbe
  * @returns Promise resolving to the loaded embedding engine
  */
 export declare function initializeEmbeddingEngine(modelName?: string, batchSize?: number): Promise<EmbeddingEngine>;
+/**
+ 
+* Create an EmbedFunction implementation using the text embedding engine
+ * This function implements the core EmbedFunction interface for dependency injection
+ * @param modelName - Optional model name override
+ * @param batchSize - Optional batch size override
+ * @returns EmbedFunction that can be injected into core components
+ */
+export declare function createTextEmbedFunction(modelName?: string, batchSize?: number): EmbedFunction;
+/**
+ * Create a text embedding engine factory function
+ * @param modelName - Optional model name override
+ * @param batchSize - Optional batch size override
+ * @returns Factory function that creates initialized embedding engines
+ */
+export declare function createTextEmbedder(modelName?: string, batchSize?: number): {
+    embedSingle(text: string): Promise<EmbeddingResult>;
+    embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
+    embedDocumentBatch(chunks: string[]): Promise<EmbeddingResult[]>;
+};
 //# sourceMappingURL=embedder.d.ts.map

package/dist/text/embedder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/text/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,qBAAqB,CAAC;AAI7B,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAUvE;;GAEG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAelD;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAgEhC;;;;OAIG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAqB7D;;;;;OAKG;YACW,6BAA6B;IAmC3C;;OAEG;YACW,8BAA8B;IAgC5C;;;;;OAKG;YACW,kBAAkB;IAqBhC;;;;OAIG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAQzD;;;;;OAKG;IACG,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IA+CtE;;;OAGG;IACH,eAAe,IAAI,MAAM;IAOzB;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,YAAY,IAAI,MAAM;IAItB;;;OAGG;IACH,YAAY,IAAI,MAAM;IAItB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAa5B;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;CAM5B;AAQD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,eAAe,CAS1F;AAED;;;;;GAKG;AACH,wBAAsB,yBAAyB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAQhH;AACD;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,aAAa,CAyB7F;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;sBAE7C,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;sBAKjC,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;+BAK5B,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;EAKzE"}

package/dist/{embedder.js → text/embedder.js}

@@ -1,7 +1,7 @@
-import { pipeline } from '@huggingface/transformers';
+import '../dom-polyfills.js';
 import { createHash } from 'crypto';
-import { config } from './config.js';
-import { handleError, ErrorCategory, ErrorSeverity, safeExecute } from './error-handler.js';
+import { config } from '../core/config.js';
+import { handleError, ErrorCategory, ErrorSeverity, safeExecute } from '../core/error-handler.js';
 /**
  * List of supported embedding models
  */
@@ -34,9 +34,25 @@ export class EmbeddingEngine {
     async loadModel() {
         await safeExecute(async () => {
             console.log(`Loading embedding model: ${this.modelName}`);
-            // Initialize the feature extraction pipeline
+            // Ensure DOM polyfills are set up before importing transformers
+            if (typeof globalThis.self === 'undefined') {
+                globalThis.self = globalThis;
+            }
+            if (typeof global.self === 'undefined') {
+                global.self = global;
+            }
+            // Additional polyfills that might be needed
+            if (typeof globalThis.window === 'undefined') {
+                globalThis.window = {};
+            }
+            if (typeof globalThis.document === 'undefined') {
+                globalThis.document = {};
+            }
+            console.log('Embedder polyfills set up. self is now:', typeof self !== 'undefined' ? 'defined' : 'undefined');
+            // Initialize the feature extraction pipeline using dynamic import
             // Let transformers.js handle model caching automatically
             try {
+                const { pipeline } = await import('@huggingface/transformers');
                 this.model = await pipeline('feature-extraction', this.modelName, {
                     cache_dir: config.model_cache_path,
                     local_files_only: false,
@@ -320,4 +336,55 @@ export async function initializeEmbeddingEngine(modelName, batchSize) {
     }
     return engine;
 }
+/**
+ 
+* Create an EmbedFunction implementation using the text embedding engine
+ * This function implements the core EmbedFunction interface for dependency injection
+ * @param modelName - Optional model name override
+ * @param batchSize - Optional batch size override
+ * @returns EmbedFunction that can be injected into core components
+ */
+export function createTextEmbedFunction(modelName, batchSize) {
+    let engine = null;
+    const embedFunction = async (query, contentType) => {
+        // Only support text content type
+        if (contentType && contentType !== 'text') {
+            throw new Error(`Text embedder only supports 'text' content type, got: ${contentType}`);
+        }
+        // Initialize engine if not already done
+        if (!engine) {
+            engine = await initializeEmbeddingEngine(modelName, batchSize);
+        }
+        // Use the existing embedSingle method
+        const result = await engine.embedSingle(query);
+        // Add contentType to the result
+        return {
+            ...result,
+            contentType: 'text'
+        };
+    };
+    return embedFunction;
+}
+/**
+ * Create a text embedding engine factory function
+ * @param modelName - Optional model name override
+ * @param batchSize - Optional batch size override
+ * @returns Factory function that creates initialized embedding engines
+ */
+export function createTextEmbedder(modelName, batchSize) {
+    return {
+        async embedSingle(text) {
+            const engine = await initializeEmbeddingEngine(modelName, batchSize);
+            return engine.embedSingle(text);
+        },
+        async embedBatch(texts) {
+            const engine = await initializeEmbeddingEngine(modelName, batchSize);
+            return engine.embedBatch(texts);
+        },
+        async embedDocumentBatch(chunks) {
+            const engine = await initializeEmbeddingEngine(modelName, batchSize);
+            return engine.embedDocumentBatch(chunks);
+        }
+    };
+}
 //# sourceMappingURL=embedder.js.map

package/dist/text/embedder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/text/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,qBAAqB,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAe,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAG/G;;GAEG;AACH,MAAM,gBAAgB,GAAG;IACvB,wCAAwC;IACxC,0BAA0B;CAC3B,CAAC;AAEF;;GAEG;AACH,MAAM,OAAO,eAAe;IAClB,KAAK,GAAe,IAAI,CAAC;IACzB,YAAY,GAAkB,IAAI,CAAC;IAC1B,SAAS,CAAS;IAClB,SAAS,CAAS;IAEnC,YAAY,SAAkB,EAAE,SAAkB;QAChD,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,MAAM,CAAC,eAAe,CAAC;QACrD,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC;QAEhD,uCAAuC;QACvC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CACb,sBAAsB,IAAI,CAAC,SAAS,IAAI;gBACxC,qBAAqB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnD,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,8CAA8C,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAC5G,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS;QACb,MAAM,WAAW,CACf,KAAK,IAAI,EAAE;YACT,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YAE1D,gEAAgE;YAChE,IAAI,OAAQ,UAAkB,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACnD,UAAkB,CAAC,IAAI,GAAG,UAAU,CAAC;YACxC,CAAC;YACD,IAAI,OAAQ,MAAc,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC/C,MAAc,CAAC,IAAI,GAAG,MAAM,CAAC;YAChC,CAAC;YAED,4CAA4C;YAC5C,IAAI,OAAQ,UAAkB,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACrD,UAAkB,CAAC,MAAM,GAAG,EAAE,CAAC;YAClC,CAAC;YACD,IAAI,OAAQ,UAAkB,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;gBACvD,UAAkB,CAAC,QAAQ,GAAG,EAAE,CAAC;YACpC,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,yCAAyC,EAAE,OAAO,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YAE9G,kEAAkE;YAClE,yDAAyD;YACzD,IAAI,CAAC;gBACH,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;gBAC/D,IAAI,CAAC,KAAK,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,SAAS,EAAE;oBAChE,SAAS,EAAE,MAAM,CAAC,gBAAgB;oBAClC,gBAAgB,EAAE,KAAK;oBACvB,KAAK,EAAE,MAAM,CAAC,+CAA+C;iBAC9D,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,sDAAsD;gBACtD,IAAI,KAAK,YAAY,KAAK,IAAI,CAC5B,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;oBACjC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;oBAClC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAC/B,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;oBACnC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;oBACtC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAClC,EAAE,CAAC;oBACF,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,CAAC,SAAS,KAAK;wBAChD,wFAAwF,CACzF,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YAED,8BAA8B;YAC9B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAEhD,OAAO,CAAC,GAAG,CAAC,uCAAuC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC1E,CAAC,EACD,eAAe,EACf;YACE,QAAQ,EAAE,aAAa,CAAC,KAAK;YAC7B,QAAQ,EAAE,aAAa,CAAC,KAAK;YAC7B,QAAQ,EAAE,CAAC;SACZ,CACF,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,4DAA4D;QAC5D,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACjD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACxE,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,6BAA6B,CAAC,KAAe,EAAE,UAAkB;QAC7E,OAAO,MAAM,WAAW,CACtB,KAAK,IAAI,EAAE;YACT,wCAAwC;YACxC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAM,CAAC,KAAK,EAAE;gBAC1C,OAAO,EAAE,MAAM;gBACf,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,oCAAoC;YACpC,MAAM,OAAO,GAAsB,EAAE,CAAC;YACtC,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;gBACxE,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;gBAElD,OAAO,CAAC,IAAI,CAAC;oBACX,YAAY;oBACZ,MAAM;iBACP,CAAC,CAAC;YACL,CAAC;YAED,OAAO,OAAO,CAAC;QACjB,CAAC,EACD,oBAAoB,KAAK,CAAC,MAAM,UAAU,EAC1C;YACE,QAAQ,EAAE,aAAa,CAAC,SAAS;YACjC,QAAQ,EAAE,aAAa,CAAC,KAAK;YAC7B,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,EAAE;SAClB,CACF,IAAI,MAAM,IAAI,CAAC,8BAA8B,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,8BAA8B,CAAC,KAAe,EAAE,UAAkB;QAC9E,WAAW,CACT,+BAA+B,KAAK,CAAC,MAAM,gDAAgD,EAC3F,4BAA4B,EAC5B;YACE,QAAQ,EAAE,aAAa,CAAC,SAAS;YACjC,QAAQ,EAAE,aAAa,CAAC,OAAO;YAC/B,SAAS,EAAE,IAAI;SAChB,CACF,CAAC;QAEF,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,YAAY,GAAG,MAAM,WAAW,CACpC,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC,EACvD,+BAA+B,UAAU,GAAG,CAAC,GAAG,EAChD;gBACE,QAAQ,EAAE,aAAa,CAAC,SAAS;gBACjC,QAAQ,EAAE,aAAa,CAAC,OAAO;gBAC/B,SAAS,EAAE,IAAI;aAChB,CACF,CAAC;YAEF,IAAI,YAAY,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,kBAAkB,CAAC,IAAY,EAAE,KAAa;QAC1D,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAM,CAAC,CAAC,IAAI,CAAC,EAAE;gBAC3C,OAAO,EAAE,MAAM;gBACf,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC3D,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;YAElD,OAAO;gBACL,YAAY;gBACZ,MAAM;aACP,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,kCAAkC;YAClC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,IAAY;QAC5B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,MAAgB;QACvC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,MAAM,SAAS,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,kBAAkB,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC;QAErH,MAAM,OAAO,GAAsB,EAAE,CAAC;QACtC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/D,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,YAAY,EAAE,UAAU,EAAE,EAAE,CAAC;YACjE,MAAM,QAAQ,GAAG,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;YAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAClE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE7C,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBAC/E,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;gBAC9B,eAAe,IAAI,YAAY,CAAC,MAAM,CAAC;gBACvC,aAAa,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEtD,sEAAsE;gBACtE,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,yBAAyB;gBAC9F,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,gBAAgB,KAAK,CAAC,IAAI,UAAU,KAAK,YAAY,GAAG,CAAC,EAAE,CAAC;oBACjF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC;oBACvE,OAAO,CAAC,GAAG,CAAC,aAAa,eAAe,OAAO,MAAM,CAAC,MAAM,YAAY,UAAU,KAAK,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,aAAa,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACnJ,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,2BAA2B,UAAU,GAAG,CAAC,IAAI,YAAY,GAAG,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACpI,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC;YAChC,CAAC;QACH,CAAC;QAED,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,yBAAyB,eAAe,gBAAgB,aAAa,wBAAwB,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,yBAAyB,eAAe,gCAAgC,CAAC,CAAC;QACxF,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC;IAC7B,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,oBAAoB;QAC1B,oEAAoE;QACpE,6EAA6E;QAC7E,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,oDAAoD;YACpD,SAAS,EAAE,KAAK;YAChB,QAAQ,EAAE,MAAM;SACjB,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpF,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IACvD,CAAC;IAED;;;;;OAKG;IACK,mBAAmB,CAAC,IAAY,EAAE,KAAa;QACrD,qDAAqD;QACrD,oFAAoF;QACpF,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3E,OAAO,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACtC,CAAC;CACF;AAED;;;GAGG;AACH,IAAI,uBAAuB,GAA2B,IAAI,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAkB,EAAE,SAAkB;IACvE,mEAAmE;IACnE,sEAAsE;IACtE,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;QAC3B,uBAAuB,GAAG,IAAI,eAAe,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACtE,CAAC;SAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACpC,uBAAuB,GAAG,IAAI,eAAe,EAAE,CAAC;IAClD,CAAC;IACD,OAAO,uBAAuB,CAAC;AACjC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,SAAkB,EAAE,SAAkB;IACpF,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAExD,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QACvB,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAC3B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AACD;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CAAC,SAAkB,EAAE,SAAkB;IAC5E,IAAI,MAAM,GAA2B,IAAI,CAAC;IAE1C,MAAM,aAAa,GAAkB,KAAK,EAAE,KAAa,EAAE,WAAoB,EAA4B,EAAE;QAC3G,iCAAiC;QACjC,IAAI,WAAW,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,yDAAyD,WAAW,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,wCAAwC;QACxC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACjE,CAAC;QAED,sCAAsC;QACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAE/C,gCAAgC;QAChC,OAAO;YACL,GAAG,MAAM;YACT,WAAW,EAAE,MAAM;SACpB,CAAC;IACJ,CAAC,CAAC;IAEF,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAkB,EAAE,SAAkB;IACvE,OAAO;QACL,KAAK,CAAC,WAAW,CAAC,IAAY;YAC5B,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACrE,OAAO,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAED,KAAK,CAAC,UAAU,CAAC,KAAe;YAC9B,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACrE,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;QAED,KAAK,CAAC,kBAAkB,CAAC,MAAgB;YACvC,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACrE,OAAO,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/text/index.d.ts

@@ -0,0 +1,7 @@
+export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction, createTextEmbedder } from './embedder.js';
+export { CrossEncoderReranker, createTextRerankFunction, createTextReranker } from './reranker.js';
+export { countTokens, getTokenizer, resetTokenizer } from './tokenizer.js';
+export { chunkDocument, type Chunk, type Document } from '../core/chunker.js';
+export { type ChunkConfig } from '../core/chunker.js';
+export * from './preprocessors/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/text/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/text/index.ts"],"names":[],"mappings":"AACA,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,yBAAyB,EACzB,uBAAuB,EACvB,kBAAkB,EACnB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,oBAAoB,EACpB,wBAAwB,EACxB,kBAAkB,EACnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,KAAK,KAAK,EAAE,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGtD,cAAc,0BAA0B,CAAC"}

package/dist/text/index.js

@@ -0,0 +1,8 @@
+// Text implementation layer exports
+export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction, createTextEmbedder } from './embedder.js';
+export { CrossEncoderReranker, createTextRerankFunction, createTextReranker } from './reranker.js';
+export { countTokens, getTokenizer, resetTokenizer } from './tokenizer.js';
+export { chunkDocument } from '../core/chunker.js';
+// Re-export preprocessors
+export * from './preprocessors/index.js';
+//# sourceMappingURL=index.js.map

package/dist/text/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/text/index.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,yBAAyB,EACzB,uBAAuB,EACvB,kBAAkB,EACnB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,oBAAoB,EACpB,wBAAwB,EACxB,kBAAkB,EACnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EAAE,aAAa,EAA6B,MAAM,oBAAoB,CAAC;AAG9E,0BAA0B;AAC1B,cAAc,0BAA0B,CAAC"}

package/dist/text/preprocessors/index.d.ts

@@ -0,0 +1,17 @@
+import { PreprocessorRegistry } from './registry.js';
+export { PreprocessorRegistry, ContentTypeDetector } from './registry.js';
+export { MdxPreprocessor } from './mdx.js';
+export { MermaidPreprocessor } from './mermaid.js';
+/**
+ * Global preprocessor registry instance
+ */
+export declare const preprocessorRegistry: PreprocessorRegistry;
+/**
+ * Validate that all required preprocessors are available in the registry
+ */
+export declare function validatePreprocessorConfiguration(requiredPreprocessors: string[]): void;
+/**
+ * Get all available preprocessor names
+ */
+export declare function getAvailablePreprocessors(): string[];
+//# sourceMappingURL=index.d.ts.map

package/dist/text/preprocessors/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/text/preprocessors/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAKrD,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAenD;;GAEG;AACH,eAAO,MAAM,oBAAoB,sBAA+B,CAAC;AAEjE;;GAEG;AACH,wBAAgB,iCAAiC,CAAC,qBAAqB,EAAE,MAAM,EAAE,GAAG,IAAI,CAOvF;AAED;;GAEG;AACH,wBAAgB,yBAAyB,IAAI,MAAM,EAAE,CAEpD"}

package/dist/text/preprocessors/index.js

@@ -0,0 +1,38 @@
+import { PreprocessorRegistry } from './registry.js';
+import { MdxPreprocessor } from './mdx.js';
+import { MermaidPreprocessor } from './mermaid.js';
+// Export all preprocessor classes
+export { PreprocessorRegistry, ContentTypeDetector } from './registry.js';
+export { MdxPreprocessor } from './mdx.js';
+export { MermaidPreprocessor } from './mermaid.js';
+/**
+ * Create and initialize the global preprocessor registry
+ */
+function createPreprocessorRegistry() {
+    const registry = new PreprocessorRegistry();
+    // Register built-in preprocessors
+    registry.register('mdx', new MdxPreprocessor());
+    registry.register('mermaid', new MermaidPreprocessor());
+    return registry;
+}
+/**
+ * Global preprocessor registry instance
+ */
+export const preprocessorRegistry = createPreprocessorRegistry();
+/**
+ * Validate that all required preprocessors are available in the registry
+ */
+export function validatePreprocessorConfiguration(requiredPreprocessors) {
+    const validation = preprocessorRegistry.validatePreprocessors(requiredPreprocessors);
+    if (!validation.valid) {
+        const missingList = validation.missing.join(', ');
+        throw new Error(`Missing required preprocessors: ${missingList}. Available: ${preprocessorRegistry.getRegisteredNames().join(', ')}`);
+    }
+}
+/**
+ * Get all available preprocessor names
+ */
+export function getAvailablePreprocessors() {
+    return preprocessorRegistry.getRegisteredNames();
+}
+//# sourceMappingURL=index.js.map

package/dist/text/preprocessors/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/text/preprocessors/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEnD,kCAAkC;AAClC,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,SAAS,0BAA0B;IACjC,MAAM,QAAQ,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAE5C,kCAAkC;IAClC,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAChD,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,mBAAmB,EAAE,CAAC,CAAC;IAExD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,0BAA0B,EAAE,CAAC;AAEjE;;GAEG;AACH,MAAM,UAAU,iCAAiC,CAAC,qBAA+B;IAC/E,MAAM,UAAU,GAAG,oBAAoB,CAAC,qBAAqB,CAAC,qBAAqB,CAAC,CAAC;IAErF,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACtB,MAAM,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,mCAAmC,WAAW,gBAAgB,oBAAoB,CAAC,kBAAkB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxI,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,yBAAyB;IACvC,OAAO,oBAAoB,CAAC,kBAAkB,EAAE,CAAC;AACnD,CAAC"}

package/dist/text/preprocessors/mdx.d.ts

@@ -0,0 +1,25 @@
+import { Preprocessor, PreprocessorOptions } from '../../types.js';
+/**
+ * MDX preprocessor for handling JSX content in Markdown files
+ * Ports the existing cleanMdxContent logic with mode-aware behavior
+ */
+export declare class MdxPreprocessor implements Preprocessor {
+    /**
+     * Check if this preprocessor applies to the given language/content type
+     * Applies to .mdx files and content with JSX syntax
+     */
+    appliesTo(language: string): boolean;
+    /**
+     * Process MDX content based on the specified mode
+     */
+    process(content: string, options: PreprocessorOptions): string;
+    /**
+     * Strip JSX content entirely - ported from cleanMdxContent logic
+     */
+    private stripJsx;
+    /**
+     * Replace JSX with descriptive placeholders
+     */
+    private replaceWithPlaceholders;
+}
+//# sourceMappingURL=mdx.d.ts.map

package/dist/text/preprocessors/mdx.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mdx.d.ts","sourceRoot":"","sources":["../../../src/text/preprocessors/mdx.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAGnE;;;GAGG;AACH,qBAAa,eAAgB,YAAW,YAAY;IAClD;;;OAGG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpC;;OAEG;IACH,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,GAAG,MAAM;IAmB9D;;OAEG;IACH,OAAO,CAAC,QAAQ;IAyChB;;OAEG;IACH,OAAO,CAAC,uBAAuB;CAiChC"}

package/dist/text/preprocessors/mdx.js

@@ -0,0 +1,101 @@
+import { ContentTypeDetector } from './registry.js';
+/**
+ * MDX preprocessor for handling JSX content in Markdown files
+ * Ports the existing cleanMdxContent logic with mode-aware behavior
+ */
+export class MdxPreprocessor {
+    /**
+     * Check if this preprocessor applies to the given language/content type
+     * Applies to .mdx files and content with JSX syntax
+     */
+    appliesTo(language) {
+        return language === 'mdx';
+    }
+    /**
+     * Process MDX content based on the specified mode
+     */
+    process(content, options) {
+        // Only process if content actually contains JSX
+        if (!ContentTypeDetector.hasJsxContent(content)) {
+            return content;
+        }
+        switch (options.mode) {
+            case 'strip':
+                return this.stripJsx(content);
+            case 'keep':
+                return content; // Keep JSX as-is
+            case 'placeholder':
+                return this.replaceWithPlaceholders(content);
+            default:
+                console.log(`Unknown MDX processing mode: ${options.mode}, using placeholder`);
+                return this.replaceWithPlaceholders(content);
+        }
+    }
+    /**
+     * Strip JSX content entirely - ported from cleanMdxContent logic
+     */
+    stripJsx(content) {
+        let cleaned = content;
+        // Remove JSX import statements (requirement 11.1)
+        // Matches: import ... from '...' or import ... from "..."
+        cleaned = cleaned.replace(/^import\s+.*?from\s+['"][^'"]*['"];?\s*$/gm, '');
+        // Remove JSX export statements (requirement 11.2)  
+        // Handle both single-line and multi-line exports
+        // Multi-line function exports: export default function() { ... }
+        cleaned = cleaned.replace(/^export\s+default\s+function[^{]*\{[\s\S]*?\n\}\s*$/gm, '');
+        // Object exports: export const metadata = { ... }
+        cleaned = cleaned.replace(/^export\s+const\s+[^=]*=\s*\{[\s\S]*?\}\s*;?\s*$/gm, '');
+        // Single line exports: export const x = ...; or export default ...
+        cleaned = cleaned.replace(/^export\s+(?:default\s+)?(?:const|let|var|function|class)\s+[^;{]*;?\s*$/gm, '');
+        // Simple exports: export default Component
+        cleaned = cleaned.replace(/^export\s+default\s+[^;{]*;?\s*$/gm, '');
+        // Remove JSX components (requirements 11.3, 11.4)
+        // Self-closing tags: <Component />
+        cleaned = cleaned.replace(/<[A-Z][a-zA-Z0-9]*[^>]*\/>/g, '');
+        // Opening and closing tags with content: <Component>content</Component>
+        // This handles nested components by replacing the outermost ones first
+        let previousLength;
+        do {
+            previousLength = cleaned.length;
+            cleaned = cleaned.replace(/<[A-Z][a-zA-Z0-9]*[^>]*>.*?<\/[A-Z][a-zA-Z0-9]*>/gs, '');
+        } while (cleaned.length !== previousLength);
+        // Clean up multiple consecutive newlines and trim
+        cleaned = cleaned.replace(/\n\s*\n\s*\n/g, '\n\n').trim();
+        // Ensure we never return empty content (requirement 6.4)
+        if (!cleaned.trim()) {
+            return '[content removed]';
+        }
+        return cleaned;
+    }
+    /**
+     * Replace JSX with descriptive placeholders
+     */
+    replaceWithPlaceholders(content) {
+        let cleaned = content;
+        // Replace JSX import statements
+        cleaned = cleaned.replace(/^import\s+.*?from\s+['"][^'"]*['"];?\s*$/gm, '[import removed]');
+        // Replace JSX export statements  
+        // Multi-line function exports: export default function() { ... }
+        cleaned = cleaned.replace(/^export\s+default\s+function[^{]*\{[\s\S]*?\n\}\s*$/gm, '[export removed]');
+        // Object exports: export const metadata = { ... }
+        cleaned = cleaned.replace(/^export\s+const\s+[^=]*=\s*\{[\s\S]*?\}\s*;?\s*$/gm, '[export removed]');
+        // Single line exports: export const x = ...; or export default ...
+        cleaned = cleaned.replace(/^export\s+(?:default\s+)?(?:const|let|var|function|class)\s+[^;{]*;?\s*$/gm, '[export removed]');
+        // Simple exports: export default Component
+        cleaned = cleaned.replace(/^export\s+default\s+[^;{]*;?\s*$/gm, '[export removed]');
+        // Replace JSX components with placeholder
+        // Self-closing tags: <Component />
+        cleaned = cleaned.replace(/<[A-Z][a-zA-Z0-9]*[^>]*\/>/g, '[component removed]');
+        // Opening and closing tags with content: <Component>content</Component>
+        // This handles nested components by replacing the outermost ones first
+        let previousLength;
+        do {
+            previousLength = cleaned.length;
+            cleaned = cleaned.replace(/<[A-Z][a-zA-Z0-9]*[^>]*>.*?<\/[A-Z][a-zA-Z0-9]*>/gs, '[component removed]');
+        } while (cleaned.length !== previousLength);
+        // Clean up multiple consecutive newlines and trim
+        cleaned = cleaned.replace(/\n\s*\n\s*\n/g, '\n\n').trim();
+        return cleaned;
+    }
+}
+//# sourceMappingURL=mdx.js.map

package/dist/text/preprocessors/mdx.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mdx.js","sourceRoot":"","sources":["../../../src/text/preprocessors/mdx.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpD;;;GAGG;AACH,MAAM,OAAO,eAAe;IAC1B;;;OAGG;IACH,SAAS,CAAC,QAAgB;QACxB,OAAO,QAAQ,KAAK,KAAK,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,OAAe,EAAE,OAA4B;QACnD,gDAAgD;QAChD,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,OAAO;gBACV,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAChC,KAAK,MAAM;gBACT,OAAO,OAAO,CAAC,CAAC,iBAAiB;YACnC,KAAK,aAAa;gBAChB,OAAO,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC/C;gBACE,OAAO,CAAC,GAAG,CAAC,gCAAgC,OAAO,CAAC,IAAI,qBAAqB,CAAC,CAAC;gBAC/E,OAAO,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,QAAQ,CAAC,OAAe;QAC9B,IAAI,OAAO,GAAG,OAAO,CAAC;QAEtB,kDAAkD;QAClD,0DAA0D;QAC1D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,4CAA4C,EAAE,EAAE,CAAC,CAAC;QAE5E,oDAAoD;QACpD,iDAAiD;QACjD,iEAAiE;QACjE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,uDAAuD,EAAE,EAAE,CAAC,CAAC;QACvF,kDAAkD;QAClD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,oDAAoD,EAAE,EAAE,CAAC,CAAC;QACpF,mEAAmE;QACnE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,4EAA4E,EAAE,EAAE,CAAC,CAAC;QAC5G,2CAA2C;QAC3C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,oCAAoC,EAAE,EAAE,CAAC,CAAC;QAEpE,kDAAkD;QAClD,mCAAmC;QACnC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC;QAE7D,wEAAwE;QACxE,uEAAuE;QACvE,IAAI,cAAc,CAAC;QACnB,GAAG,CAAC;YACF,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;YAChC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,oDAAoD,EAAE,EAAE,CAAC,CAAC;QACtF,CAAC,QAAQ,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE;QAE5C,kDAAkD;QAClD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QAE1D,yDAAyD;QACzD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACpB,OAAO,mBAAmB,CAAC;QAC7B,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,uBAAuB,CAAC,OAAe;QAC7C,IAAI,OAAO,GAAG,OAAO,CAAC;QAEtB,gCAAgC;QAChC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,4CAA4C,EAAE,kBAAkB,CAAC,CAAC;QAE5F,kCAAkC;QAClC,iEAAiE;QACjE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,uDAAuD,EAAE,kBAAkB,CAAC,CAAC;QACvG,kDAAkD;QAClD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,oDAAoD,EAAE,kBAAkB,CAAC,CAAC;QACpG,mEAAmE;QACnE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,4EAA4E,EAAE,kBAAkB,CAAC,CAAC;QAC5H,2CAA2C;QAC3C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,oCAAoC,EAAE,kBAAkB,CAAC,CAAC;QAEpF,0CAA0C;QAC1C,mCAAmC;QACnC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,6BAA6B,EAAE,qBAAqB,CAAC,CAAC;QAEhF,wEAAwE;QACxE,uEAAuE;QACvE,IAAI,cAAc,CAAC;QACnB,GAAG,CAAC;YACF,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;YAChC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,oDAAoD,EAAE,qBAAqB,CAAC,CAAC;QACzG,CAAC,QAAQ,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE;QAE5C,kDAAkD;QAClD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QAE1D,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}