rag-lite-ts 2.1.1 → 2.3.0

package/dist/esm/multimodal/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Multimodal implementation layer exports
+ * Provides CLIP-based embedders for cross-modal search capabilities
+ */
+export { CLIPEmbedder } from './clip-embedder.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/esm/multimodal/index.js

@@ -0,0 +1,6 @@
+/**
+ * Multimodal implementation layer exports
+ * Provides CLIP-based embedders for cross-modal search capabilities
+ */
+export { CLIPEmbedder } from './clip-embedder.js';
+//# sourceMappingURL=index.js.map

package/dist/esm/preprocess.d.ts

@@ -0,0 +1,19 @@
+import { PreprocessingConfig } from './types.js';
+/**
+ * Main preprocessing function that processes document content based on configuration
+ * @param content - Raw document content to preprocess
+ * @param filePath - Path to the file being processed (for content type detection)
+ * @param config - Preprocessing configuration
+ * @returns Processed content string
+ */
+export declare function preprocessDocument(content: string, filePath: string, config: PreprocessingConfig): string;
+/**
+ * Utility function to get preprocessing statistics for debugging
+ */
+export declare function getPreprocessingStats(originalContent: string, processedContent: string): {
+    originalLength: number;
+    processedLength: number;
+    reductionRatio: number;
+    linesRemoved: number;
+};
+//# sourceMappingURL=preprocess.d.ts.map

package/dist/esm/preprocess.js

@@ -0,0 +1,203 @@
+import { mergePreprocessingConfig, validatePreprocessingConfig } from './core/config.js';
+import { preprocessorRegistry, ContentTypeDetector } from './preprocessors/index.js';
+/**
+ * Main preprocessing function that processes document content based on configuration
+ * @param content - Raw document content to preprocess
+ * @param filePath - Path to the file being processed (for content type detection)
+ * @param config - Preprocessing configuration
+ * @returns Processed content string
+ */
+export function preprocessDocument(content, filePath, config) {
+    try {
+        // Validate configuration
+        validatePreprocessingConfig(config);
+        // Resolve mode configuration with overrides
+        const resolvedConfig = mergePreprocessingConfig(config);
+        // Detect content type from file path
+        const contentType = detectContentType(content, filePath);
+        // Early return if no preprocessing needed
+        if (!needsPreprocessing(content, contentType)) {
+            return content;
+        }
+        // Process content based on detected type
+        let processedContent = content;
+        try {
+            processedContent = processContentByType(content, contentType, resolvedConfig);
+        }
+        catch (error) {
+            // Fallback handling for processing failures
+            console.error(`Preprocessing failed for ${filePath}: ${error instanceof Error ? error.message : String(error)}`);
+            processedContent = handleProcessingFailure(content, contentType);
+        }
+        // Ensure processed content is never empty
+        if (!processedContent || processedContent.trim().length === 0) {
+            console.log(`Warning: Preprocessing resulted in empty content for ${filePath}, using fallback`);
+            processedContent = '[content processed but empty]';
+        }
+        return processedContent;
+    }
+    catch (error) {
+        // Configuration or other critical errors - use fallback
+        console.error(`Critical preprocessing error for ${filePath}: ${error instanceof Error ? error.message : String(error)}`);
+        return handleCriticalFailure(content);
+    }
+}
+/**
+ * Detect content type from file path and content analysis
+ */
+function detectContentType(content, filePath) {
+    // First try file extension detection
+    const extensionType = ContentTypeDetector.detectFromExtension(filePath);
+    if (extensionType) {
+        return extensionType;
+    }
+    // Fallback to content analysis
+    if (ContentTypeDetector.hasJsxContent(content)) {
+        return 'mdx';
+    }
+    if (ContentTypeDetector.hasMermaidContent(content)) {
+        return 'mermaid';
+    }
+    // Default to markdown for unknown types
+    return 'markdown';
+}
+/**
+ * Check if content needs preprocessing based on type and content analysis
+ */
+function needsPreprocessing(content, contentType) {
+    // Always preprocess MDX files
+    if (contentType === 'mdx') {
+        return true;
+    }
+    // Check for Mermaid diagrams in any markdown content
+    if (ContentTypeDetector.hasMermaidContent(content)) {
+        return true;
+    }
+    // Check for JSX content in markdown files
+    if (contentType === 'markdown' && ContentTypeDetector.hasJsxContent(content)) {
+        return true;
+    }
+    // Check for code blocks that might need processing
+    if (content.includes('```')) {
+        return true;
+    }
+    return false;
+}
+/**
+ * Process content based on detected content type and configuration
+ */
+function processContentByType(content, contentType, resolvedConfig) {
+    let processedContent = content;
+    // Process MDX content if present
+    if (contentType === 'mdx' || ContentTypeDetector.hasJsxContent(content)) {
+        processedContent = processWithPreprocessor(processedContent, 'mdx', { mode: resolvedConfig.mdx });
+    }
+    // Process Mermaid diagrams if present (check original content, not processed)
+    if (ContentTypeDetector.hasMermaidContent(content)) {
+        processedContent = processWithPreprocessor(processedContent, 'mermaid', { mode: resolvedConfig.mermaid });
+    }
+    // Process code blocks if present and not already handled by MDX processor
+    if (processedContent.includes('```')) {
+        processedContent = processCodeBlocks(processedContent, resolvedConfig.code);
+    }
+    return processedContent;
+}
+/**
+ * Process content with a specific preprocessor
+ */
+function processWithPreprocessor(content, preprocessorName, options) {
+    const preprocessor = preprocessorRegistry.get(preprocessorName);
+    if (!preprocessor) {
+        console.log(`Unknown preprocessor: ${preprocessorName}, using fallback`);
+        return handleUnknownContentType(content, preprocessorName);
+    }
+    try {
+        // Call the preprocessor directly - it will check if it needs to process the content
+        return preprocessor.process(content, options);
+    }
+    catch (error) {
+        console.error(`Preprocessor ${preprocessorName} failed: ${error instanceof Error ? error.message : String(error)}`);
+        return handleUnknownContentType(content, preprocessorName);
+    }
+}
+/**
+ * Process code blocks based on configuration
+ */
+function processCodeBlocks(content, mode) {
+    if (mode === 'keep') {
+        return content;
+    }
+    // Find all code blocks
+    const codeBlockRegex = /```[\s\S]*?```/g;
+    if (mode === 'strip') {
+        return content.replace(codeBlockRegex, '');
+    }
+    if (mode === 'placeholder') {
+        return content.replace(codeBlockRegex, (match) => {
+            const language = ContentTypeDetector.extractCodeFenceLanguage(match);
+            return language ? `[${language} code block removed]` : '[code block removed]';
+        });
+    }
+    return content;
+}
+/**
+ * Handle unknown content types with fallback
+ */
+function handleUnknownContentType(content, contentType) {
+    console.log(`Unknown content type detected: ${contentType}, using fallback`);
+    return '[content removed]';
+}
+/**
+ * Handle processing failures with graceful fallback
+ */
+function handleProcessingFailure(content, contentType) {
+    console.log(`Processing failed for content type: ${contentType}, using placeholder`);
+    // Try to preserve basic structure while removing problematic content
+    switch (contentType) {
+        case 'mdx':
+            return '[MDX content removed due to processing error]';
+        case 'mermaid':
+            return '[diagram removed due to processing error]';
+        default:
+            return '[content removed due to processing error]';
+    }
+}
+/**
+ * Handle critical failures that prevent any processing
+ */
+function handleCriticalFailure(content) {
+    console.error('Critical preprocessing failure, returning minimal fallback');
+    // For plain content without problematic syntax, return as-is
+    if (!content.includes('<') && !content.includes('```') &&
+        !content.includes('import ') && !content.includes('export ')) {
+        return content;
+    }
+    // Return a safe fallback that preserves some content structure
+    const lines = content.split('\n');
+    const safeLines = lines.filter(line => {
+        // Keep lines that look like plain text or basic markdown
+        return line.trim().length > 0 &&
+            !line.includes('<') &&
+            !line.includes('```') &&
+            !line.startsWith('import ') &&
+            !line.startsWith('export ');
+    });
+    if (safeLines.length > 0) {
+        return safeLines.join('\n');
+    }
+    return '[content could not be processed safely]';
+}
+/**
+ * Utility function to get preprocessing statistics for debugging
+ */
+export function getPreprocessingStats(originalContent, processedContent) {
+    const originalLines = originalContent.split('\n').length;
+    const processedLines = processedContent.split('\n').length;
+    return {
+        originalLength: originalContent.length,
+        processedLength: processedContent.length,
+        reductionRatio: processedContent.length / originalContent.length,
+        linesRemoved: originalLines - processedLines
+    };
+}
+//# sourceMappingURL=preprocess.js.map

package/dist/esm/preprocessors/index.d.ts

@@ -0,0 +1,17 @@
+import { PreprocessorRegistry } from './registry.js';
+export { PreprocessorRegistry, ContentTypeDetector } from './registry.js';
+export { MdxPreprocessor } from './mdx.js';
+export { MermaidPreprocessor } from './mermaid.js';
+/**
+ * Global preprocessor registry instance
+ */
+export declare const preprocessorRegistry: PreprocessorRegistry;
+/**
+ * Validate that all required preprocessors are available in the registry
+ */
+export declare function validatePreprocessorConfiguration(requiredPreprocessors: string[]): void;
+/**
+ * Get all available preprocessor names
+ */
+export declare function getAvailablePreprocessors(): string[];
+//# sourceMappingURL=index.d.ts.map

package/dist/esm/preprocessors/index.js

@@ -0,0 +1,38 @@
+import { PreprocessorRegistry } from './registry.js';
+import { MdxPreprocessor } from './mdx.js';
+import { MermaidPreprocessor } from './mermaid.js';
+// Export all preprocessor classes
+export { PreprocessorRegistry, ContentTypeDetector } from './registry.js';
+export { MdxPreprocessor } from './mdx.js';
+export { MermaidPreprocessor } from './mermaid.js';
+/**
+ * Create and initialize the global preprocessor registry
+ */
+function createPreprocessorRegistry() {
+    const registry = new PreprocessorRegistry();
+    // Register built-in preprocessors
+    registry.register('mdx', new MdxPreprocessor());
+    registry.register('mermaid', new MermaidPreprocessor());
+    return registry;
+}
+/**
+ * Global preprocessor registry instance
+ */
+export const preprocessorRegistry = createPreprocessorRegistry();
+/**
+ * Validate that all required preprocessors are available in the registry
+ */
+export function validatePreprocessorConfiguration(requiredPreprocessors) {
+    const validation = preprocessorRegistry.validatePreprocessors(requiredPreprocessors);
+    if (!validation.valid) {
+        const missingList = validation.missing.join(', ');
+        throw new Error(`Missing required preprocessors: ${missingList}. Available: ${preprocessorRegistry.getRegisteredNames().join(', ')}`);
+    }
+}
+/**
+ * Get all available preprocessor names
+ */
+export function getAvailablePreprocessors() {
+    return preprocessorRegistry.getRegisteredNames();
+}
+//# sourceMappingURL=index.js.map

package/dist/esm/preprocessors/mdx.d.ts

@@ -0,0 +1,25 @@
+import { Preprocessor, PreprocessorOptions } from '../types.js';
+/**
+ * MDX preprocessor for handling JSX content in Markdown files
+ * Ports the existing cleanMdxContent logic with mode-aware behavior
+ */
+export declare class MdxPreprocessor implements Preprocessor {
+    /**
+     * Check if this preprocessor applies to the given language/content type
+     * Applies to .mdx files and content with JSX syntax
+     */
+    appliesTo(language: string): boolean;
+    /**
+     * Process MDX content based on the specified mode
+     */
+    process(content: string, options: PreprocessorOptions): string;
+    /**
+     * Strip JSX content entirely - ported from cleanMdxContent logic
+     */
+    private stripJsx;
+    /**
+     * Replace JSX with descriptive placeholders
+     */
+    private replaceWithPlaceholders;
+}
+//# sourceMappingURL=mdx.d.ts.map

package/dist/esm/preprocessors/mdx.js

@@ -0,0 +1,101 @@
+import { ContentTypeDetector } from './registry.js';
+/**
+ * MDX preprocessor for handling JSX content in Markdown files
+ * Ports the existing cleanMdxContent logic with mode-aware behavior
+ */
+export class MdxPreprocessor {
+    /**
+     * Check if this preprocessor applies to the given language/content type
+     * Applies to .mdx files and content with JSX syntax
+     */
+    appliesTo(language) {
+        return language === 'mdx';
+    }
+    /**
+     * Process MDX content based on the specified mode
+     */
+    process(content, options) {
+        // Only process if content actually contains JSX
+        if (!ContentTypeDetector.hasJsxContent(content)) {
+            return content;
+        }
+        switch (options.mode) {
+            case 'strip':
+                return this.stripJsx(content);
+            case 'keep':
+                return content; // Keep JSX as-is
+            case 'placeholder':
+                return this.replaceWithPlaceholders(content);
+            default:
+                console.log(`Unknown MDX processing mode: ${options.mode}, using placeholder`);
+                return this.replaceWithPlaceholders(content);
+        }
+    }
+    /**
+     * Strip JSX content entirely - ported from cleanMdxContent logic
+     */
+    stripJsx(content) {
+        let cleaned = content;
+        // Remove JSX import statements (requirement 11.1)
+        // Matches: import ... from '...' or import ... from "..."
+        cleaned = cleaned.replace(/^import\s+.*?from\s+['"][^'"]*['"];?\s*$/gm, '');
+        // Remove JSX export statements (requirement 11.2)  
+        // Handle both single-line and multi-line exports
+        // Multi-line function exports: export default function() { ... }
+        cleaned = cleaned.replace(/^export\s+default\s+function[^{]*\{[\s\S]*?\n\}\s*$/gm, '');
+        // Object exports: export const metadata = { ... }
+        cleaned = cleaned.replace(/^export\s+const\s+[^=]*=\s*\{[\s\S]*?\}\s*;?\s*$/gm, '');
+        // Single line exports: export const x = ...; or export default ...
+        cleaned = cleaned.replace(/^export\s+(?:default\s+)?(?:const|let|var|function|class)\s+[^;{]*;?\s*$/gm, '');
+        // Simple exports: export default Component
+        cleaned = cleaned.replace(/^export\s+default\s+[^;{]*;?\s*$/gm, '');
+        // Remove JSX components (requirements 11.3, 11.4)
+        // Self-closing tags: <Component />
+        cleaned = cleaned.replace(/<[A-Z][a-zA-Z0-9]*[^>]*\/>/g, '');
+        // Opening and closing tags with content: <Component>content</Component>
+        // This handles nested components by replacing the outermost ones first
+        let previousLength;
+        do {
+            previousLength = cleaned.length;
+            cleaned = cleaned.replace(/<[A-Z][a-zA-Z0-9]*[^>]*>.*?<\/[A-Z][a-zA-Z0-9]*>/gs, '');
+        } while (cleaned.length !== previousLength);
+        // Clean up multiple consecutive newlines and trim
+        cleaned = cleaned.replace(/\n\s*\n\s*\n/g, '\n\n').trim();
+        // Ensure we never return empty content (requirement 6.4)
+        if (!cleaned.trim()) {
+            return '[content removed]';
+        }
+        return cleaned;
+    }
+    /**
+     * Replace JSX with descriptive placeholders
+     */
+    replaceWithPlaceholders(content) {
+        let cleaned = content;
+        // Replace JSX import statements
+        cleaned = cleaned.replace(/^import\s+.*?from\s+['"][^'"]*['"];?\s*$/gm, '[import removed]');
+        // Replace JSX export statements  
+        // Multi-line function exports: export default function() { ... }
+        cleaned = cleaned.replace(/^export\s+default\s+function[^{]*\{[\s\S]*?\n\}\s*$/gm, '[export removed]');
+        // Object exports: export const metadata = { ... }
+        cleaned = cleaned.replace(/^export\s+const\s+[^=]*=\s*\{[\s\S]*?\}\s*;?\s*$/gm, '[export removed]');
+        // Single line exports: export const x = ...; or export default ...
+        cleaned = cleaned.replace(/^export\s+(?:default\s+)?(?:const|let|var|function|class)\s+[^;{]*;?\s*$/gm, '[export removed]');
+        // Simple exports: export default Component
+        cleaned = cleaned.replace(/^export\s+default\s+[^;{]*;?\s*$/gm, '[export removed]');
+        // Replace JSX components with placeholder
+        // Self-closing tags: <Component />
+        cleaned = cleaned.replace(/<[A-Z][a-zA-Z0-9]*[^>]*\/>/g, '[component removed]');
+        // Opening and closing tags with content: <Component>content</Component>
+        // This handles nested components by replacing the outermost ones first
+        let previousLength;
+        do {
+            previousLength = cleaned.length;
+            cleaned = cleaned.replace(/<[A-Z][a-zA-Z0-9]*[^>]*>.*?<\/[A-Z][a-zA-Z0-9]*>/gs, '[component removed]');
+        } while (cleaned.length !== previousLength);
+        // Clean up multiple consecutive newlines and trim
+        cleaned = cleaned.replace(/\n\s*\n\s*\n/g, '\n\n').trim();
+        return cleaned;
+    }
+}
+//# sourceMappingURL=mdx.js.map

package/dist/esm/preprocessors/mermaid.d.ts

@@ -0,0 +1,68 @@
+import { Preprocessor, PreprocessorOptions } from '../types.js';
+/**
+ * Mermaid preprocessor for handling Mermaid diagrams in Markdown files
+ * Supports strip, extract, and placeholder modes
+ */
+export declare class MermaidPreprocessor implements Preprocessor {
+    /**
+     * Check if this preprocessor applies to the given language/content type
+     * Applies to mermaid code blocks and content with Mermaid syntax
+     */
+    appliesTo(language: string): boolean;
+    /**
+     * Process Mermaid content based on the specified mode
+     */
+    process(content: string, options: PreprocessorOptions): string;
+    /**
+     * Strip Mermaid diagrams entirely
+     */
+    private stripMermaid;
+    /**
+     * Replace Mermaid diagrams with descriptive placeholders
+     */
+    private replaceWithPlaceholders;
+    /**
+     * Extract semantic information from Mermaid diagrams
+     * Converts diagram edges to plain text while ignoring styling and layout instructions
+     */
+    private extractMermaidEdges;
+    /**
+     * Extract edges and relationships from a Mermaid diagram
+     * Ignores styling, layout, and formatting instructions
+     */
+    private extractEdgesFromDiagram;
+    /**
+     * Build a mapping of node IDs to their labels by scanning all lines
+     */
+    private buildNodeLabelMap;
+    /**
+     * Check if a line is a layout instruction (should be ignored)
+     */
+    private isLayoutInstruction;
+    /**
+     * Check if a line is a styling instruction (should be ignored)
+     */
+    private isStyleInstruction;
+    /**
+     * Extract the meaningful label from a node definition
+     * Examples: A[Start] -> "Start", B{Decision} -> "Decision", C((End)) -> "End", D -> "D"
+     */
+    private extractNodeLabel;
+    /**
+     * Extract edge information from a single line
+     */
+    private extractEdgeFromLine;
+    /**
+     * Interpret flowchart connector symbols
+     */
+    private interpretConnector;
+    /**
+     * Interpret class diagram connector symbols
+     */
+    private interpretClassConnector;
+    /**
+     * Interpret ER diagram connector symbols
+     */
+    private interpretERConnector;
+}
+//# sourceMappingURL=mermaid.d.ts.map