rag-lite-ts 2.1.0 → 2.2.0

package/dist/esm/text/preprocessors/mermaid.js

@@ -0,0 +1,330 @@
+import { ContentTypeDetector } from './registry.js';
+/**
+ * Mermaid preprocessor for handling Mermaid diagrams in Markdown files
+ * Supports strip, extract, and placeholder modes
+ */
+export class MermaidPreprocessor {
+    /**
+     * Check if this preprocessor applies to the given language/content type
+     * Applies to mermaid code blocks and content with Mermaid syntax
+     */
+    appliesTo(language) {
+        return language === 'mermaid';
+    }
+    /**
+     * Process Mermaid content based on the specified mode
+     */
+    process(content, options) {
+        // Only process if content actually contains Mermaid diagrams
+        if (!ContentTypeDetector.hasMermaidContent(content)) {
+            return content;
+        }
+        switch (options.mode) {
+            case 'strip':
+                return this.stripMermaid(content);
+            case 'extract':
+                return this.extractMermaidEdges(content);
+            case 'placeholder':
+                return this.replaceWithPlaceholders(content);
+            default:
+                console.log(`Unknown Mermaid processing mode: ${options.mode}, using placeholder`);
+                return this.replaceWithPlaceholders(content);
+        }
+    }
+    /**
+     * Strip Mermaid diagrams entirely
+     */
+    stripMermaid(content) {
+        let cleaned = content;
+        // Remove mermaid code blocks: ```mermaid ... ```
+        cleaned = cleaned.replace(/```mermaid[\s\S]*?```/gi, '');
+        // Clean up multiple consecutive newlines and trim
+        cleaned = cleaned.replace(/\n\s*\n\s*\n/g, '\n\n').trim();
+        // Ensure we never return empty content (requirement 6.4)
+        if (!cleaned.trim()) {
+            return '[content removed]';
+        }
+        return cleaned;
+    }
+    /**
+     * Replace Mermaid diagrams with descriptive placeholders
+     */
+    replaceWithPlaceholders(content) {
+        let cleaned = content;
+        // Replace mermaid code blocks with placeholder
+        cleaned = cleaned.replace(/```mermaid[\s\S]*?```/gi, '[diagram removed]');
+        // Clean up multiple consecutive newlines and trim
+        cleaned = cleaned.replace(/\n\s*\n\s*\n/g, '\n\n').trim();
+        return cleaned;
+    }
+    /**
+     * Extract semantic information from Mermaid diagrams
+     * Converts diagram edges to plain text while ignoring styling and layout instructions
+     */
+    extractMermaidEdges(content) {
+        let processed = content;
+        // Process each mermaid code block
+        processed = processed.replace(/```mermaid([\s\S]*?)```/gi, (match, diagramContent) => {
+            const edges = this.extractEdgesFromDiagram(diagramContent);
+            return edges.length > 0 ? edges.join('\n') : '[diagram content extracted]';
+        });
+        return processed;
+    }
+    /**
+     * Extract edges and relationships from a Mermaid diagram
+     * Ignores styling, layout, and formatting instructions
+     */
+    extractEdgesFromDiagram(diagramContent) {
+        const edges = [];
+        const lines = diagramContent.split('\n');
+        // Detect diagram type from first non-empty line
+        let diagramType = 'flowchart'; // default
+        for (const line of lines) {
+            const trimmed = line.trim();
+            if (trimmed && !trimmed.startsWith('%')) {
+                if (trimmed.includes('sequenceDiagram'))
+                    diagramType = 'sequence';
+                else if (trimmed.includes('classDiagram'))
+                    diagramType = 'class';
+                else if (trimmed.includes('stateDiagram'))
+                    diagramType = 'state';
+                else if (trimmed.includes('erDiagram'))
+                    diagramType = 'er';
+                else if (trimmed.includes('graph') || trimmed.includes('flowchart'))
+                    diagramType = 'flowchart';
+                break;
+            }
+        }
+        // For flowcharts, build a node label mapping first
+        const nodeLabelMap = new Map();
+        if (diagramType === 'flowchart') {
+            this.buildNodeLabelMap(lines, nodeLabelMap);
+        }
+        for (const line of lines) {
+            const trimmedLine = line.trim();
+            // Skip empty lines, comments, and diagram type declarations
+            if (!trimmedLine ||
+                trimmedLine.startsWith('%') ||
+                trimmedLine.startsWith('%%') ||
+                this.isLayoutInstruction(trimmedLine) ||
+                this.isStyleInstruction(trimmedLine)) {
+                continue;
+            }
+            // Extract edges from different diagram types
+            const edge = this.extractEdgeFromLine(trimmedLine, diagramType, nodeLabelMap);
+            if (edge) {
+                edges.push(edge);
+            }
+        }
+        return edges;
+    }
+    /**
+     * Build a mapping of node IDs to their labels by scanning all lines
+     */
+    buildNodeLabelMap(lines, nodeLabelMap) {
+        for (const line of lines) {
+            const trimmedLine = line.trim();
+            if (!trimmedLine || trimmedLine.startsWith('%') || trimmedLine.startsWith('%%')) {
+                continue;
+            }
+            // Find all node definitions in the line (both source and target)
+            const regex = /(\w+)(?:\[([^\]]+)\]|\{([^}]+)\}|\(\(([^)]+)\)\)|\(([^)]+)\))/g;
+            let match;
+            while ((match = regex.exec(trimmedLine)) !== null) {
+                const [, nodeId, squareLabel, curlyLabel, doubleParenLabel, parenLabel] = match;
+                const label = squareLabel || curlyLabel || doubleParenLabel || parenLabel;
+                if (label) {
+                    nodeLabelMap.set(nodeId, label.trim());
+                }
+            }
+        }
+    }
+    /**
+     * Check if a line is a layout instruction (should be ignored)
+     */
+    isLayoutInstruction(line) {
+        const layoutPatterns = [
+            /^(graph|flowchart)\s+(TD|TB|BT|RL|LR)/i,
+            /^(sequenceDiagram|classDiagram|stateDiagram|erDiagram|journey|gantt)/i,
+            /^pie\s+title/i,
+            /^direction\s+(TD|TB|BT|RL|LR)/i,
+            /^subgraph/i,
+            /^end$/i
+        ];
+        return layoutPatterns.some(pattern => pattern.test(line));
+    }
+    /**
+     * Check if a line is a styling instruction (should be ignored)
+     */
+    isStyleInstruction(line) {
+        const stylePatterns = [
+            /^classDef\s+/i,
+            /^class\s+.*\s+\w+$/i,
+            /^style\s+/i,
+            /^fill:/i,
+            /^stroke:/i,
+            /^color:/i,
+            /:::.*$/, // CSS class assignments
+            /^linkStyle\s+/i,
+            /^click\s+/i
+        ];
+        return stylePatterns.some(pattern => pattern.test(line));
+    }
+    /**
+     * Extract the meaningful label from a node definition
+     * Examples: A[Start] -> "Start", B{Decision} -> "Decision", C((End)) -> "End", D -> "D"
+     */
+    extractNodeLabel(nodeText, nodeLabelMap) {
+        // Handle square brackets: A[Start]
+        const squareMatch = nodeText.match(/\w+\[([^\]]+)\]/);
+        if (squareMatch) {
+            return squareMatch[1].trim();
+        }
+        // Handle curly braces: B{Decision}
+        const curlyMatch = nodeText.match(/\w+\{([^}]+)\}/);
+        if (curlyMatch) {
+            return curlyMatch[1].trim();
+        }
+        // Handle double parentheses: C((End))
+        const doubleParenMatch = nodeText.match(/\w+\(\(([^)]+)\)\)/);
+        if (doubleParenMatch) {
+            return doubleParenMatch[1].trim();
+        }
+        // Handle single parentheses: D(Text)
+        const parenMatch = nodeText.match(/\w+\(([^)]+)\)/);
+        if (parenMatch) {
+            return parenMatch[1].trim();
+        }
+        // If no inline label found, check the node label map
+        const idMatch = nodeText.match(/(\w+)/);
+        const nodeId = idMatch ? idMatch[1] : nodeText;
+        if (nodeLabelMap && nodeLabelMap.has(nodeId)) {
+            return nodeLabelMap.get(nodeId);
+        }
+        // Return the node ID as fallback
+        return nodeId;
+    }
+    /**
+     * Extract edge information from a single line
+     */
+    extractEdgeFromLine(line, diagramType = 'flowchart', nodeLabelMap) {
+        // Handle based on diagram type
+        if (diagramType === 'sequence') {
+            // Sequence diagram: A->>B: message or A-->>B: message
+            const sequenceMatch = line.match(/(\w+)\s*--?>>?\+?\s*(\w+)\s*:\s*(.+)/);
+            if (sequenceMatch) {
+                const [, from, to, message] = sequenceMatch;
+                return `${from} sends to ${to}: ${message.trim()}`;
+            }
+        }
+        if (diagramType === 'class') {
+            // Class diagram inheritance: Animal --|> Dog
+            const inheritanceMatch = line.match(/(\w+)\s*--\|>\s*(\w+)/);
+            if (inheritanceMatch) {
+                const [, from, to] = inheritanceMatch;
+                return `${from} inherits from ${to}`;
+            }
+            // Class diagram other relationships
+            const classMatch = line.match(/(\w+)\s*(<\|--|--\||<\|--\|>|\*--|--\*|o--|--o)\s*(\w+)/);
+            if (classMatch) {
+                const [, from, connector, to] = classMatch;
+                const relationship = this.interpretClassConnector(connector);
+                return `${from} ${relationship} ${to}`;
+            }
+        }
+        if (diagramType === 'er') {
+            // ER diagram: CUSTOMER ||--o{ ORDER
+            const erMatch = line.match(/(\w+)\s*(\|\|--o\{|\}o--\|\||\|\|--\|\||o\{--\|\|)\s*(\w+)/);
+            if (erMatch) {
+                const [, from, connector, to] = erMatch;
+                const relationship = this.interpretERConnector(connector);
+                return `${from} ${relationship} ${to}`;
+            }
+        }
+        if (diagramType === 'state') {
+            // State diagram: state1 --> state2 (but not [*] --> state)
+            const stateMatch = line.match(/^(?!\s*\[\*\])\s*(\w+)\s*-->\s*(\w+)$/);
+            if (stateMatch) {
+                const [, from, to] = stateMatch;
+                return `${from} transitions to ${to}`;
+            }
+        }
+        if (diagramType === 'flowchart') {
+            // Flowchart/Graph edges with labels: A --> B | label | (not A -->|label| B)
+            const flowchartLabelMatch = line.match(/(\w+(?:\[[^\]]+\]|\{[^}]+\}|\(\([^)]+\)\)|\([^)]+\))?)\s*(-->|---|-.->|-\.-|->|--)\s+(\w+(?:\[[^\]]+\]|\{[^}]+\}|\(\([^)]+\)\)|\([^)]+\))?)\s*\|\s*([^|]+)\s*\|/);
+            if (flowchartLabelMatch) {
+                const [, fromNode, connector, toNode, edgeLabel] = flowchartLabelMatch;
+                const from = this.extractNodeLabel(fromNode, nodeLabelMap);
+                const to = this.extractNodeLabel(toNode, nodeLabelMap);
+                const relationship = this.interpretConnector(connector);
+                return `${from} ${relationship} ${to} (${edgeLabel.trim()})`;
+            }
+            // Alternative label format: A -->|label| B (handle nodes with spaces in labels)
+            const flowchartLabelMatch2 = line.match(/(\w+(?:\[[^\]]+\]|\{[^}]+\}|\(\([^)]+\)\)|\([^)]+\))?)\s*(-->|---|-.->|-\.-|->|--)\|([^|]+)\|\s*(.+)/);
+            if (flowchartLabelMatch2) {
+                const [, fromNode, connector, edgeLabel, toNode] = flowchartLabelMatch2;
+                const from = this.extractNodeLabel(fromNode, nodeLabelMap);
+                const to = this.extractNodeLabel(toNode.trim(), nodeLabelMap);
+                const relationship = this.interpretConnector(connector);
+                return `${from} ${relationship} ${to} (${edgeLabel.trim()})`;
+            }
+            // Flowchart/Graph edges: A --> B, A --- B, A -.-> B, etc. (with optional node styling)
+            const flowchartEdgeMatch = line.match(/(\w+(?:\[[^\]]+\]|\{[^}]+\}|\(\([^)]+\)\)|\([^)]+\))?)\s*(-->|---|-.->|-\.-|->|--)\s*(.+)/);
+            if (flowchartEdgeMatch) {
+                const [, fromNode, connector, toNode] = flowchartEdgeMatch;
+                // Skip if this looks like a label format (contains |)
+                if (toNode.includes('|')) {
+                    return null;
+                }
+                const from = this.extractNodeLabel(fromNode, nodeLabelMap);
+                const to = this.extractNodeLabel(toNode.trim(), nodeLabelMap);
+                const relationship = this.interpretConnector(connector);
+                return `${from} ${relationship} ${to}`;
+            }
+        }
+        return null;
+    }
+    /**
+     * Interpret flowchart connector symbols
+     */
+    interpretConnector(connector) {
+        if (connector.includes('-->'))
+            return 'leads to';
+        if (connector.includes('-.->'))
+            return 'optionally leads to';
+        if (connector.includes('---'))
+            return 'connects to';
+        if (connector.includes('-.-'))
+            return 'optionally connects to';
+        return 'relates to';
+    }
+    /**
+     * Interpret class diagram connector symbols
+     */
+    interpretClassConnector(connector) {
+        switch (connector) {
+            case '--|>': return 'inherits from';
+            case '<|--': return 'is inherited by';
+            case '--||': return 'implements';
+            case '<|--|>': return 'has bidirectional inheritance with';
+            case '*--': return 'composes';
+            case '--*': return 'is composed by';
+            case 'o--': return 'aggregates';
+            case '--o': return 'is aggregated by';
+            default: return 'relates to';
+        }
+    }
+    /**
+     * Interpret ER diagram connector symbols
+     */
+    interpretERConnector(connector) {
+        switch (connector) {
+            case '||--o{': return 'has one-to-many relationship with';
+            case '}o--||': return 'has many-to-one relationship with';
+            case '||--||': return 'has one-to-one relationship with';
+            case 'o{--||': return 'has many-to-one relationship with';
+            default: return 'has relationship with';
+        }
+    }
+}
+//# sourceMappingURL=mermaid.js.map

package/dist/esm/text/preprocessors/registry.d.ts

@@ -0,0 +1,56 @@
+import { Preprocessor } from '../../types.js';
+/**
+ * Registry for managing preprocessors and content type detection
+ */
+export declare class PreprocessorRegistry {
+    private preprocessors;
+    /**
+     * Register a preprocessor with a given name
+     */
+    register(name: string, preprocessor: Preprocessor): void;
+    /**
+     * Get a specific preprocessor by name
+     */
+    get(name: string): Preprocessor | undefined;
+    /**
+     * Get all preprocessors that apply to the given language/content type
+     */
+    getApplicable(language: string): Preprocessor[];
+    /**
+     * Get all registered preprocessor names
+     */
+    getRegisteredNames(): string[];
+    /**
+     * Validate that all required preprocessors are available
+     */
+    validatePreprocessors(requiredNames: string[]): {
+        valid: boolean;
+        missing: string[];
+    };
+}
+/**
+ * Content type detection utilities
+ */
+export declare class ContentTypeDetector {
+    /**
+     * Detect content type from file extension
+     */
+    static detectFromExtension(filePath: string): string | null;
+    /**
+     * Detect content type from code fence language identifier
+     */
+    static detectFromCodeFence(language: string): string;
+    /**
+     * Detect JSX content in text
+     */
+    static hasJsxContent(content: string): boolean;
+    /**
+     * Detect Mermaid diagram content
+     */
+    static hasMermaidContent(content: string): boolean;
+    /**
+     * Extract code fence language from a code block
+     */
+    static extractCodeFenceLanguage(codeBlock: string): string | null;
+}
+//# sourceMappingURL=registry.d.ts.map

package/dist/esm/text/preprocessors/registry.js

@@ -0,0 +1,180 @@
+/**
+ * Registry for managing preprocessors and content type detection
+ */
+export class PreprocessorRegistry {
+    preprocessors = new Map();
+    /**
+     * Register a preprocessor with a given name
+     */
+    register(name, preprocessor) {
+        this.preprocessors.set(name, preprocessor);
+    }
+    /**
+     * Get a specific preprocessor by name
+     */
+    get(name) {
+        return this.preprocessors.get(name);
+    }
+    /**
+     * Get all preprocessors that apply to the given language/content type
+     */
+    getApplicable(language) {
+        const applicable = [];
+        const preprocessors = Array.from(this.preprocessors.values());
+        for (const preprocessor of preprocessors) {
+            if (preprocessor.appliesTo(language)) {
+                applicable.push(preprocessor);
+            }
+        }
+        return applicable;
+    }
+    /**
+     * Get all registered preprocessor names
+     */
+    getRegisteredNames() {
+        return Array.from(this.preprocessors.keys());
+    }
+    /**
+     * Validate that all required preprocessors are available
+     */
+    validatePreprocessors(requiredNames) {
+        const missing = requiredNames.filter(name => !this.preprocessors.has(name));
+        return {
+            valid: missing.length === 0,
+            missing
+        };
+    }
+}
+/**
+ * Content type detection utilities
+ */
+export class ContentTypeDetector {
+    /**
+     * Detect content type from file extension
+     */
+    static detectFromExtension(filePath) {
+        const extension = filePath.toLowerCase().split('.').pop();
+        switch (extension) {
+            case 'mdx':
+                return 'mdx';
+            case 'md':
+                return 'markdown';
+            case 'js':
+            case 'jsx':
+                return 'javascript';
+            case 'ts':
+            case 'tsx':
+                return 'typescript';
+            case 'py':
+                return 'python';
+            case 'java':
+                return 'java';
+            case 'cpp':
+            case 'cc':
+            case 'cxx':
+                return 'cpp';
+            case 'c':
+                return 'c';
+            case 'cs':
+                return 'csharp';
+            case 'php':
+                return 'php';
+            case 'rb':
+                return 'ruby';
+            case 'go':
+                return 'go';
+            case 'rs':
+                return 'rust';
+            case 'swift':
+                return 'swift';
+            case 'kt':
+                return 'kotlin';
+            case 'scala':
+                return 'scala';
+            case 'sh':
+            case 'bash':
+                return 'bash';
+            case 'ps1':
+                return 'powershell';
+            case 'sql':
+                return 'sql';
+            case 'html':
+                return 'html';
+            case 'css':
+                return 'css';
+            case 'json':
+                return 'json';
+            case 'xml':
+                return 'xml';
+            case 'yaml':
+            case 'yml':
+                return 'yaml';
+            default:
+                return null;
+        }
+    }
+    /**
+     * Detect content type from code fence language identifier
+     */
+    static detectFromCodeFence(language) {
+        // Normalize language identifier
+        const normalized = language.toLowerCase().trim();
+        // Handle common aliases
+        switch (normalized) {
+            case 'js':
+                return 'javascript';
+            case 'ts':
+                return 'typescript';
+            case 'py':
+                return 'python';
+            case 'sh':
+            case 'shell':
+                return 'bash';
+            case 'yml':
+                return 'yaml';
+            default:
+                return normalized;
+        }
+    }
+    /**
+     * Detect JSX content in text
+     */
+    static hasJsxContent(content) {
+        // Look for JSX patterns: <Component>, <div className=, etc.
+        const jsxPatterns = [
+            /<[A-Z][a-zA-Z0-9]*(?:\s[^>]*)?\/?>/, // Component tags
+            /<[a-z]+\s+className=/, // className attribute
+            /\{[^}]*\}/, // JSX expressions
+            /import\s+.*\s+from\s+['"][^'"]*['"]/, // ES6 imports
+            /export\s+(default\s+)?/ // ES6 exports
+        ];
+        return jsxPatterns.some(pattern => pattern.test(content));
+    }
+    /**
+     * Detect Mermaid diagram content
+     */
+    static hasMermaidContent(content) {
+        // Look for mermaid code blocks or mermaid keywords
+        const mermaidPatterns = [
+            /```mermaid/i,
+            /graph\s+(TD|TB|BT|RL|LR)/i,
+            /flowchart\s+(TD|TB|BT|RL|LR)/i,
+            /sequenceDiagram/i,
+            /classDiagram/i,
+            /stateDiagram/i,
+            /erDiagram/i,
+            /journey/i,
+            /gantt/i,
+            /pie\s+title/i
+        ];
+        return mermaidPatterns.some(pattern => pattern.test(content));
+    }
+    /**
+     * Extract code fence language from a code block
+     */
+    static extractCodeFenceLanguage(codeBlock) {
+        const match = codeBlock.match(/^```(\w+)/);
+        return match ? this.detectFromCodeFence(match[1]) : null;
+    }
+}
+//# sourceMappingURL=registry.js.map

package/dist/esm/text/reranker.d.ts

@@ -0,0 +1,49 @@
+import '../dom-polyfills.js';
+import type { SearchResult, RerankFunction } from '../core/types.js';
+/**
+ * Embedding-based reranker for improving search result quality
+ * Uses embedding similarity to rerank initial vector search results
+ */
+export declare class CrossEncoderReranker {
+    private model;
+    private tokenizer;
+    private modelName;
+    /**
+     * Ensure DOM polyfills are set up for transformers.js
+     */
+    private ensurePolyfills;
+    /**
+     * Load the embedding model
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Try to load a specific model
+     */
+    private tryLoadModel;
+    /**
+     * Rerank search results using embedding similarity scoring
+     * @param query - Original search query
+     * @param results - Initial search results from vector search
+     * @returns Promise resolving to reranked results
+     */
+    rerank(query: string, results: SearchResult[]): Promise<SearchResult[]>;
+    /**
+     * Check if the model is loaded
+     */
+    isLoaded(): boolean;
+    /**
+     * Get the model name being used
+     */
+    getModelName(): string;
+    /**
+     * Compute cosine similarity between two Float32Array embeddings
+     */
+    private cosineSimilarity;
+    /**
+     * Simple text-based reranking using keyword matching and text similarity
+     * This is a fallback when embedding similarity doesn't provide good discrimination
+     */
+    private simpleTextReranking;
+}
+export declare function createTextRerankFunction(modelName?: string): RerankFunction;
+//# sourceMappingURL=reranker.d.ts.map