signalwire-agents 0.1.13__py3-none-any.whl → 1.0.17.dev4__py3-none-any.whl

signalwire_agents/search/__init__.py

@@ -68,6 +68,8 @@ if _SEARCH_AVAILABLE:
         from .index_builder import IndexBuilder
         from .search_engine import SearchEngine
         from .search_service import SearchService
+        from .models import MODEL_ALIASES, DEFAULT_MODEL, resolve_model_alias
+        from .migration import SearchIndexMigrator
         
         __all__ = [
             'preprocess_query',
@@ -75,7 +77,11 @@ if _SEARCH_AVAILABLE:
             'DocumentProcessor',
             'IndexBuilder',
             'SearchEngine',
-            'SearchService'
+            'SearchService',
+            'MODEL_ALIASES',
+            'DEFAULT_MODEL',
+            'resolve_model_alias',
+            'SearchIndexMigrator'
         ]
     except ImportError as e:
         # Some search components failed to import

signalwire_agents/search/document_processor.py

@@ -74,29 +74,51 @@ logger = logging.getLogger(__name__)
 class DocumentProcessor:
     """Enhanced document processor with smart chunking capabilities"""
     
-    def __init__(self, chunking_strategy: str = 'sentence', 
-                 max_sentences_per_chunk: int = 50,
-                 chunk_size: int = 50, 
-                 overlap_size: int = 10,
-                 split_newlines: Optional[int] = None):
+    def __init__(
+        self,
+        chunking_strategy: str = 'sentence',
+        max_sentences_per_chunk: int = 5,
+        chunk_size: int = 50,
+        chunk_overlap: int = 10,
+        split_newlines: Optional[int] = None,
+        index_nlp_backend: str = 'nltk',
+        verbose: bool = False,
+        semantic_threshold: float = 0.5,
+        topic_threshold: float = 0.3
+    ):
         """
-        Initialize document processor with chunking strategy
-        
+        Initialize document processor
+
         Args:
-            chunking_strategy: 'sentence', 'sliding', 'paragraph', or 'page'
-            max_sentences_per_chunk: For sentence strategy (default: 50)
+            chunking_strategy: Strategy for chunking documents:
+                - 'sentence': Sentence-based chunking with overlap
+                - 'sliding': Sliding window with word-based chunks
+                - 'paragraph': Natural paragraph boundaries
+                - 'page': Page-based chunking (for PDFs)
+                - 'semantic': Semantic similarity-based chunking
+                - 'topic': Topic modeling-based chunking
+                - 'qa': Question-answer optimized chunking
+                - 'json': JSON structure-aware chunking
+                - 'markdown': Markdown structure-aware chunking with code block detection
+            max_sentences_per_chunk: For sentence strategy (default: 5)
             chunk_size: For sliding strategy - words per chunk (default: 50)
-            overlap_size: For sliding strategy - overlap in words (default: 10)
+            chunk_overlap: For sliding strategy - overlap in words (default: 10)
             split_newlines: For sentence strategy - split on multiple newlines (optional)
+            index_nlp_backend: NLP backend for indexing (default: 'nltk')
+            verbose: Whether to enable verbose logging (default: False)
+            semantic_threshold: Similarity threshold for semantic chunking (default: 0.5)
+            topic_threshold: Similarity threshold for topic chunking (default: 0.3)
         """
         self.chunking_strategy = chunking_strategy
         self.max_sentences_per_chunk = max_sentences_per_chunk
         self.chunk_size = chunk_size
-        self.overlap_size = overlap_size
+        self.chunk_overlap = chunk_overlap
         self.split_newlines = split_newlines
+        self.semantic_threshold = semantic_threshold
+        self.topic_threshold = topic_threshold
         
         # Legacy support for old character-based chunking
-        self.chunk_overlap = overlap_size
+        self.chunk_overlap = chunk_overlap
     
     def create_chunks(self, content: str, filename: str, 
                      file_type: str) -> List[Dict[str, Any]]:
@@ -121,6 +143,17 @@ class DocumentProcessor:
             return self._chunk_by_paragraphs(content, filename, file_type)
         elif self.chunking_strategy == 'page':
             return self._chunk_by_pages(content, filename, file_type)
+        elif self.chunking_strategy == 'semantic':
+            return self._chunk_by_semantic(content, filename, file_type)
+        elif self.chunking_strategy == 'topic':
+            return self._chunk_by_topics(content, filename, file_type)
+        elif self.chunking_strategy == 'qa':
+            return self._chunk_by_qa_optimization(content, filename, file_type)
+        elif self.chunking_strategy == 'json':
+            return self._chunk_from_json(content, filename, file_type)
+        elif self.chunking_strategy == 'markdown':
+            # Use markdown-aware chunking for better structure preservation
+            return self._chunk_markdown_enhanced(content, filename)
         else:
             # Fallback to sentence-based chunking
             return self._chunk_by_sentences(content, filename, file_type)
@@ -318,75 +351,114 @@ class DocumentProcessor:
         return chunks
     
     def _chunk_markdown_enhanced(self, content: str, filename: str) -> List[Dict[str, Any]]:
-        """Enhanced markdown chunking with better header handling"""
+        """Enhanced markdown chunking with code block detection and rich metadata
+
+        Features:
+        - Tracks header hierarchy for section paths
+        - Detects code blocks and extracts language
+        - Adds 'code' tags to chunks containing code
+        - Preserves markdown structure for better search
+        """
         chunks = []
         lines = content.split('\n')
-        
+
         current_section = None
         current_hierarchy = []  # Track header hierarchy
         current_chunk = []
         current_size = 0
         line_start = 1
-        
+        in_code_block = False
+        code_languages = []  # Track languages in current chunk
+        has_code = False
+
         for line_num, line in enumerate(lines, 1):
+            # Check for code block fences
+            code_fence_match = re.match(r'^```(\w+)?', line)
+            if code_fence_match:
+                in_code_block = not in_code_block
+                if in_code_block:
+                    # Starting code block
+                    has_code = True
+                    lang = code_fence_match.group(1)
+                    if lang and lang not in code_languages:
+                        code_languages.append(lang)
+
             # Check for headers with hierarchy tracking
-            header_match = re.match(r'^(#{1,6})\s+(.+)', line)
+            header_match = re.match(r'^(#{1,6})\s+(.+)', line) if not in_code_block else None
             if header_match:
                 header_level = len(header_match.group(1))
                 header_text = header_match.group(2).strip()
-                
+
                 # Save current chunk if it exists
                 if current_chunk:
+                    chunk_metadata = self._build_markdown_metadata(
+                        current_hierarchy, code_languages, has_code
+                    )
                     chunks.append(self._create_chunk(
                         content='\n'.join(current_chunk),
                         filename=filename,
                         section=self._build_section_path(current_hierarchy),
                         start_line=line_start,
-                        end_line=line_num - 1
+                        end_line=line_num - 1,
+                        metadata=chunk_metadata
                     ))
-                
+
                 # Update hierarchy
                 current_hierarchy = current_hierarchy[:header_level-1] + [header_text]
                 current_section = header_text
                 current_chunk = [line]
                 current_size = len(line)
                 line_start = line_num
-            
+                code_languages = []
+                has_code = False
+
             else:
                 current_chunk.append(line)
                 current_size += len(line) + 1
-                
+
                 # Check if chunk is getting too large - use smart splitting
-                if current_size >= self.chunk_size:
+                # But don't split inside code blocks
+                if current_size >= self.chunk_size and not in_code_block:
                     # Try to split at paragraph boundary first
                     split_point = self._find_best_split_point(current_chunk)
-                    
+
                     chunk_to_save = current_chunk[:split_point]
+                    chunk_metadata = self._build_markdown_metadata(
+                        current_hierarchy, code_languages, has_code
+                    )
                     chunks.append(self._create_chunk(
                         content='\n'.join(chunk_to_save),
                         filename=filename,
                         section=self._build_section_path(current_hierarchy),
                         start_line=line_start,
-                        end_line=line_start + split_point - 1
+                        end_line=line_start + split_point - 1,
+                        metadata=chunk_metadata
                     ))
-                    
+
                     # Start new chunk with overlap
                     overlap_lines = self._get_overlap_lines(chunk_to_save)
                     remaining_lines = current_chunk[split_point:]
                     current_chunk = overlap_lines + remaining_lines
                     current_size = sum(len(line) + 1 for line in current_chunk)
                     line_start = line_start + split_point - len(overlap_lines)
-        
+                    # Reset code tracking for new chunk
+                    code_languages = []
+                    has_code = False
+
         # Add final chunk
         if current_chunk:
+            chunk_metadata = self._build_markdown_metadata(
+                current_hierarchy, code_languages, has_code
+            )
             chunks.append(self._create_chunk(
                 content='\n'.join(current_chunk),
                 filename=filename,
                 section=self._build_section_path(current_hierarchy),
                 start_line=line_start,
-                end_line=len(lines)
+                end_line=len(lines),
+                metadata=chunk_metadata
             ))
-        
+
         return chunks
     
     def _chunk_python_enhanced(self, content: str, filename: str) -> List[Dict[str, Any]]:
@@ -554,6 +626,49 @@ class DocumentProcessor:
     def _build_section_path(self, hierarchy: List[str]) -> str:
         """Build hierarchical section path from header hierarchy"""
         return ' > '.join(hierarchy) if hierarchy else None
+
+    def _build_markdown_metadata(self, hierarchy: List[str], code_languages: List[str], has_code: bool) -> Dict[str, Any]:
+        """Build rich metadata for markdown chunks
+
+        Args:
+            hierarchy: Current header hierarchy (e.g., ['Installation', 'Requirements', 'Python'])
+            code_languages: List of code block languages found in chunk (e.g., ['python', 'bash'])
+            has_code: Whether chunk contains any code blocks
+
+        Returns:
+            Dictionary with markdown-specific metadata including tags
+        """
+        metadata = {
+            'chunk_type': 'markdown',
+        }
+
+        # Add header level metadata
+        if hierarchy:
+            for i, header in enumerate(hierarchy, 1):
+                metadata[f'h{i}'] = header
+
+        # Add code-related metadata
+        if has_code:
+            metadata['has_code'] = True
+            if code_languages:
+                metadata['code_languages'] = code_languages
+
+        # Build tags for enhanced searching
+        tags = []
+        if has_code:
+            tags.append('code')
+            # Add language-specific tags
+            for lang in code_languages:
+                tags.append(f'code:{lang}')
+
+        # Add tags for header levels (searchable by section depth)
+        if len(hierarchy) > 0:
+            tags.append(f'depth:{len(hierarchy)}')
+
+        if tags:
+            metadata['tags'] = tags
+
+        return metadata
     
     def _build_python_section(self, class_name: Optional[str], function_name: Optional[str]) -> str:
         """Build section name for Python code"""
@@ -674,7 +789,7 @@ class DocumentProcessor:
         chunk_index = 0
         
         # Create overlapping chunks
-        for i in range(0, len(words), self.chunk_size - self.overlap_size):
+        for i in range(0, len(words), self.chunk_size - self.chunk_overlap):
             chunk_words = words[i:i + self.chunk_size]
             if chunk_words:
                 chunk_content = ' '.join(chunk_words)
@@ -686,7 +801,7 @@ class DocumentProcessor:
                         'chunk_method': 'sliding_window',
                         'chunk_index': chunk_index,
                         'chunk_size_words': self.chunk_size,
-                        'overlap_size_words': self.overlap_size,
+                        'overlap_size_words': self.chunk_overlap,
                         'start_word': i,
                         'end_word': i + len(chunk_words)
                     }
@@ -761,4 +876,348 @@ class DocumentProcessor:
                     }
                 ))
         
-        return chunks 
+        return chunks
+    
+    def _chunk_by_semantic(self, content: str, filename: str, file_type: str) -> List[Dict[str, Any]]:
+        """Chunk based on semantic similarity between sentences"""
+        if isinstance(content, list):
+            content = '\n'.join(content)
+        
+        # Get sentences
+        if sent_tokenize:
+            sentences = sent_tokenize(content)
+        else:
+            sentences = content.split('. ')
+            sentences = [s.strip() + '.' for s in sentences if s.strip()]
+        
+        if len(sentences) <= 1:
+            return [self._create_chunk(content, filename, "Section 1", 
+                                     metadata={'chunk_method': 'semantic', 'chunk_index': 0})]
+        
+        # Generate embeddings for sentences (using the same model as the index)
+        try:
+            from sentence_transformers import SentenceTransformer
+            from sklearn.metrics.pairwise import cosine_similarity
+            import numpy as np
+            
+            model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2')
+            embeddings = model.encode(sentences, show_progress_bar=False)
+            
+            # Calculate similarity between adjacent sentences
+            similarities = []
+            for i in range(len(embeddings) - 1):
+                sim = cosine_similarity([embeddings[i]], [embeddings[i + 1]])[0][0]
+                similarities.append(sim)
+            
+            # Find split points where similarity drops below threshold
+            split_points = [0]
+            for i, sim in enumerate(similarities):
+                if sim < self.semantic_threshold:
+                    split_points.append(i + 1)
+            split_points.append(len(sentences))
+            
+            # Create chunks
+            chunks = []
+            for i in range(len(split_points) - 1):
+                start_idx = split_points[i]
+                end_idx = split_points[i + 1]
+                chunk_sentences = sentences[start_idx:end_idx]
+                
+                # Ensure minimum chunk size
+                if len(chunk_sentences) < 2 and i > 0:
+                    # Merge with previous chunk
+                    chunks[-1]['content'] += ' ' + ' '.join(chunk_sentences)
+                    continue
+                    
+                chunk_content = ' '.join(chunk_sentences)
+                chunks.append(self._create_chunk(
+                    content=chunk_content,
+                    filename=filename,
+                    section=f"Semantic Section {i+1}",
+                    metadata={
+                        'chunk_method': 'semantic',
+                        'chunk_index': i,
+                        'semantic_threshold': self.semantic_threshold,
+                        'sentence_count': len(chunk_sentences)
+                    }
+                ))
+            
+            return chunks if chunks else [self._create_chunk(content, filename, "Section 1",
+                                                           metadata={'chunk_method': 'semantic', 'chunk_index': 0})]
+            
+        except ImportError:
+            # Fallback to sentence-based chunking
+            return self._chunk_by_sentences(content, filename, file_type)
+
+    def _chunk_by_topics(self, content: str, filename: str, file_type: str) -> List[Dict[str, Any]]:
+        """Chunk based on topic changes using keyword analysis"""
+        if isinstance(content, list):
+            content = '\n'.join(content)
+        
+        if sent_tokenize:
+            sentences = sent_tokenize(content)
+        else:
+            sentences = content.split('. ')
+            sentences = [s.strip() + '.' for s in sentences if s.strip()]
+            
+        if len(sentences) <= 3:
+            return [self._create_chunk(content, filename, "Topic 1",
+                                     metadata={'chunk_method': 'topic', 'chunk_index': 0})]
+        
+        try:
+            # Simple topic detection using keyword overlap
+            from collections import Counter
+            import re
+            
+            # Extract keywords from each sentence
+            sentence_keywords = []
+            for sentence in sentences:
+                # Simple keyword extraction (could be enhanced with NLP)
+                words = re.findall(r'\b[a-zA-Z]{3,}\b', sentence.lower())
+                # Filter common words (basic stopwords)
+                stopwords = {'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had', 'her', 'was', 'one', 'our', 'out', 'day', 'get', 'has', 'him', 'his', 'how', 'its', 'may', 'new', 'now', 'old', 'see', 'two', 'who', 'boy', 'did', 'man', 'way', 'she', 'use', 'her', 'many', 'oil', 'sit', 'set', 'run', 'eat', 'far', 'sea', 'eye', 'ask', 'own', 'say', 'too', 'any', 'try', 'us', 'an', 'as', 'at', 'be', 'he', 'if', 'in', 'is', 'it', 'my', 'of', 'on', 'or', 'to', 'up', 'we', 'go', 'no', 'so', 'am', 'by', 'do', 'me'}
+                keywords = [w for w in words if w not in stopwords and len(w) > 3]
+                sentence_keywords.append(set(keywords))
+            
+            # Find topic boundaries based on keyword overlap
+            chunks = []
+            current_chunk = [sentences[0]]
+            current_keywords = sentence_keywords[0]
+            
+            for i in range(1, len(sentences)):
+                # Calculate keyword overlap with current chunk
+                overlap = len(current_keywords.intersection(sentence_keywords[i]))
+                total_keywords = len(current_keywords.union(sentence_keywords[i]))
+                
+                if total_keywords > 0:
+                    similarity = overlap / total_keywords
+                else:
+                    similarity = 0
+                
+                # If similarity is low, start new chunk
+                if similarity < self.topic_threshold and len(current_chunk) >= 2:
+                    chunk_content = ' '.join(current_chunk)
+                    chunks.append(self._create_chunk(
+                        content=chunk_content,
+                        filename=filename,
+                        section=f"Topic {len(chunks)+1}",
+                        metadata={
+                            'chunk_method': 'topic',
+                            'chunk_index': len(chunks),
+                            'topic_keywords': list(current_keywords)[:10],  # Top keywords
+                            'sentence_count': len(current_chunk),
+                            'topic_threshold': self.topic_threshold
+                        }
+                    ))
+                    current_chunk = [sentences[i]]
+                    current_keywords = sentence_keywords[i]
+                else:
+                    current_chunk.append(sentences[i])
+                    current_keywords = current_keywords.union(sentence_keywords[i])
+            
+            # Add final chunk
+            if current_chunk:
+                chunk_content = ' '.join(current_chunk)
+                chunks.append(self._create_chunk(
+                    content=chunk_content,
+                    filename=filename,
+                    section=f"Topic {len(chunks)+1}",
+                    metadata={
+                        'chunk_method': 'topic',
+                        'chunk_index': len(chunks),
+                        'topic_keywords': list(current_keywords)[:10],
+                        'sentence_count': len(current_chunk),
+                        'topic_threshold': self.topic_threshold
+                    }
+                ))
+            
+            return chunks if chunks else [self._create_chunk(content, filename, "Topic 1",
+                                                           metadata={'chunk_method': 'topic', 'chunk_index': 0})]
+            
+        except Exception:
+            # Fallback to sentence-based chunking
+            return self._chunk_by_sentences(content, filename, file_type)
+
+    def _chunk_by_qa_optimization(self, content: str, filename: str, file_type: str) -> List[Dict[str, Any]]:
+        """Create chunks optimized for question-answering"""
+        if isinstance(content, list):
+            content = '\n'.join(content)
+        
+        if sent_tokenize:
+            sentences = sent_tokenize(content)
+        else:
+            sentences = content.split('. ')
+            sentences = [s.strip() + '.' for s in sentences if s.strip()]
+        
+        # Patterns that indicate Q&A structure
+        question_patterns = [
+            r'\?',  # Questions
+            r'^(what|how|why|when|where|who|which|can|does|is|are|will|would|should)',
+            r'(step|steps|process|procedure|method|way to)',
+            r'(example|examples|instance|case)',
+            r'(definition|meaning|refers to|means)',
+        ]
+        
+        chunks = []
+        current_chunk = []
+        current_context = []
+        
+        for i, sentence in enumerate(sentences):
+            sentence_lower = sentence.lower().strip()
+            
+            # Check if this sentence contains Q&A indicators
+            is_qa_relevant = any(re.search(pattern, sentence_lower) for pattern in question_patterns)
+            
+            if is_qa_relevant or len(current_chunk) == 0:
+                current_chunk.append(sentence)
+                # Add surrounding context (previous and next sentences)
+                if i > 0 and sentences[i-1] not in current_chunk:
+                    current_context.append(sentences[i-1])
+                if i < len(sentences) - 1:
+                    current_context.append(sentences[i+1])
+            else:
+                current_chunk.append(sentence)
+            
+            # Create chunk when we have enough content or reach a natural break
+            if (len(current_chunk) >= 3 and 
+                (i == len(sentences) - 1 or  # Last sentence
+                 sentence.endswith('.') and len(current_chunk) >= 5)):  # Natural break
+                
+                # Combine chunk with context
+                full_content = current_context + current_chunk
+                chunk_content = ' '.join(full_content)
+                
+                chunks.append(self._create_chunk(
+                    content=chunk_content,
+                    filename=filename,
+                    section=f"QA Section {len(chunks)+1}",
+                    metadata={
+                        'chunk_method': 'qa_optimized',
+                        'chunk_index': len(chunks),
+                        'has_question': any('?' in s for s in current_chunk),
+                        'has_process': any(re.search(r'(step|process|method)', s.lower()) for s in current_chunk),
+                        'sentence_count': len(full_content)
+                    }
+                ))
+                
+                current_chunk = []
+                current_context = []
+        
+        # Handle remaining content
+        if current_chunk:
+            chunk_content = ' '.join(current_context + current_chunk)
+            chunks.append(self._create_chunk(
+                content=chunk_content,
+                filename=filename,
+                section=f"QA Section {len(chunks)+1}",
+                metadata={
+                    'chunk_method': 'qa_optimized',
+                    'chunk_index': len(chunks),
+                    'sentence_count': len(current_context + current_chunk)
+                }
+            ))
+        
+        return chunks if chunks else [self._create_chunk(content, filename, "QA Section 1",
+                                                       metadata={'chunk_method': 'qa_optimized', 'chunk_index': 0})]
+    
+    def _chunk_from_json(self, content: str, filename: str, file_type: str) -> List[Dict[str, Any]]:
+        """
+        Create chunks from pre-processed JSON content
+        
+        This strategy expects content to be a JSON string with the following structure:
+        {
+            "chunks": [
+                {
+                    "chunk_id": "unique_id",
+                    "type": "content|toc",
+                    "content": "text content",
+                    "metadata": {
+                        "url": "https://...",
+                        "section_number": 1,
+                        "related_toc": "toc_id",
+                        ...
+                    }
+                },
+                ...
+            ]
+        }
+        
+        Args:
+            content: JSON string containing pre-chunked content
+            filename: Name of the source file
+            file_type: Should be 'json'
+            
+        Returns:
+            List of chunk dictionaries formatted for the search index
+        """
+        try:
+            # Parse JSON content
+            data = json.loads(content)
+            
+            if not isinstance(data, dict) or 'chunks' not in data:
+                logger.error(f"Invalid JSON structure in {filename}: expected 'chunks' key")
+                # Fallback to treating it as plain text
+                return self._chunk_by_sentences(content, filename, file_type)
+            
+            chunks = []
+            for idx, json_chunk in enumerate(data['chunks']):
+                if not isinstance(json_chunk, dict) or 'content' not in json_chunk:
+                    logger.warning(f"Skipping invalid chunk {idx} in {filename}")
+                    continue
+                
+                # Extract metadata from JSON chunk
+                json_metadata = json_chunk.get('metadata', {})
+                chunk_type = json_chunk.get('type', 'content')
+                
+                # Build chunk metadata (excluding tags which go at top level)
+                metadata = {
+                    'chunk_method': 'json',
+                    'chunk_index': idx,
+                    'chunk_type': chunk_type,
+                    'original_chunk_id': json_chunk.get('chunk_id', f'chunk_{idx}')
+                }
+                
+                # Extract tags before merging metadata
+                tags = json_metadata.get('tags', [])
+                
+                # Merge JSON metadata (this includes all fields including tags)
+                # We'll keep tags in metadata for backward compatibility but also set at top level
+                metadata.update(json_metadata)
+                
+                # Determine section name
+                if chunk_type == 'toc':
+                    section = f"TOC: {json_chunk.get('content', '')[:50]}"
+                else:
+                    section = json_metadata.get('section', f"Section {json_metadata.get('section_number', idx + 1)}")
+                
+                # Create chunk with proper structure
+                chunk = self._create_chunk(
+                    content=json_chunk['content'],
+                    filename=filename,
+                    section=section,
+                    metadata=metadata
+                )
+                
+                # Set tags at the top level for proper tag filtering
+                if tags:
+                    chunk['tags'] = tags
+                elif chunk_type == 'toc':
+                    # For TOC entries, add special tags if none provided
+                    chunk['tags'] = ['toc', 'navigation']
+                
+                chunks.append(chunk)
+            
+            if not chunks:
+                logger.warning(f"No valid chunks found in JSON file {filename}")
+                return self._chunk_by_sentences(str(data), filename, file_type)
+            
+            logger.info(f"Created {len(chunks)} chunks from JSON file {filename}")
+            return chunks
+            
+        except json.JSONDecodeError as e:
+            logger.error(f"Failed to parse JSON in {filename}: {e}")
+            # Fallback to sentence chunking
+            return self._chunk_by_sentences(content, filename, file_type)
+        except Exception as e:
+            logger.error(f"Unexpected error processing JSON chunks in {filename}: {e}")
+            return self._chunk_by_sentences(content, filename, file_type)