local-deep-research 0.1.0__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_local.py

@@ -0,0 +1,901 @@
+from typing import Dict, List, Any, Optional, Tuple, Union
+import os
+import json
+import hashlib
+import time
+from datetime import datetime
+from pathlib import Path
+import tiktoken
+import logging
+import re
+import pickle
+
+from langchain_core.language_models import BaseLLM
+from langchain_community.document_loaders import (
+    PyPDFLoader, 
+    TextLoader, 
+    UnstructuredMarkdownLoader,
+    UnstructuredWordDocumentLoader,
+    CSVLoader,
+    UnstructuredExcelLoader,
+    DirectoryLoader
+)
+from langchain_community.document_loaders.base import BaseLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import (
+    HuggingFaceEmbeddings,
+    OllamaEmbeddings,
+    SentenceTransformerEmbeddings
+)
+
+from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
+from local_deep_research import config
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class LocalEmbeddingManager:
+    """Handles embedding generation and storage for local document search"""
+    
+    def __init__(
+        self,
+        embedding_model: str = "all-MiniLM-L6-v2",
+        embedding_device: str = "cpu",
+        embedding_model_type: str = "sentence_transformers",  # or 'ollama'
+        ollama_base_url: Optional[str] = None,
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200,
+        cache_dir: str = ".cache/local_search",
+    ):
+        """
+        Initialize the embedding manager for local document search.
+        
+        Args:
+            embedding_model: Name of the embedding model to use
+            embedding_device: Device to run embeddings on ('cpu' or 'cuda')
+            embedding_model_type: Type of embedding model ('sentence_transformers' or 'ollama')
+            ollama_base_url: Base URL for Ollama API if using ollama embeddings
+            chunk_size: Size of text chunks for splitting documents
+            chunk_overlap: Overlap between chunks
+            cache_dir: Directory to store embedding cache and index
+        """
+        
+        self.embedding_model = embedding_model
+        self.embedding_device = embedding_device
+        self.embedding_model_type = embedding_model_type
+        self.ollama_base_url = ollama_base_url
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.cache_dir = Path(cache_dir)
+        
+        # Create cache directory if it doesn't exist
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Initialize the embedding model
+        self._embeddings = None
+        
+        # Initialize the text splitter
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size, 
+            chunk_overlap=chunk_overlap
+        )
+        
+        # Track indexed folders and their metadata
+        self.indexed_folders = self._load_indexed_folders()
+        
+        # Vector store cache
+        self.vector_stores = {}
+    @property
+    def embeddings(self):
+        """
+        Lazily initialize embeddings when first accessed.
+        This allows the LocalEmbeddingManager to be created without 
+        immediately loading models, which is helpful when no local search is performed.
+        """
+        if self._embeddings is None:
+            logger.info("Initializing embeddings on first use")
+            self._embeddings = self._initialize_embeddings()
+        return self._embeddings
+
+    def _initialize_embeddings(self):
+        """Initialize the embedding model based on configuration"""
+        try:
+            if self.embedding_model_type == "ollama":
+                # Use Ollama for embeddings
+                if not self.ollama_base_url:
+                    self.ollama_base_url = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
+                    
+                logger.info(f"Initializing Ollama embeddings with model {self.embedding_model}")
+                return OllamaEmbeddings(
+                    model=self.embedding_model,
+                    base_url=self.ollama_base_url
+                )
+            else:
+                # Default: Use SentenceTransformers/HuggingFace
+                logger.info(f"Initializing SentenceTransformerEmbeddings with model {self.embedding_model}")
+                return SentenceTransformerEmbeddings(
+                    model_name=self.embedding_model,
+                    model_kwargs={"device": self.embedding_device}
+                )
+        except Exception as e:
+            logger.error(f"Error initializing embeddings: {e}")
+            logger.warning("Falling back to HuggingFaceEmbeddings with all-MiniLM-L6-v2")
+            return HuggingFaceEmbeddings(
+                model_name="sentence-transformers/all-MiniLM-L6-v2"
+            )
+    def _load_or_create_vector_store(self):
+        """Load the vector store from disk or create it if needed"""
+        vector_store_path = self._get_vector_store_path()
+        
+        # Check if vector store exists and is up to date
+        if vector_store_path.exists() and not self._check_folders_modified():
+            logger.info(f"Loading existing vector store from {vector_store_path}")
+            try:
+                vector_store = FAISS.load_local(
+                    str(vector_store_path),
+                    self.embeddings,
+                    allow_dangerous_deserialization=True
+                )
+                
+                # Add this code to show document count
+                doc_count = len(vector_store.index_to_docstore_id)
+                logger.info(f"Loaded index with {doc_count} document chunks")
+                
+                return vector_store
+            except Exception as e:
+                logger.error(f"Error loading vector store: {e}")
+                logger.info("Will create a new vector store")
+        
+        # Create a new vector store
+        return self._create_vector_store()       
+    def _load_indexed_folders(self) -> Dict[str, Dict[str, Any]]:
+        """Load metadata about indexed folders from disk"""
+        index_metadata_path = self.cache_dir / "index_metadata.json"
+        
+        if index_metadata_path.exists():
+            try:
+                with open(index_metadata_path, "r") as f:
+                    return json.load(f)
+            except Exception as e:
+                logger.error(f"Error loading index metadata: {e}")
+        
+        return {}
+    
+    def _save_indexed_folders(self):
+        """Save metadata about indexed folders to disk"""
+        index_metadata_path = self.cache_dir / "index_metadata.json"
+        
+        try:
+            with open(index_metadata_path, "w") as f:
+                json.dump(self.indexed_folders, f, indent=2)
+        except Exception as e:
+            logger.error(f"Error saving index metadata: {e}")
+    
+    def _get_folder_hash(self, folder_path: str) -> str:
+        """Generate a hash for a folder based on its path"""
+        return hashlib.md5(folder_path.encode()).hexdigest()
+    
+    def _get_index_path(self, folder_path: str) -> Path:
+        """Get the path where the index for a specific folder should be stored"""
+        folder_hash = self._get_folder_hash(folder_path)
+        return self.cache_dir / f"index_{folder_hash}"
+    
+    def _check_folder_modified(self, folder_path: str) -> bool:
+        """Check if a folder has been modified since it was last indexed"""
+        folder_path = Path(folder_path)
+        
+        if not folder_path.exists() or not folder_path.is_dir():
+            return False
+        
+        folder_hash = self._get_folder_hash(str(folder_path))
+        
+        # If folder has never been indexed, it's considered modified
+        if folder_hash not in self.indexed_folders:
+            return True
+        
+        last_indexed = self.indexed_folders[folder_hash].get("last_indexed", 0)
+        
+        # Check if any file in the folder has been modified since last indexing
+        for root, _, files in os.walk(folder_path):
+            for file in files:
+                file_path = Path(root) / file
+                if file_path.stat().st_mtime > last_indexed:
+                    return True
+        
+        return False
+    
+    def get_file_loader(self, file_path: str) -> Optional[BaseLoader]:
+        """Get an appropriate document loader for a file based on its extension"""
+        file_path = Path(file_path)
+        extension = file_path.suffix.lower()
+        
+        try:
+            if extension == ".pdf":
+                return PyPDFLoader(str(file_path))
+            elif extension == ".txt":
+                return TextLoader(str(file_path))
+            elif extension in [".md", ".markdown"]:
+                return UnstructuredMarkdownLoader(str(file_path))
+            elif extension in [".doc", ".docx"]:
+                return UnstructuredWordDocumentLoader(str(file_path))
+            elif extension == ".csv":
+                return CSVLoader(str(file_path))
+            elif extension in [".xls", ".xlsx"]:
+                return UnstructuredExcelLoader(str(file_path))
+            else:
+                # Try the text loader as a fallback for unknown extensions
+                logger.warning(f"Unknown file extension for {file_path}, trying TextLoader")
+                return TextLoader(str(file_path), encoding="utf-8")
+        except Exception as e:
+            logger.error(f"Error creating loader for {file_path}: {e}")
+            return None
+    
+    def index_folder(self, folder_path: str, force_reindex: bool = False) -> bool:
+        """
+        Index all documents in a folder for vector search.
+        
+        Args:
+            folder_path: Path to the folder to index
+            force_reindex: Whether to force reindexing even if unchanged
+            
+        Returns:
+            bool: True if indexing was successful, False otherwise
+        """
+        folder_path = Path(folder_path)
+        
+        # Validate folder
+        if not folder_path.exists():
+            logger.error(f"Folder not found: {folder_path}")
+            return False
+            
+        if not folder_path.is_dir():
+            logger.error(f"Path is not a directory: {folder_path}")
+            return False
+        
+        folder_str = str(folder_path)
+        folder_hash = self._get_folder_hash(folder_str)
+        index_path = self._get_index_path(folder_str)
+        
+        # Check if folder needs to be reindexed
+        if not force_reindex and not self._check_folder_modified(folder_str):
+            logger.info(f"Folder {folder_path} has not been modified since last indexing")
+            
+            # Load the vector store from disk if not already loaded
+            if folder_hash not in self.vector_stores:
+                try:
+                    self.vector_stores[folder_hash] = FAISS.load_local(
+                        str(index_path),
+                        self.embeddings,
+                        allow_dangerous_deserialization=True
+                    )
+                    logger.info(f"Loaded index for {folder_path} from disk")
+                except Exception as e:
+                    logger.error(f"Error loading index for {folder_path}: {e}")
+                    # If loading fails, force reindexing
+                    force_reindex = True
+            else:
+                logger.info(f"Using cached index for {folder_path}")
+                
+            # If no reindexing is needed and vector store loaded successfully
+            if not force_reindex and folder_hash in self.vector_stores:
+                return True
+        
+        logger.info(f"Indexing folder: {folder_path}")
+        start_time = time.time()
+        
+        # Find documents to index
+        all_docs = []
+        file_count = 0
+        error_count = 0
+        
+        for root, _, files in os.walk(folder_path):
+            for file in files:
+                file_path = Path(root) / file
+                
+                # Skip hidden files and directories
+                if file.startswith(".") or any(part.startswith(".") for part in file_path.parts):
+                    continue
+                    
+                # Get a loader for this file
+                loader = self.get_file_loader(str(file_path))
+                
+                if loader:
+                    try:
+                        # Load the document
+                        docs = loader.load()
+                        
+                        # Add source path metadata
+                        for doc in docs:
+                            doc.metadata["source"] = str(file_path)
+                            doc.metadata["filename"] = file
+                            
+                        all_docs.extend(docs)
+                        file_count += 1
+                    except Exception as e:
+                        logger.error(f"Error loading {file_path}: {e}")
+                        error_count += 1
+        
+        if not all_docs:
+            logger.warning(f"No documents found in {folder_path} or all documents failed to load")
+            return False
+        
+        # Split documents into chunks
+        logger.info(f"Splitting {len(all_docs)} documents into chunks")
+        splits = self.text_splitter.split_documents(all_docs)
+        logger.info(f"Created {len(splits)} chunks from {file_count} files")
+        
+        # Create vector store
+        logger.info(f"Creating vector store with {len(splits)} chunks")
+        vector_store = FAISS.from_documents(splits, self.embeddings)
+        
+        # Save the vector store to disk
+        logger.info(f"Saving index to {index_path}")
+        vector_store.save_local(str(index_path))
+        
+        # Update cache
+        self.vector_stores[folder_hash] = vector_store
+        
+        # Update metadata
+        self.indexed_folders[folder_hash] = {
+            "path": folder_str,
+            "last_indexed": time.time(),
+            "file_count": file_count,
+            "chunk_count": len(splits),
+            "error_count": error_count,
+            "embedding_model": self.embedding_model,
+            "chunk_size": self.chunk_size,
+            "chunk_overlap": self.chunk_overlap
+        }
+        
+        # Save updated metadata
+        self._save_indexed_folders()
+        
+        elapsed_time = time.time() - start_time
+        logger.info(f"Indexed {file_count} files in {elapsed_time:.2f} seconds")
+        
+        return True
+    
+    def search(
+        self, 
+        query: str, 
+        folder_paths: List[str],
+        limit: int = 10,
+        score_threshold: float = 0.0,
+    ) -> List[Dict[str, Any]]:
+        """
+        Search for documents relevant to a query across specified folders.
+        
+        Args:
+            query: The search query
+            folder_paths: List of folder paths to search in
+            limit: Maximum number of results to return
+            score_threshold: Minimum similarity score threshold
+            
+        Returns:
+            List of results with document content and metadata
+        """
+        # Add detailed debugging for each folder
+        for folder_path in folder_paths:
+            folder_hash = self._get_folder_hash(folder_path)
+            index_path = self._get_index_path(folder_path)
+            
+            logger.info(f"Diagnostic for {folder_path}:")
+            logger.info(f"  - Folder hash: {folder_hash}")
+            logger.info(f"  - Index path: {index_path}")
+            logger.info(f"  - Index exists on disk: {index_path.exists()}")
+            logger.info(f"  - Is in indexed_folders: {folder_hash in self.indexed_folders}")
+            
+            if folder_hash in self.indexed_folders:
+                meta = self.indexed_folders[folder_hash]
+                logger.info(f"  - Metadata: file_count={meta.get('file_count', 0)}, chunk_count={meta.get('chunk_count', 0)}")
+    
+        # Validate folders exist
+        valid_folder_paths = []
+        for path in folder_paths:
+            if os.path.exists(path) and os.path.isdir(path):
+                valid_folder_paths.append(path)
+            else:
+                logger.warning(f"Skipping non-existent folder in search: {path}")
+        
+        # If no valid folders, return empty results
+        if not valid_folder_paths:
+            logger.warning(f"No valid folders to search among: {folder_paths}")
+            return []
+            
+        all_results = []
+        
+        for folder_path in valid_folder_paths:
+            folder_hash = self._get_folder_hash(folder_path)
+            
+            # Skip folders that haven't been indexed
+            if folder_hash not in self.indexed_folders:
+                logger.warning(f"Folder {folder_path} has not been indexed")
+                continue
+            
+            # Make sure the vector store is loaded
+            if folder_hash not in self.vector_stores:
+                index_path = self._get_index_path(folder_path)
+                try:
+                    self.vector_stores[folder_hash] = FAISS.load_local(
+                        str(index_path),
+                        self.embeddings,
+                        allow_dangerous_deserialization=True
+                    )
+                except Exception as e:
+                    logger.error(f"Error loading index for {folder_path}: {e}")
+                    continue
+            
+            # Search in this folder
+            vector_store = self.vector_stores[folder_hash]
+            
+            try:
+                docs_with_scores = vector_store.similarity_search_with_score(query, k=limit)
+                
+                for doc, score in docs_with_scores:
+                    # Convert score from distance to similarity (lower distance = higher similarity)
+                    # FAISS cosine distance is in [0, 2], where 0 is identical and 2 is opposite
+                    # Convert to a similarity score in [0, 1]
+                    similarity = 1.0 - (score / 2.0)
+                    
+                    # Skip results below the threshold
+                    if similarity < score_threshold:
+                        continue
+                    
+                    result = {
+                        "content": doc.page_content,
+                        "metadata": doc.metadata,
+                        "similarity": float(similarity),
+                        "folder": folder_path
+                    }
+                    
+                    all_results.append(result)
+            except Exception as e:
+                logger.error(f"Error searching in {folder_path}: {e}")
+        
+        # Sort by similarity (highest first)
+        all_results.sort(key=lambda x: x["similarity"], reverse=True)
+        
+        # Limit to the requested number
+        return all_results[:limit]
+    
+    def clear_cache(self):
+        """Clear all cached vector stores from memory (not disk)"""
+        self.vector_stores.clear()
+    
+    def get_indexed_folders_info(self) -> List[Dict[str, Any]]:
+        """Get information about all indexed folders"""
+        info = []
+        
+        for folder_hash, metadata in self.indexed_folders.items():
+            folder_info = metadata.copy()
+            
+            # Add formatted last indexed time
+            if "last_indexed" in folder_info:
+                folder_info["last_indexed_formatted"] = datetime.fromtimestamp(
+                    folder_info["last_indexed"]
+                ).strftime("%Y-%m-%d %H:%M:%S")
+            
+            # Check if index file exists
+            index_path = self._get_index_path(folder_info["path"])
+            folder_info["index_exists"] = index_path.exists()
+            
+            info.append(folder_info)
+        
+        return info
+
+
+class LocalSearchEngine(BaseSearchEngine):
+    """Local document search engine with two-phase retrieval"""
+    
+    def __init__(
+        self,
+        folder_paths: List[str],
+        llm: Optional[BaseLLM] = None,
+        max_results: int = 10,
+        max_filtered_results: Optional[int] = None,
+        embedding_model: str = "all-MiniLM-L6-v2",
+        embedding_device: str = "cpu",
+        embedding_model_type: str = "sentence_transformers",
+        ollama_base_url: Optional[str] = None,
+        force_reindex: bool = False,
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200,
+        cache_dir: str = ".cache/local_search",
+        collections: Optional[Dict[str, Dict[str, Any]]] = None,
+    ):
+        """
+        Initialize the local search engine.
+        
+        Args:
+            folder_paths: List of folder paths to search in
+            llm: Language model for relevance filtering
+            max_results: Maximum number of results to return
+            max_filtered_results: Maximum results after filtering
+            embedding_model: Name of the embedding model to use
+            embedding_device: Device to run embeddings on ('cpu' or 'cuda')
+            embedding_model_type: Type of embedding model
+            ollama_base_url: Base URL for Ollama API
+            force_reindex: Whether to force reindexing
+            chunk_size: Size of text chunks for splitting documents
+            chunk_overlap: Overlap between chunks
+            cache_dir: Directory to store embedding cache and index
+            collections: Dictionary of named collections with paths and descriptions
+        """
+        # Initialize the base search engine
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results)
+        
+        # Validate folder paths
+        self.folder_paths = folder_paths
+        self.valid_folder_paths = []
+        for path in folder_paths:
+            if os.path.exists(path) and os.path.isdir(path):
+                self.valid_folder_paths.append(path)
+            else:
+                logger.warning(f"Folder not found or is not a directory: {path}")
+        
+        # If no valid folders, log a clear message
+        if not self.valid_folder_paths and folder_paths:
+            logger.warning(f"No valid folders found among: {folder_paths}")
+            logger.warning("This search engine will return no results until valid folders are configured")
+            
+        self.max_results = max_results
+        self.collections = collections or {"default": {"paths": folder_paths, "description": "Default collection"}}
+        
+        # Initialize the embedding manager with only valid folders
+        self.embedding_manager = LocalEmbeddingManager(
+            embedding_model=embedding_model,
+            embedding_device=embedding_device,
+            embedding_model_type=embedding_model_type,
+            ollama_base_url=ollama_base_url,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            cache_dir=cache_dir
+        )
+        
+        # Index all folders
+        self._index_folders(force_reindex)
+    
+    def _index_folders(self, force_reindex: bool = False):
+        """Index all valid configured folders"""
+        indexed = []
+        failed = []
+        skipped = []
+        
+        # Keep track of invalid folders
+        for folder in self.folder_paths:
+            if folder not in self.valid_folder_paths:
+                skipped.append(folder)
+                continue
+                
+            success = self.embedding_manager.index_folder(folder, force_reindex)
+            if success:
+                indexed.append(folder)
+            else:
+                failed.append(folder)
+        
+        if indexed:
+            logger.info(f"Successfully indexed {len(indexed)} folders: {', '.join(indexed)}")
+        
+        if failed:
+            logger.warning(f"Failed to index {len(failed)} folders: {', '.join(failed)}")
+            
+        if skipped:
+            logger.warning(f"Skipped {len(skipped)} invalid folders: {', '.join(skipped)}")
+    
+    def _get_previews(self, query: str, collection_names: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        """
+        Get preview information for documents matching the query.
+        
+        Args:
+            query: The search query
+            collection_names: Specific collections to search within (if None, search all)
+            
+        Returns:
+            List of preview dictionaries
+        """
+        # Determine which collections to search
+        if collection_names:
+            # Search only in specified collections
+            collections_to_search = {name: self.collections[name] for name in collection_names 
+                                    if name in self.collections}
+            if not collections_to_search:
+                logger.warning(f"No valid collections found among: {collection_names}")
+                return []
+        else:
+            # Search in all collections
+            collections_to_search = self.collections
+        
+        # Extract all folder paths from the collections to search
+        search_paths = []
+        for collection_config in collections_to_search.values():
+            if "paths" in collection_config:
+                search_paths.extend(collection_config["paths"])
+        
+        logger.info(f"Searching local documents in collections: {list(collections_to_search.keys())}")
+        
+        # Filter out invalid paths
+        valid_search_paths = [path for path in search_paths if path in self.valid_folder_paths]
+        
+        if not valid_search_paths:
+            logger.warning(f"No valid folders to search in collections: {list(collections_to_search.keys())}")
+            return []
+            
+        # Search across the valid selected folders
+        raw_results = self.embedding_manager.search(
+            query=query,
+            folder_paths=valid_search_paths,
+            limit=self.max_results,
+            score_threshold=0.1  # Skip very low relevance results
+        )
+        
+        if not raw_results:
+            logger.info(f"No local documents found for query: {query}")
+            return []
+        
+        # Convert to preview format
+        previews = []
+        for i, result in enumerate(raw_results):
+            # Create a unique ID
+            result_id = f"local-{i}-{hashlib.md5(result['content'][:50].encode()).hexdigest()}"
+            
+            # Extract filename and path
+            source_path = result['metadata'].get('source', 'Unknown')
+            filename = result['metadata'].get('filename', os.path.basename(source_path))
+            
+            # Create preview snippet (first ~200 chars of content)
+            snippet = result['content'][:200] + "..." if len(result['content']) > 200 else result['content']
+            
+            # Determine which collection this document belongs to
+            collection_name = "Unknown"
+            folder_path = result['folder']
+            for name, collection in self.collections.items():
+                if any(folder_path.startswith(path) for path in collection.get("paths", [])):
+                    collection_name = name
+                    break
+            
+            # Format the preview
+            preview = {
+                "id": result_id,
+                "title": filename,
+                "snippet": snippet,
+                "link": source_path,
+                "similarity": result['similarity'],
+                "folder": folder_path,
+                "collection": collection_name,
+                "collection_description": self.collections.get(collection_name, {}).get("description", ""),
+                "_full_content": result['content'],  # Store full content for later
+                "_metadata": result['metadata']  # Store metadata for later
+            }
+            
+            previews.append(preview)
+        
+        logger.info(f"Found {len(previews)} local document matches")
+        return previews
+    
+    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Get full content for the relevant documents.
+        For local search, the full content is already available.
+        
+        Args:
+            relevant_items: List of relevant preview dictionaries
+            
+        Returns:
+            List of result dictionaries with full content
+        """
+        # Check if we should add full content
+        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+            logger.info("Snippet-only mode, skipping full content addition")
+            return relevant_items
+        
+        # For local search, we already have the full content
+        results = []
+        for item in relevant_items:
+            # Create a copy with full content
+            result = item.copy()
+            
+            # Add full content if we have it
+            if "_full_content" in item:
+                result["content"] = item["_full_content"]
+                result["full_content"] = item["_full_content"]
+                
+                # Remove temporary fields
+                if "_full_content" in result:
+                    del result["_full_content"]
+            
+            # Add metadata if we have it
+            if "_metadata" in item:
+                result["document_metadata"] = item["_metadata"]
+                
+                # Remove temporary fields
+                if "_metadata" in result:
+                    del result["_metadata"]
+            
+            results.append(result)
+        
+        return results
+    
+    def run(self, query: str, collection_names: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        """
+        Execute a search using the two-phase approach.
+        
+        Args:
+            query: The search query
+            collection_names: Specific collections to search within (if None, search all)
+            
+        Returns:
+            List of search result dictionaries with full content
+        """
+        logger.info(f"---Execute a search using Local Documents---")
+        
+        # Check if we have any special collection parameters in the query
+        collection_prefix = "collection:"
+        remaining_query = query
+        specified_collections = []
+        
+        # Parse query for collection specifications like "collection:research_papers query terms"
+        query_parts = query.split()
+        for part in query_parts:
+            if part.lower().startswith(collection_prefix):
+                collection_name = part[len(collection_prefix):].strip()
+                if collection_name in self.collections:
+                    specified_collections.append(collection_name)
+                    # Remove this part from the query
+                    remaining_query = remaining_query.replace(part, "", 1).strip()
+        
+        # If collections were specified in the query, they override the parameter
+        if specified_collections:
+            collection_names = specified_collections
+            query = remaining_query
+        
+        # Phase 1: Get previews (with collection filtering)
+        previews = self._get_previews(query, collection_names)
+        
+        if not previews:
+            return []
+        
+        # Phase 2: Filter for relevance
+        relevant_items = self._filter_for_relevance(previews, query)
+        
+        if not relevant_items:
+            return []
+        
+        # Phase 3: Get full content for relevant items
+        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+            print("Returning snippet-only results as per config")
+            results = relevant_items
+        else:
+            results = self._get_full_content(relevant_items)
+        
+        # Clean up temporary data
+        self.embedding_manager.clear_cache()
+        
+        return results
+    
+    def get_collections_info(self) -> List[Dict[str, Any]]:
+        """
+        Get information about all collections, including indexing status.
+        
+        Returns:
+            List of collection information dictionaries
+        """
+        collections_info = []
+        
+        for name, collection in self.collections.items():
+            paths = collection.get("paths", [])
+            description = collection.get("description", "")
+            
+            # Get indexing information for each path
+            paths_info = []
+            for path in paths:
+                # Check if folder exists
+                exists = os.path.exists(path) and os.path.isdir(path)
+                
+                # Check if folder is indexed
+                folder_hash = self.embedding_manager._get_folder_hash(path)
+                indexed = folder_hash in self.embedding_manager.indexed_folders
+                
+                # Get index details if available
+                index_info = {}
+                if indexed:
+                    index_info = self.embedding_manager.indexed_folders[folder_hash].copy()
+                
+                paths_info.append({
+                    "path": path,
+                    "exists": exists,
+                    "indexed": indexed,
+                    "index_info": index_info
+                })
+            
+            collections_info.append({
+                "name": name,
+                "description": description,
+                "paths": paths,
+                "paths_info": paths_info,
+                "document_count": sum(info.get("index_info", {}).get("file_count", 0) for info in paths_info),
+                "chunk_count": sum(info.get("index_info", {}).get("chunk_count", 0) for info in paths_info),
+                "all_indexed": all(info["indexed"] for info in paths_info if info["exists"])
+            })
+        
+        return collections_info
+    
+    def reindex_collection(self, collection_name: str) -> bool:
+        """
+        Reindex a specific collection.
+        
+        Args:
+            collection_name: Name of the collection to reindex
+            
+        Returns:
+            True if reindexing was successful, False otherwise
+        """
+        if collection_name not in self.collections:
+            logger.error(f"Collection '{collection_name}' not found")
+            return False
+        
+        paths = self.collections[collection_name].get("paths", [])
+        success = True
+        
+        for path in paths:
+            if not self.embedding_manager.index_folder(path, force_reindex=True):
+                success = False
+        
+        return success
+    
+    @classmethod
+    def from_config(cls, config_dict: Dict[str, Any], llm: Optional[BaseLLM] = None) -> "LocalSearchEngine":
+        """
+        Create a LocalSearchEngine instance from a configuration dictionary.
+        
+        Args:
+            config_dict: Configuration dictionary
+            llm: Language model for relevance filtering
+            
+        Returns:
+            Initialized LocalSearchEngine instance
+        """
+        # Required parameters
+        folder_paths = []
+        collections = config_dict.get("collections", {})
+        
+        # Extract all folder paths from collections
+        for collection_config in collections.values():
+            if "paths" in collection_config:
+                folder_paths.extend(collection_config["paths"])
+        
+        # Fall back to folder_paths if no collections defined
+        if not folder_paths:
+            folder_paths = config_dict.get("folder_paths", [])
+            # Create a default collection if using folder_paths
+            if folder_paths:
+                collections = {"default": {"paths": folder_paths, "description": "Default collection"}}
+        
+        # Optional parameters with defaults
+        max_results = config_dict.get("max_results", 10)
+        max_filtered_results = config_dict.get("max_filtered_results")
+        embedding_model = config_dict.get("embedding_model", "all-MiniLM-L6-v2")
+        embedding_device = config_dict.get("embedding_device", "cpu")
+        embedding_model_type = config_dict.get("embedding_model_type", "sentence_transformers")
+        ollama_base_url = config_dict.get("ollama_base_url")
+        force_reindex = config_dict.get("force_reindex", False)
+        chunk_size = config_dict.get("chunk_size", 1000)
+        chunk_overlap = config_dict.get("chunk_overlap", 200)
+        cache_dir = config_dict.get("cache_dir", ".cache/local_search")
+        
+        return cls(
+            folder_paths=folder_paths,
+            collections=collections,
+            llm=llm,
+            max_results=max_results,
+            max_filtered_results=max_filtered_results,
+            embedding_model=embedding_model,
+            embedding_device=embedding_device,
+            embedding_model_type=embedding_model_type,
+            ollama_base_url=ollama_base_url,
+            force_reindex=force_reindex,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            cache_dir=cache_dir
+        )