local-deep-research 0.1.18__py3-none-any.whl → 0.1.20__py3-none-any.whl

local_deep_research/config.py

@@ -145,6 +145,19 @@ def init_config_files():
             if not os.path.exists(search_engines_file) and os.path.exists(default_engines):
                 shutil.copyfile(default_engines, search_engines_file)
                 logger.info(f"Created search_engines.toml at {search_engines_file}")
+
+                    # Create .env.template if it doesn't exist
+            env_template_file = CONFIG_DIR / ".env.template"
+            if not env_template_file.exists():
+                shutil.copy(defaults_dir / ".env.template", env_template_file)
+                logger.info(f"Created .env.template at {env_template_file}")
+                
+                # Optionally create an empty .env file if it doesn't exist
+                env_file = CONFIG_DIR / ".env"
+                if not env_file.exists():
+                    with open(env_file, "w") as f:
+                        f.write("# Add your environment variables here\n")
+                    logger.info(f"Created empty .env file at {env_file}")
         except Exception as e:
             logger.error(f"Error initializing Windows config files: {e}")
     else:
@@ -183,7 +196,17 @@ def init_config_files():
         if not search_engines_file.exists():
             shutil.copy(defaults_dir / "search_engines.toml", search_engines_file)
             logger.info(f"Created search_engines.toml at {search_engines_file}")
+        env_template_file = CONFIG_DIR / ".env.template"
+        if not env_template_file.exists():
+            shutil.copy(defaults_dir / ".env.template", env_template_file)
+            logger.info(f"Created .env.template at {env_template_file}")
             
+            # Optionally create an empty .env file if it doesn't exist
+            env_file = CONFIG_DIR / ".env"
+            if not env_file.exists():
+                with open(env_file, "w") as f:
+                    f.write("# Add your environment variables here\n")
+                logger.info(f"Created empty .env file at {env_file}")            
         secrets_file = CONFIG_DIR / ".secrets.toml"
         if not secrets_file.exists():
             with open(secrets_file, "w") as f:

local_deep_research/defaults/llm_config.py

@@ -12,78 +12,50 @@ from langchain_community.llms import VLLM
 from local_deep_research.config import settings
 import os
 import logging
-from enum import Enum, auto
 
 # Initialize environment
 logger = logging.getLogger(__name__)
 
-# Provider enum
-class ModelProvider(Enum):
-    OLLAMA = auto()
-    OPENAI = auto()
-    ANTHROPIC = auto()
-    VLLM = auto()
-    OPENAI_ENDPOINT = auto()
-    NONE = auto()
-
-# ================================
-# USER CONFIGURATION SECTION
-# ================================
-
-# Set your preferred model provider here
-DEFAULT_PROVIDER = ModelProvider.OLLAMA  # Change this to your preferred provider
-
-# Set your default model name here
-DEFAULT_MODEL = "gemma3:12b"  # Your default model
-
-# Set default model parameters
-DEFAULT_TEMPERATURE = 0.7
-MAX_TOKENS = 30000
-
-# Server URLs
-OPENAI_ENDPOINT_URL = "https://openrouter.ai/api/v1"  # For OpenRouter or compatible services
-OLLAMA_BASE_URL = "http://localhost:11434"  # URL for Ollama server
-
-
-
+# Valid provider options
+VALID_PROVIDERS = ["ollama", "openai", "anthropic", "vllm", "openai_endpoint", "lmstudio", "llamacpp", "none"]
 
 # ================================
 # LLM FUNCTIONS
 # ================================
 
-
-
-    
-
 def get_llm(model_name=None, temperature=None, provider=None):
     """
     Get LLM instance based on model name and provider.
     
     Args:
-        model_name: Name of the model to use (if None, uses DEFAULT_MODEL)
-        temperature: Model temperature (if None, uses DEFAULT_TEMPERATURE)
-        provider: Provider to use (if None, uses DEFAULT_PROVIDER)
+        model_name: Name of the model to use (if None, uses settings.llm.model)
+        temperature: Model temperature (if None, uses settings.llm.temperature)
+        provider: Provider to use (if None, uses settings.llm.provider)
     
     Returns:
         A LangChain LLM instance
     """
+    # Use settings values for parameters if not provided
     if model_name is None:
-        model_name = DEFAULT_MODEL
+        model_name = settings.llm.model
     
     if temperature is None:
-        temperature = DEFAULT_TEMPERATURE
+        temperature = settings.llm.temperature
         
     if provider is None:
-        provider = DEFAULT_PROVIDER
+        provider = settings.llm.provider.lower()
+        if provider not in VALID_PROVIDERS:
+            logger.error(f"Invalid provider in settings: {provider}")
+            raise ValueError(f"Invalid provider: {provider}. Must be one of: {VALID_PROVIDERS}")
     
     # Common parameters for all models
     common_params = {
         "temperature": temperature,
-        "max_tokens": MAX_TOKENS,
+        "max_tokens": settings.llm.max_tokens,
     }
     
     # Handle different providers
-    if provider == ModelProvider.ANTHROPIC:
+    if provider == "anthropic":
         api_key = settings.get('ANTHROPIC_API_KEY', '')
         if not api_key:
             logger.warning("ANTHROPIC_API_KEY not found. Falling back to default model.")
@@ -93,7 +65,7 @@ def get_llm(model_name=None, temperature=None, provider=None):
             model=model_name, anthropic_api_key=api_key, **common_params
         )
     
-    elif provider == ModelProvider.OPENAI:
+    elif provider == "openai":
         api_key = settings.get('OPENAI_API_KEY', '')
         if not api_key:
             logger.warning("OPENAI_API_KEY not found. Falling back to default model.")
@@ -101,21 +73,24 @@ def get_llm(model_name=None, temperature=None, provider=None):
         
         return ChatOpenAI(model=model_name, api_key=api_key, **common_params)
     
-    elif provider == ModelProvider.OPENAI_ENDPOINT:
-        api_key = settings.OPENAI_ENDPOINT_API_KEY
+    elif provider == "openai_endpoint":
+        api_key = settings.get('OPENAI_ENDPOINT_API_KEY', '')
 
         if not api_key:
             logger.warning("OPENAI_ENDPOINT_API_KEY not found. Falling back to default model.")
             return get_fallback_model(temperature)
         
+        # Get endpoint URL from settings
+        openai_endpoint_url = settings.llm.openai_endpoint_url
+        
         return ChatOpenAI(
             model=model_name, 
             api_key=api_key,
-            openai_api_base=OPENAI_ENDPOINT_URL, 
+            openai_api_base=openai_endpoint_url, 
             **common_params
         )
     
-    elif provider == ModelProvider.VLLM:
+    elif provider == "vllm":
         try:
             return VLLM(
                 model=model_name,
@@ -130,19 +105,60 @@ def get_llm(model_name=None, temperature=None, provider=None):
             logger.warning("Falling back.")
             return get_fallback_model(temperature)
     
-    elif provider == ModelProvider.OLLAMA:
+    elif provider == "ollama":
         try:
             # Use the configurable Ollama base URL
-            base_url = settings.get('OLLAMA_BASE_URL', OLLAMA_BASE_URL)
+            base_url = settings.get('OLLAMA_BASE_URL', settings.llm.get('ollama_base_url', 'http://localhost:11434'))
             return ChatOllama(model=model_name, base_url=base_url, **common_params)
         except Exception as e:
             logger.error(f"Error loading Ollama model: {e}")
             return get_fallback_model(temperature)
     
+    elif provider == "lmstudio":
+
+            # LM Studio supports OpenAI API format, so we can use ChatOpenAI directly
+            lmstudio_url = settings.llm.get('lmstudio_url', "http://localhost:1234")
+            
+            return ChatOpenAI(
+                model=model_name,
+                api_key="lm-studio",  # LM Studio doesn't require a real API key
+                base_url=f"{lmstudio_url}/v1",  # Use the configured URL with /v1 endpoint
+                temperature=temperature,
+                max_tokens=settings.llm.max_tokens
+            )
+
+ 
+    elif provider == "llamacpp":
+
+            # Import LlamaCpp
+            from langchain_community.llms import LlamaCpp
+            
+            # Get LlamaCpp model path from settings
+            model_path = settings.llm.get('llamacpp_model_path', "")
+            if not model_path:
+                logger.error("llamacpp_model_path not set in settings")
+                raise ValueError("llamacpp_model_path not set in settings.toml")
+                
+            # Get additional LlamaCpp parameters
+            n_gpu_layers = settings.llm.get('llamacpp_n_gpu_layers', 1)
+            n_batch = settings.llm.get('llamacpp_n_batch', 512)
+            f16_kv = settings.llm.get('llamacpp_f16_kv', True)
+            
+            # Create LlamaCpp instance
+            return LlamaCpp(
+                model_path=model_path,
+                temperature=temperature,
+                max_tokens=settings.llm.max_tokens,
+                n_gpu_layers=n_gpu_layers,
+                n_batch=n_batch,
+                f16_kv=f16_kv,
+                verbose=True
+            )
+    
     else:
         return get_fallback_model(temperature)
 
-def get_fallback_model(temperature=DEFAULT_TEMPERATURE):
+def get_fallback_model(temperature=None):
     """Create a dummy model for when no providers are available"""
     from langchain_community.llms.fake import FakeListLLM
     return FakeListLLM(
@@ -169,6 +185,12 @@ def get_available_provider_types():
     if is_openai_endpoint_available():
         providers["openai_endpoint"] = "OpenAI-compatible Endpoint"
     
+    if is_lmstudio_available():
+        providers["lmstudio"] = "LM Studio (local models)"
+        
+    if is_llamacpp_available():
+        providers["llamacpp"] = "LlamaCpp (local models)"
+    
     # Check for VLLM capability
     try:
         import torch
@@ -183,9 +205,6 @@ def get_available_provider_types():
     
     return providers
 
-
-
-
 # ================================
 # HELPER FUNCTIONS
 # ================================
@@ -193,7 +212,7 @@ def get_available_provider_types():
 def is_openai_available():
     """Check if OpenAI is available"""
     try:
-        api_key = settings.api_keys.get('OPENAI_API_KEY', '')
+        api_key = settings.get('OPENAI_API_KEY', '')
         return bool(api_key)
     except:
         return False
@@ -201,18 +220,15 @@ def is_openai_available():
 def is_anthropic_available():
     """Check if Anthropic is available"""
     try:
-        api_key = settings.api_keys.get('ANTHROPIC_API_KEY', '')
+        api_key = settings.get('ANTHROPIC_API_KEY', '')
         return bool(api_key)
     except:
         return False
 
-
-
 def is_openai_endpoint_available():
     """Check if OpenAI endpoint is available"""
-    
     try:
-        api_key = settings.OPENAI_ENDPOINT_API_KEY
+        api_key = settings.get('OPENAI_ENDPOINT_API_KEY', '')
         return bool(api_key) 
     except:
         return False
@@ -221,7 +237,7 @@ def is_ollama_available():
     """Check if Ollama is running"""
     try:
         import requests
-        base_url = settings.get('OLLAMA_BASE_URL', OLLAMA_BASE_URL)
+        base_url = settings.get('OLLAMA_BASE_URL', settings.llm.get('ollama_base_url', 'http://localhost:11434'))
         response = requests.get(f"{base_url}/api/tags", timeout=1.0)
         return response.status_code == 200
     except:
@@ -236,34 +252,35 @@ def is_vllm_available():
     except ImportError:
         return False
 
+def is_lmstudio_available():
+    """Check if LM Studio is available"""
+    try:
+        import requests
+        lmstudio_url = settings.llm.get('lmstudio_url', 'http://localhost:1234')
+        # LM Studio typically uses OpenAI-compatible endpoints
+        response = requests.get(f"{lmstudio_url}/v1/models", timeout=1.0)
+        return response.status_code == 200
+    except:
+        return False
+
+def is_llamacpp_available():
+    """Check if LlamaCpp is available and configured"""
+    try:
+        from langchain_community.llms import LlamaCpp
+        model_path = settings.llm.get('llamacpp_model_path', '')
+        return bool(model_path) and os.path.exists(model_path)
+    except:
+        return False
+
 def get_available_providers():
     """Get dictionary of available providers"""
-    providers = {}
-    
-    if is_ollama_available():
-        providers[ModelProvider.OLLAMA] = "Ollama (local models)"
-    
-    if is_openai_available():
-        providers[ModelProvider.OPENAI] = "OpenAI API"
-    
-    if is_anthropic_available():
-        providers[ModelProvider.ANTHROPIC] = "Anthropic API"
-    
-    if is_openai_endpoint_available():
-        providers[ModelProvider.OPENAI_ENDPOINT] = "OpenAI-compatible Endpoint"
-    
-    if is_vllm_available():
-        providers[ModelProvider.VLLM] = "VLLM (local models)"
-    
-    if not providers:
-        providers[ModelProvider.NONE] = "No model providers available"
-    
-    return providers
+    return get_available_provider_types()
 
 # Log which providers are available
 AVAILABLE_PROVIDERS = get_available_providers()
-logger.info(f"Available providers: {[p.name for p in AVAILABLE_PROVIDERS.keys()]}")
+logger.info(f"Available providers: {list(AVAILABLE_PROVIDERS.keys())}")
 
 # Check if selected provider is available
-if DEFAULT_PROVIDER not in AVAILABLE_PROVIDERS and DEFAULT_PROVIDER != ModelProvider.NONE:
-    logger.warning(f"Selected provider {DEFAULT_PROVIDER.name} is not available.")
+selected_provider = settings.llm.provider.lower()
+if selected_provider not in AVAILABLE_PROVIDERS and selected_provider != "none":
+    logger.warning(f"Selected provider {selected_provider} is not available.")

local_deep_research/defaults/local_collections.toml

@@ -1,6 +1,3 @@
-# Default local document collections configuration
-# Each collection functions as an independent search engine
-
 # Project Documents Collection
 [project_docs]
 name = "Project Documents"
@@ -15,6 +12,9 @@ max_filtered_results = 5
 chunk_size = 1000
 chunk_overlap = 200
 cache_dir = "__CACHE_DIR__/local_search/project_docs"
+strengths = ["project documentation", "specifications", "internal documents"]
+weaknesses = ["no external information", "limited to organizational knowledge"]
+reliability = 0.9
 
 # Research Papers Collection
 [research_papers]
@@ -30,6 +30,9 @@ max_filtered_results = 5
 chunk_size = 800
 chunk_overlap = 150
 cache_dir = "__CACHE_DIR__/local_search/research_papers"
+strengths = ["academic research", "scientific papers", "scholarly content"]
+weaknesses = ["potentially outdated", "limited to collected papers"]
+reliability = 0.85
 
 # Personal Notes Collection
 [personal_notes]
@@ -44,4 +47,7 @@ max_results = 30
 max_filtered_results = 10
 chunk_size = 500
 chunk_overlap = 100
-cache_dir = "__CACHE_DIR__/local_search/personal_notes"
+cache_dir = "__CACHE_DIR__/local_search/personal_notes"
+strengths = ["personal knowledge", "notes", "private documents"]
+weaknesses = ["subjective content", "informal information"]
+reliability = 0.75

local_deep_research/defaults/main.toml

@@ -1,23 +1,41 @@
 
 # Main configuration for Local Deep Research
 
+[web]
+port = 5000
+host = "0.0.0.0"
+debug = true
+
+[llm]
+# LLM provider (one of: ollama, openai, anthropic, vllm, openai_endpoint, lmstudio, llamacpp)
+provider = "ollama"
+# Model name
+model = "gemma3:12b"
+# Temperature
+temperature = 0.7
+# Maximum tokens
+max_tokens = 30000
+# OpenAI-compatible endpoint URL
+openai_endpoint_url = "https://openrouter.ai/api/v1"
+# LM Studio URL (default: http://localhost:1234)
+lmstudio_url = "http://localhost:1234"
+# LlamaCpp model path
+llamacpp_model_path = ""
+# LlamaCpp parameters
+llamacpp_n_gpu_layers = 1
+llamacpp_n_batch = 512
+llamacpp_f16_kv = true
+
 [general]
 # Directory for research outputs (relative to user data directory)
 output_dir = "research_outputs"
-
 # Knowledge accumulation approach (NONE, QUESTION, or ITERATION)
 knowledge_accumulation = "ITERATION"
-
 # Maximum context size for knowledge accumulation
 knowledge_accumulation_context_limit = 2000000
-
 # Enable fact checking (experimental, works better with large LLMs)
 enable_fact_checking = false
 
-[web]
-port = 5000
-host = "0.0.0.0"
-debug = true
 
 [search]
 # Search tool to use (auto, wikipedia, arxiv, duckduckgo, serp, google_pse, etc.)

local_deep_research/web/static/sounds/README.md

@@ -0,0 +1,29 @@
+# Sound Files for Notifications
+
+This directory contains sound files used for notifications in the Deep Research application.
+
+## Required Files
+
+1. `success.mp3` - Played when research completes successfully
+2. `error.mp3` - Played when research fails or encounters an error
+
+## Sound Sources
+
+You can download copyright-free sound files from these sources:
+
+- [Freesound](https://freesound.org/)
+- [Free Sound Library](https://www.freesoundslibrary.com/)
+
+## Recommended Sounds
+
+### Success Sound
+- [Success Sound by grunz](https://freesound.org/people/grunz/sounds/109662/)
+- Direct download: https://freesound.org/data/previews/109/109662_945474-lq.mp3
+
+### Error Sound
+- [Error Sound by Autistic Lucario](https://freesound.org/people/Autistic%20Lucario/sounds/142608/)
+- Direct download: https://freesound.org/data/previews/142/142608_1840739-lq.mp3
+
+## Usage
+
+The application will automatically use these sounds when research tasks complete or fail, but only when the browser tab is not in focus. 

local_deep_research/web_search_engines/engines/meta_search_engine.py

@@ -91,14 +91,23 @@ class MetaSearchEngine(BaseSearchEngine):
         if not self.available_engines:
             logger.warning("No search engines available")
             return []
-        
-        # Create engine descriptions for the prompt
-        engine_descriptions = "\n".join([
-            f"- {name.upper()}: Good for {', '.join(SEARCH_ENGINES[name]['strengths'][:3])}. "
-            f"Weaknesses: {', '.join(SEARCH_ENGINES[name]['weaknesses'][:2])}. "
-            f"Reliability: {SEARCH_ENGINES[name]['reliability']*100:.0f}%"
-            for name in self.available_engines
-        ])
+        engine_descriptions = []
+        for name in self.available_engines:
+            logger.info(f"Processing search engine: {name}")
+            try:
+                description = f"- {name.upper()}: Good for {', '.join(SEARCH_ENGINES[name]['strengths'][:3])}. " \
+                            f"Weaknesses: {', '.join(SEARCH_ENGINES[name]['weaknesses'][:2])}. " \
+                            f"Reliability: {SEARCH_ENGINES[name]['reliability']*100:.0f}%"
+                engine_descriptions.append(description)
+            except KeyError as e:
+                logger.error(f"Missing key for engine {name}: {e}")
+                # Add a basic description for engines with missing configuration
+                engine_descriptions.append(f"- {name.upper()}: General purpose search engine.")
+            except Exception as e:
+                logger.error(f"Error processing engine {name}: {e}")
+                engine_descriptions.append(f"- {name.upper()}: General purpose search engine.")
+
+        engine_descriptions = "\n".join(engine_descriptions)
         
         prompt = f"""Analyze this search query and rank the available search engines in order of most to least appropriate for answering it.
         

local_deep_research/web_search_engines/engines/search_engine_local.py

@@ -10,6 +10,7 @@ import logging
 import re
 import pickle
 
+from faiss import normalize_L2
 from langchain_core.language_models import BaseLLM
 from langchain_community.document_loaders import (
     PyPDFLoader, 
@@ -23,6 +24,7 @@ from langchain_community.document_loaders import (
 from langchain_community.document_loaders.base import BaseLoader
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import FAISS
+from langchain_community.vectorstores.utils import DistanceStrategy
 from langchain_community.embeddings import (
     HuggingFaceEmbeddings,
     OllamaEmbeddings,
@@ -136,7 +138,8 @@ class LocalEmbeddingManager:
                 vector_store = FAISS.load_local(
                     str(vector_store_path),
                     self.embeddings,
-                    allow_dangerous_deserialization=True
+                    allow_dangerous_deserialization=True,
+                    normalize_L2=True
                 )
                 
                 # Add this code to show document count
@@ -175,6 +178,10 @@ class LocalEmbeddingManager:
     
     def _get_folder_hash(self, folder_path: str) -> str:
         """Generate a hash for a folder based on its path"""
+        # Strip trailing slashes if we have them.
+        if folder_path.endswith("/"):
+            folder_path = folder_path[:-1]
+
         return hashlib.md5(folder_path.encode()).hexdigest()
     
     def _get_index_path(self, folder_path: str) -> Path:
@@ -205,6 +212,32 @@ class LocalEmbeddingManager:
                     return True
         
         return False
+
+    def _check_config_changed(self, folder_path: str) -> bool:
+        """
+        Checks if the embedding configuration for a folder has been changed
+        since it was last indexed.
+        """
+        folder_hash = self._get_folder_hash(folder_path)
+
+        if folder_hash not in self.indexed_folders:
+            # It hasn't been indexed at all. That's a new configuration,
+            # technically.
+            return True
+
+        embedding_config = self.indexed_folders[folder_hash]
+        chunk_size = embedding_config.get("chunk_size", 0)
+        chunk_overlap = embedding_config.get("chunk_overlap", 0)
+        embedding_model = embedding_config.get("embedding_model", "")
+
+        if (chunk_size, chunk_overlap, embedding_model) != (
+                self.chunk_size, self.chunk_overlap, self.embedding_model
+        ):
+            logger.info(
+                "Embedding configuration has changed, re-indexing folder."
+            )
+            return True
+        return False
     
     def get_file_loader(self, file_path: str) -> Optional[BaseLoader]:
         """Get an appropriate document loader for a file based on its extension"""
@@ -257,9 +290,10 @@ class LocalEmbeddingManager:
         folder_str = str(folder_path)
         folder_hash = self._get_folder_hash(folder_str)
         index_path = self._get_index_path(folder_str)
-        
+
         # Check if folder needs to be reindexed
-        if not force_reindex and not self._check_folder_modified(folder_str):
+        if (not force_reindex and not self._check_folder_modified(folder_str)
+                and not self._check_config_changed(folder_str)):
             logger.info(f"Folder {folder_path} has not been modified since last indexing")
             
             # Load the vector store from disk if not already loaded
@@ -268,7 +302,8 @@ class LocalEmbeddingManager:
                     self.vector_stores[folder_hash] = FAISS.load_local(
                         str(index_path),
                         self.embeddings,
-                        allow_dangerous_deserialization=True
+                        allow_dangerous_deserialization=True,
+                        normalize_L2=True,
                     )
                     logger.info(f"Loaded index for {folder_path} from disk")
                 except Exception as e:
@@ -328,7 +363,11 @@ class LocalEmbeddingManager:
         
         # Create vector store
         logger.info(f"Creating vector store with {len(splits)} chunks")
-        vector_store = FAISS.from_documents(splits, self.embeddings)
+        vector_store = FAISS.from_documents(
+            splits,
+            self.embeddings,
+            normalize_L2=True
+        )
         
         # Save the vector store to disk
         logger.info(f"Saving index to {index_path}")
@@ -421,7 +460,8 @@ class LocalEmbeddingManager:
                     self.vector_stores[folder_hash] = FAISS.load_local(
                         str(index_path),
                         self.embeddings,
-                        allow_dangerous_deserialization=True
+                        allow_dangerous_deserialization=True,
+                        nomalize_L2=True
                     )
                 except Exception as e:
                     logger.error(f"Error loading index for {folder_path}: {e}")
@@ -431,14 +471,14 @@ class LocalEmbeddingManager:
             vector_store = self.vector_stores[folder_hash]
             
             try:
-                docs_with_scores = vector_store.similarity_search_with_score(query, k=limit)
+                docs_with_scores = (
+                    vector_store.similarity_search_with_relevance_scores(
+                        query,
+                        k=limit
+                    )
+                )
                 
-                for doc, score in docs_with_scores:
-                    # Convert score from distance to similarity (lower distance = higher similarity)
-                    # FAISS cosine distance is in [0, 2], where 0 is identical and 2 is opposite
-                    # Convert to a similarity score in [0, 1]
-                    similarity = 1.0 - (score / 2.0)
-                    
+                for doc, similarity in docs_with_scores:
                     # Skip results below the threshold
                     if similarity < score_threshold:
                         continue
@@ -491,7 +531,7 @@ class LocalSearchEngine(BaseSearchEngine):
     
     def __init__(
         self,
-        folder_paths: List[str],
+        paths: List[str],
         llm: Optional[BaseLLM] = None,
         max_results: int = 10,
         max_filtered_results: Optional[int] = None,
@@ -509,7 +549,7 @@ class LocalSearchEngine(BaseSearchEngine):
         Initialize the local search engine.
         
         Args:
-            folder_paths: List of folder paths to search in
+            paths: List of folder paths to search in
             llm: Language model for relevance filtering
             max_results: Maximum number of results to return
             max_filtered_results: Maximum results after filtering
@@ -527,21 +567,21 @@ class LocalSearchEngine(BaseSearchEngine):
         super().__init__(llm=llm, max_filtered_results=max_filtered_results)
         
         # Validate folder paths
-        self.folder_paths = folder_paths
+        self.folder_paths = paths
         self.valid_folder_paths = []
-        for path in folder_paths:
+        for path in paths:
             if os.path.exists(path) and os.path.isdir(path):
                 self.valid_folder_paths.append(path)
             else:
                 logger.warning(f"Folder not found or is not a directory: {path}")
         
         # If no valid folders, log a clear message
-        if not self.valid_folder_paths and folder_paths:
-            logger.warning(f"No valid folders found among: {folder_paths}")
+        if not self.valid_folder_paths and paths:
+            logger.warning(f"No valid folders found among: {paths}")
             logger.warning("This search engine will return no results until valid folders are configured")
             
         self.max_results = max_results
-        self.collections = collections or {"default": {"paths": folder_paths, "description": "Default collection"}}
+        self.collections = collections or {"default": {"paths": paths, "description": "Default collection"}}
         
         # Initialize the embedding manager with only valid folders
         self.embedding_manager = LocalEmbeddingManager(
@@ -885,7 +925,7 @@ class LocalSearchEngine(BaseSearchEngine):
         cache_dir = config_dict.get("cache_dir", ".cache/local_search")
         
         return cls(
-            folder_paths=folder_paths,
+            paths=folder_paths,
             collections=collections,
             llm=llm,
             max_results=max_results,

local_deep_research/web_search_engines/engines/search_engine_local_all.py

@@ -4,11 +4,13 @@ Search engine that searches across all local collections
 
 import logging
 from typing import Dict, List, Any, Optional
+
+import toml
 from langchain_core.language_models import BaseLLM
 
 from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
 from local_deep_research.web_search_engines.search_engine_factory import create_search_engine
-from local_deep_research import config
+from local_deep_research.config import LOCAL_COLLECTIONS_FILE
 
 # Setup logging
 logger = logging.getLogger(__name__)
@@ -18,7 +20,7 @@ class LocalAllSearchEngine(BaseSearchEngine):
     Search engine that searches across all local document collections.
     Acts as a meta search engine specifically for local collections.
     """
-    
+
     def __init__(
         self,
         llm: Optional[BaseLLM] = None,
@@ -41,9 +43,9 @@ class LocalAllSearchEngine(BaseSearchEngine):
         # Find all local collection search engines
         self.local_engines = {}
         try:
-            from local_collections import LOCAL_COLLECTIONS
-            
-            for collection_id, collection in LOCAL_COLLECTIONS.items():
+            local_collections = toml.load(LOCAL_COLLECTIONS_FILE)
+
+            for collection_id, collection in local_collections.items():
                 if not collection.get("enabled", True):
                     continue
                     

local_deep_research/web_search_engines/search_engines_config.py

@@ -6,7 +6,7 @@ import logging
 import os
 import toml
 from pathlib import Path
-from local_deep_research.config import CONFIG_DIR
+from local_deep_research.config import CONFIG_DIR, LOCAL_COLLECTIONS_FILE
 
 logger = logging.getLogger(__name__)
 
@@ -41,13 +41,30 @@ if 'auto' in SEARCH_ENGINES and 'meta' not in SEARCH_ENGINES:
     SEARCH_ENGINES['meta'] = SEARCH_ENGINES['auto']
 
 # Register local document collections
-try:
-    from local_deep_research.local_collections import register_local_collections
-    register_local_collections(SEARCH_ENGINES)
-    logger.info(f"Registered local document collections as search engines")
-except ImportError:
-    logger.info("No local collections configuration found. Local document search is disabled.")
 
+if os.path.exists(LOCAL_COLLECTIONS_FILE):
+    try:
+        local_collections_data = toml.load(LOCAL_COLLECTIONS_FILE)
+
+        for collection, config in local_collections_data.items():
+            # Create a new dictionary with required search engine fields
+            engine_config = {
+                "module_path": "local_deep_research.web_search_engines.engines.search_engine_local",
+                "class_name": "LocalSearchEngine",
+                "default_params": config,
+                "requires_llm": True
+            }
+            
+            # Copy these specific fields to the top level if they exist
+            for field in ["strengths", "weaknesses", "reliability", "description"]:
+                if field in config:
+                    engine_config[field] = config[field]
+            
+            SEARCH_ENGINES[collection] = engine_config
+            
+        logger.info(f"Registered local document collections as search engines")
+    except Exception as e:
+        logger.error(f"Error loading local collections from TOML file: {e}")
 # Ensure the meta search engine is still available at the end if it exists
 if 'auto' in SEARCH_ENGINES:
     meta_config = SEARCH_ENGINES["auto"]

{local_deep_research-0.1.18.dist-info → local_deep_research-0.1.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: local-deep-research
-Version: 0.1.18
+Version: 0.1.20
 Summary: AI-powered research assistant with deep, iterative analysis using LLMs and web searches
 Author-email: LearningCircuit <185559241+LearningCircuit@users.noreply.github.com>, HashedViking <6432677+HashedViking@users.noreply.github.com>
 License: MIT License
@@ -71,6 +71,14 @@ Dynamic: license-file
 
 A powerful AI-powered research assistant that performs deep, iterative analysis using multiple LLMs and web searches. The system can be run locally for privacy or configured to use cloud-based LLMs for enhanced capabilities.
 
+<div align="center">
+  <a href="https://www.youtube.com/watch?v=0ISreg9q0p0">
+    <img src="https://img.youtube.com/vi/0ISreg9q0p0/0.jpg" alt="Local Deep Research">
+    <br>
+    <span>▶️ Watch Video</span>
+  </a>
+</div>
+
 ## Quick Start
 
 ```bash
@@ -143,6 +151,8 @@ The package automatically creates and manages configuration files in your user d
 
 ### Default Configuration Files
 
+If you prefere environment variables please refere to this file: https://github.com/LearningCircuit/local-deep-research/blob/main/docs/env_configuration.md
+
 When you first run the tool, it creates these configuration files:
 
 | File | Purpose |
@@ -215,6 +225,10 @@ The `OPENAI_ENDPOINT` provider can access any service with an OpenAI-compatible
 
 The system includes multiple search engines. Some require API keys:
 
+Use .env in config folder if .secrets.toml doesnt work. 
+
+You can also overwrite other settings via environment variables, e.g. to overwrite [web] port setting in settings.toml please use: **LDR_WEB__PORT=8080**
+
 ```toml
 # Add to .secrets.toml
 SERP_API_KEY = "your-serpapi-key-here"        # For Google results via SerpAPI

{local_deep_research-0.1.18.dist-info → local_deep_research-0.1.20.dist-info}/RECORD

@@ -1,13 +1,13 @@
 local_deep_research/__init__.py,sha256=pfHzjzYc6Szo8VCNLtFZRXyAlEz7CViY7r2fH9O7yms,584
 local_deep_research/citation_handler.py,sha256=v_fwTy-2XvUuoH3OQRzmBrvaiN7mBk8jbNfySslmt5g,4357
-local_deep_research/config.py,sha256=PAruLZutlrjkGOKrv49hk8U4q9JPWWgAKLiY8Ukpsks,8572
+local_deep_research/config.py,sha256=3g8-QPMrxoIMjHvyjSJBFUELmAIyOQFHApUnd8p50a8,9881
 local_deep_research/main.py,sha256=uQXtGQ6LtZNd5Qw63D5ke4Q_LjYimouWVSUknVsk3JQ,3645
 local_deep_research/report_generator.py,sha256=EvaArnWirMgg42fMzmZeJczoEYujEbJ2ryHHYuuoXx8,8058
 local_deep_research/search_system.py,sha256=yY3BEzX68vdtUcYF9h6lC3yVao0YA_NSBj6W3-RwlKk,15459
 local_deep_research/defaults/__init__.py,sha256=2Vvlkl-gmP_qPYWegE4JBgummypogl3VXrQ1XzptFDU,1381
-local_deep_research/defaults/llm_config.py,sha256=T03pntyNtOk1fvu-RZ-iEoh7L2D2hcICr8usIPpULuo,7870
-local_deep_research/defaults/local_collections.toml,sha256=_edVWVHrhunMfazjejhJlGPRkHKKIP51qQtNkMgNEiA,1406
-local_deep_research/defaults/main.toml,sha256=DLhFq88vdE2_psLaWhPV9BWPixqTHvR2Rllaj_rmjJ4,1403
+local_deep_research/defaults/llm_config.py,sha256=7wTIugVYD_ypG7Xwvu3DBt0yO8TWBf_drOIQOSOkdQQ,9628
+local_deep_research/defaults/local_collections.toml,sha256=zNa03PVnFrZ757JdZOuW6QDxkOc6ep5tG8baGBrMmXM,1778
+local_deep_research/defaults/main.toml,sha256=6Lzbc5sVLxMwu83bLBp_tpYOZgmtThCfPL1L42eTGro,1939
 local_deep_research/defaults/search_engines.toml,sha256=TYkOqVaZq9JPawz4fIPyGdkAtYa4t8F9H50VY-wv2ak,8101
 local_deep_research/utilties/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 local_deep_research/utilties/enums.py,sha256=TVAZiu9szNbdacfb7whgaQJJlSk7oYByADaAierD4CE,229
@@ -18,6 +18,9 @@ local_deep_research/web/__init__.py,sha256=3oHMatNu8r24FBtpojriIVbHYOVSHj4Q-quyc
 local_deep_research/web/app.py,sha256=5_VLNdhJOqdgacucglUdS_lVURNgYNbXhK9vME6JmzA,72431
 local_deep_research/web/static/css/styles.css,sha256=_26yBV1fKM51Dfv67CxKSbK8aeoYK5Tl7b2TPs5Whuo,24641
 local_deep_research/web/static/js/app.js,sha256=GPncdWpw2YNTs56JY-0tjTTr9JnX-fIZSZX0agwKZMU,172813
+local_deep_research/web/static/sounds/README.md,sha256=yNfVJIpKoSHSdAEj-lpxkjGy8F-OMStXCiIo1fY5I-0,1003
+local_deep_research/web/static/sounds/error.mp3,sha256=OM3K-pDxkPDCcptqb7c4bIwkHTQa7cLREs4xdYAODPs,3177
+local_deep_research/web/static/sounds/success.mp3,sha256=8EJRxWER-dt6vG6X6GDK3DNb8zoNa_1eDzusYJVcWLI,11818
 local_deep_research/web/templates/api_keys_config.html,sha256=jA8Y-nfUGJ1dTvbw2jK_8xPy2x6UG_5gHpbrTJAex2g,3527
 local_deep_research/web/templates/collections_config.html,sha256=Dci7KumXBON8rAXRX8TVjgqS-bbht7d6aQiedDUnxQ0,3560
 local_deep_research/web/templates/index.html,sha256=IW4cU5NgXVFXF6BxMhLuFzwkte_iYmLo3DQssxuYLZw,17490
@@ -29,27 +32,27 @@ local_deep_research/web/templates/settings_dashboard.html,sha256=De-v1KNdVvkXme5
 local_deep_research/web_search_engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 local_deep_research/web_search_engines/search_engine_base.py,sha256=QmhfjuHK2deomh8tARghKuYnF-5t3wwBB661odS2VtU,8065
 local_deep_research/web_search_engines/search_engine_factory.py,sha256=Sld6bYTwcyTxgVLx04t00sD7vfJhSHFOl6iiGJ08ZUE,11118
-local_deep_research/web_search_engines/search_engines_config.py,sha256=bNCuR09NOk5cjnKIgDQfhPipqmvDKeE7WP_6p8LLZf0,1979
+local_deep_research/web_search_engines/search_engines_config.py,sha256=5C0tCmy_Jpv1YHLZLlyS7h5B2XToYcWPAaBDEOsxMo0,2739
 local_deep_research/web_search_engines/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 local_deep_research/web_search_engines/engines/full_search.py,sha256=BuOz8dX-XocazCG7gGBKFnIY99FZtNFI0-Wq3fhsfp4,4689
-local_deep_research/web_search_engines/engines/meta_search_engine.py,sha256=Zky4sowCortEaIj1pBU0sKuggXr5izkubgrD7cM8IOQ,11485
+local_deep_research/web_search_engines/engines/meta_search_engine.py,sha256=0zU_L5thHzAmAx-BDsV6QNnSk10CfJ3quCRGjfC_Ys0,12130
 local_deep_research/web_search_engines/engines/search_engine_arxiv.py,sha256=cf8OzhSzE1zqaiOZ6EFQGy_6hTCJMaTysYd8rs1KJNU,15408
 local_deep_research/web_search_engines/engines/search_engine_brave.py,sha256=J242byUGG5ROQ_bh-mU292_t7Q7m20_9O0r1w5z6d9A,9688
 local_deep_research/web_search_engines/engines/search_engine_ddg.py,sha256=qK2i65dbPtr_ppoKPU_YA0mDqM_sDAvN6ZztvdFjsCk,4910
 local_deep_research/web_search_engines/engines/search_engine_github.py,sha256=qqipsw2ycjlRbR6mmMmxzGU3LEcFDJJJ7Ez7xUgWjRM,26768
 local_deep_research/web_search_engines/engines/search_engine_google_pse.py,sha256=YkXvBmgcqTImCxuyy6580SGRAvImGc6SzInXZgo1kNE,11294
 local_deep_research/web_search_engines/engines/search_engine_guardian.py,sha256=MW4WIwtNAwcpdigNXronyezAxr50EIZTV1NMedrAv2o,23912
-local_deep_research/web_search_engines/engines/search_engine_local.py,sha256=rfmPiA9DVmjbaB3KQtlq7s6BRMgHRgzP7AhktZNDw2M,36772
-local_deep_research/web_search_engines/engines/search_engine_local_all.py,sha256=CRNcxBzNd9kanyIJYaUDB7qfXYxVCvd4L2mX8jL73v0,5955
+local_deep_research/web_search_engines/engines/search_engine_local.py,sha256=qKgiohPL8oyvpT6S6jSmNFuR_vuNVVVqO7O4gwliLqw,37981
+local_deep_research/web_search_engines/engines/search_engine_local_all.py,sha256=7s7MHuFZTR28bDTxRUj19pzKv7Xzc5SG3yhtGG957eg,5981
 local_deep_research/web_search_engines/engines/search_engine_pubmed.py,sha256=MayfzM2R0XoI7cpXlG1XJ1ktfTN_6H-Xs9RmD89UAao,39236
 local_deep_research/web_search_engines/engines/search_engine_searxng.py,sha256=GMy6qDMSaVBtjWRm48XBu6TjLAy1HfcO2EFTwr8S9rk,18048
 local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py,sha256=6VMymjFJ7pyV2nv5dRfFofXgg0kG82rkwbICVnNDNH4,23352
 local_deep_research/web_search_engines/engines/search_engine_serpapi.py,sha256=XikEYnM-pAaR70VeAJ28lbqpRzCj4bCA9xY29taTV8g,9215
 local_deep_research/web_search_engines/engines/search_engine_wayback.py,sha256=astAvSLajDZ6rwgthJ3iBcHSWuDSYPO7uilIxaJhXmU,18132
 local_deep_research/web_search_engines/engines/search_engine_wikipedia.py,sha256=KSGJECbEcxZpVK-PhYsTCtzedSK0l1AjQmvGtx8KBks,9799
-local_deep_research-0.1.18.dist-info/licenses/LICENSE,sha256=Qg2CaTdu6SWnSqk1_JtgBPp_Da-LdqJDhT1Vt1MUc5s,1072
-local_deep_research-0.1.18.dist-info/METADATA,sha256=6F15QvEwfPPqHVEGRUCPEYVXKoR8PvbXOwzM9Vemem8,15013
-local_deep_research-0.1.18.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-local_deep_research-0.1.18.dist-info/entry_points.txt,sha256=u-Y6Z3MWtR3dmsTDFYhXyfkPv7mALUA7YAnY4Fi1XDs,97
-local_deep_research-0.1.18.dist-info/top_level.txt,sha256=h6-uVE_wSuLOcoWwT9szhX23mBWufu77MqmM25UfbCY,20
-local_deep_research-0.1.18.dist-info/RECORD,,
+local_deep_research-0.1.20.dist-info/licenses/LICENSE,sha256=Qg2CaTdu6SWnSqk1_JtgBPp_Da-LdqJDhT1Vt1MUc5s,1072
+local_deep_research-0.1.20.dist-info/METADATA,sha256=29URKDSkO8eCFRa5NkCoPIZ_lHYH5xOeK8ORQp5-v6k,15608
+local_deep_research-0.1.20.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
+local_deep_research-0.1.20.dist-info/entry_points.txt,sha256=u-Y6Z3MWtR3dmsTDFYhXyfkPv7mALUA7YAnY4Fi1XDs,97
+local_deep_research-0.1.20.dist-info/top_level.txt,sha256=h6-uVE_wSuLOcoWwT9szhX23mBWufu77MqmM25UfbCY,20
+local_deep_research-0.1.20.dist-info/RECORD,,

{local_deep_research-0.1.18.dist-info → local_deep_research-0.1.20.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (77.0.3)
+Generator: setuptools (78.0.2)
 Root-Is-Purelib: true
 Tag: py3-none-any