local-deep-research 0.1.26__py3-none-any.whl → 0.2.2__py3-none-any.whl

local_deep_research/{config.py → config/config_files.py}

@@ -1,39 +1,48 @@
-# local_deep_research/config.py
-from dynaconf import Dynaconf
-from pathlib import Path
 import logging
-from platformdirs import user_documents_dir
 import os
+import platform
+from pathlib import Path
+
+from dotenv import load_dotenv
+from dynaconf import Dynaconf
+from platformdirs import user_documents_dir
+
 # Setup logging
 logger = logging.getLogger(__name__)
-from dotenv import load_dotenv
-import platform
 
-# Get config directory
+
 def get_config_dir():
 
-    
     if platform.system() == "Windows":
         # Windows: Use Documents directory
         from platformdirs import user_documents_dir
-        config_dir = Path(user_documents_dir()) / "LearningCircuit" / "local-deep-research"
+
+        config_dir = (
+            Path(user_documents_dir()) / "LearningCircuit" / "local-deep-research"
+        )
     else:
         # Linux/Mac: Use standard config directory
         from platformdirs import user_config_dir
+
         config_dir = Path(user_config_dir("local_deep_research", "LearningCircuit"))
-    
+
     logger.info(f"Looking for config in: {config_dir}")
     return config_dir
+
+
 # Define config paths
 CONFIG_DIR = get_config_dir() / "config"
 CONFIG_DIR.mkdir(parents=True, exist_ok=True)
 SETTINGS_FILE = CONFIG_DIR / "settings.toml"
 SECRETS_FILE = CONFIG_DIR / ".secrets.toml"
-LLM_CONFIG_FILE = CONFIG_DIR / "llm_config.py"
 SEARCH_ENGINES_FILE = CONFIG_DIR / "search_engines.toml"
 
 LOCAL_COLLECTIONS_FILE = CONFIG_DIR / "local_collections.toml"
 
+# Define data directory for database
+DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "data"))
+os.makedirs(DATA_DIR, exist_ok=True)
+DB_PATH = os.path.join(DATA_DIR, "ldr.db")
 
 env_file = CONFIG_DIR / ".env"
 
@@ -41,7 +50,9 @@ if env_file.exists():
     logger.info(f"Loading environment variables from: {env_file}")
     load_dotenv(dotenv_path=env_file)
 else:
-    logger.warning(f"Warning: .env file not found at {env_file}. Trying secondary location.")
+    logger.warning(
+        f"Warning: .env file not found at {env_file}. Trying secondary location."
+    )
     env_file_secondary = get_config_dir() / ".env"
     if env_file_secondary.exists():
         get_config_dir() / "config"
@@ -56,114 +67,77 @@ docs_base = Path(user_documents_dir()) / "local_deep_research"
 os.environ["DOCS_DIR"] = str(docs_base)
 
 
-# Expose get_llm function
-def get_llm(*args, **kwargs):
-    """
-    Helper function to get LLM from llm_config.py
-    """
-    # Import here to avoid circular imports
-    import importlib.util
-    import sys
-    
-    llm_config_path = CONFIG_DIR / "llm_config.py"
-    
-    # If llm_config.py exists, use it
-    if llm_config_path.exists():
-        if str(CONFIG_DIR) not in sys.path:
-            sys.path.insert(0, str(CONFIG_DIR))
-            
-        spec = importlib.util.spec_from_file_location("llm_config", llm_config_path)
-        llm_config = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(llm_config)
-        
-        if hasattr(llm_config, "get_llm"):
-            return llm_config.get_llm(*args, **kwargs)
-    
-    # Fallback to utility function
-    from .utilties.llm_utils import get_model
-    return get_model(*args, **kwargs)
-
-# Expose get_search function
-def get_search(search_tool=None):
-    """
-    Helper function to get search engine
-    """
-    
-    # Use specified tool or default from settings
-    tool = search_tool or settings.search.tool
-    logger.info(f"Search tool is: {tool}")
-    
-    # Import here to avoid circular imports
-    from .web_search_engines.search_engine_factory import get_search as factory_get_search
-    
-    # Get search parameters
-    params = {
-        "search_tool": tool,
-        "llm_instance": get_llm(),
-        "max_results": settings.search.max_results,
-        "region": settings.search.region,
-        "time_period": settings.search.time_period,
-        "safe_search": settings.search.safe_search,
-        "search_snippets_only": settings.search.snippets_only,
-        "search_language": settings.search.search_language,
-        "max_filtered_results": settings.search.max_filtered_results
-    }
-    logger.info(f"Search config params: {params}")
-    # Create and return search engine
-    return factory_get_search(**params)
-
 def init_config_files():
     """Initialize config files if they don't exist"""
-    import shutil
     import os
-    import sys
     import platform
-    
+    import shutil
+
     # Ensure CONFIG_DIR exists with explicit creation
     os.makedirs(CONFIG_DIR, exist_ok=True)
-    
+
     # Get default files path with more reliable approach for Windows
     if platform.system() == "Windows":
         # Use a more reliable method on Windows
         from pkg_resources import resource_filename
+
         try:
-            defaults_dir = Path(resource_filename('local_deep_research', 'defaults'))
+            defaults_dir = Path(resource_filename("local_deep_research", "defaults"))
             logger.info(f"Using pkg_resources for Windows: {defaults_dir}")
-            
+
             # Create settings.toml if it doesn't exist (with explicit Windows paths)
             settings_file = os.path.join(CONFIG_DIR, "settings.toml")
             default_settings = os.path.join(defaults_dir, "main.toml")
             if not os.path.exists(settings_file) and os.path.exists(default_settings):
                 shutil.copyfile(default_settings, settings_file)
                 logger.info(f"Created settings.toml at {settings_file}")
-            
-            # Create llm_config.py if it doesn't exist
-            llm_config_file = os.path.join(CONFIG_DIR, "llm_config.py")
-            default_llm = os.path.join(defaults_dir, "llm_config.py")
-            if not os.path.exists(llm_config_file) and os.path.exists(default_llm):
-                shutil.copyfile(default_llm, llm_config_file)
-                logger.info(f"Created llm_config.py at {llm_config_file}")
-                
+                # Add note about database-first settings
+                with open(settings_file, "a") as f:
+                    f.write(
+                        "\n\n# NOTE: Settings in this file are used as fallback only.\n"
+                    )
+                    f.write(
+                        "# Settings stored in the database (ldr.db) take precedence.\n"
+                    )
+                    f.write(
+                        "# To modify settings permanently, use the web interface settings page.\n"
+                    )
+
             # Create local_collections.toml if it doesn't exist
             collections_file = os.path.join(CONFIG_DIR, "local_collections.toml")
             default_collections = os.path.join(defaults_dir, "local_collections.toml")
-            if not os.path.exists(collections_file) and os.path.exists(default_collections):
+            if not os.path.exists(collections_file) and os.path.exists(
+                default_collections
+            ):
                 shutil.copyfile(default_collections, collections_file)
                 logger.info(f"Created local_collections.toml at {collections_file}")
-            
+
             # Create search_engines.toml if it doesn't exist
             search_engines_file = os.path.join(CONFIG_DIR, "search_engines.toml")
             default_engines = os.path.join(defaults_dir, "search_engines.toml")
-            if not os.path.exists(search_engines_file) and os.path.exists(default_engines):
+            if not os.path.exists(search_engines_file) and os.path.exists(
+                default_engines
+            ):
                 shutil.copyfile(default_engines, search_engines_file)
                 logger.info(f"Created search_engines.toml at {search_engines_file}")
-
-                    # Create .env.template if it doesn't exist
+                # Add note about database-first settings
+                with open(search_engines_file, "a") as f:
+                    f.write(
+                        "\n\n# NOTE: Settings in this file are used as fallback only.\n"
+                    )
+                    f.write(
+                        "# Settings stored in the database (ldr.db) take precedence.\n"
+                    )
+                    f.write(
+                        "# To modify search settings permanently, use the web interface settings page.\n"
+                    )
+
+                # Create .env.template if it doesn't exist
             env_template_file = CONFIG_DIR / ".env.template"
             if not env_template_file.exists():
                 shutil.copy(defaults_dir / ".env.template", env_template_file)
                 logger.info(f"Created .env.template at {env_template_file}")
-                
+
                 # Optionally create an empty .env file if it doesn't exist
                 env_file = CONFIG_DIR / ".env"
                 if not env_file.exists():
@@ -176,53 +150,68 @@ def init_config_files():
         """Initialize config files if they don't exist"""
         import shutil
         from importlib.resources import files
-        
+
         # Get default files path
         try:
-            defaults_dir = files('local_deep_research.defaults')
+            defaults_dir = files("local_deep_research.defaults")
         except ImportError:
             # Fallback for older Python versions
             from pkg_resources import resource_filename
-            defaults_dir = Path(resource_filename('local_deep_research', 'defaults'))
-        
+
+            defaults_dir = Path(resource_filename("local_deep_research", "defaults"))
+
         # Create settings.toml if it doesn't exist
         settings_file = CONFIG_DIR / "settings.toml"
         if not settings_file.exists():
             shutil.copy(defaults_dir / "main.toml", settings_file)
             logger.info(f"Created settings.toml at {settings_file}")
-        
-        # Create llm_config.py if it doesn't exist
-        llm_config_file = CONFIG_DIR / "llm_config.py"
-        if not llm_config_file.exists():
-            shutil.copy(defaults_dir / "llm_config.py", llm_config_file)
-            logger.info(f"Created llm_config.py at {llm_config_file}")
-            
+            # Add note about database-first settings
+            with open(settings_file, "a") as f:
+                f.write(
+                    "\n\n# NOTE: Settings in this file are used as fallback only.\n"
+                )
+                f.write("# Settings stored in the database (ldr.db) take precedence.\n")
+                f.write(
+                    "# To modify settings permanently, use the web interface settings page.\n"
+                )
+
         # Create local_collections.toml if it doesn't exist
         collections_file = CONFIG_DIR / "local_collections.toml"
         if not collections_file.exists():
             shutil.copy(defaults_dir / "local_collections.toml", collections_file)
             logger.info(f"Created local_collections.toml at {collections_file}")
-        
+
         # Create search_engines.toml if it doesn't exist
         search_engines_file = CONFIG_DIR / "search_engines.toml"
         if not search_engines_file.exists():
             shutil.copy(defaults_dir / "search_engines.toml", search_engines_file)
             logger.info(f"Created search_engines.toml at {search_engines_file}")
+            # Add note about database-first settings
+            with open(search_engines_file, "a") as f:
+                f.write(
+                    "\n\n# NOTE: Settings in this file are used as fallback only.\n"
+                )
+                f.write("# Settings stored in the database (ldr.db) take precedence.\n")
+                f.write(
+                    "# To modify search settings permanently, use the web interface settings page.\n"
+                )
+
         env_template_file = CONFIG_DIR / ".env.template"
         if not env_template_file.exists():
             shutil.copy(defaults_dir / ".env.template", env_template_file)
             logger.info(f"Created .env.template at {env_template_file}")
-            
+
             # Optionally create an empty .env file if it doesn't exist
             env_file = CONFIG_DIR / ".env"
             if not env_file.exists():
                 with open(env_file, "w") as f:
                     f.write("# Add your environment variables here\n")
-                logger.info(f"Created empty .env file at {env_file}")            
+                logger.info(f"Created empty .env file at {env_file}")
         secrets_file = CONFIG_DIR / ".secrets.toml"
         if not secrets_file.exists():
             with open(secrets_file, "w") as f:
-                f.write("""
+                f.write(
+                    """
     # ANTHROPIC_API_KEY = "your-api-key-here"
     # OPENAI_API_KEY = "your-openai-key-here"
     # GOOGLE_API_KEY = "your-google-key-here"
@@ -230,19 +219,20 @@ def init_config_files():
     # GUARDIAN_API_KEY = "your-api-key-here"
     # GOOGLE_PSE_API_KEY = "your-google-api-key-here"
     # GOOGLE_PSE_ENGINE_ID = "your-programmable-search-engine-id-here"
-    """)
-            
-# Initialize config files on import
-init_config_files()
+    """
+                )
+
 
-# Use an absolute path to your .secrets.toml for testing
-secrets_file = Path(SECRETS_FILE)
+# Add a comment to explain the DB-first settings approach
+logger.info(
+    "Using database-first settings approach. TOML files are used as fallback only."
+)
 
 settings = Dynaconf(
     settings_files=[
         str(SETTINGS_FILE),
         str(LOCAL_COLLECTIONS_FILE),
-        str(SEARCH_ENGINES_FILE),  
+        str(SEARCH_ENGINES_FILE),
     ],
     secrets=str(SECRETS_FILE),
     env_prefix="LDR",
@@ -251,3 +241,5 @@ settings = Dynaconf(
     env_file=str(CONFIG_DIR / ".env"),
 )
 
+# Initialize config files on import
+init_config_files()