local-deep-research 0.2.3__py3-none-any.whl → 0.3.0__py3-none-any.whl

local_deep_research/web/services/settings_manager.py

@@ -1,15 +1,14 @@
+import importlib.resources as pkg_resources
+import json
 import logging
 import os
-from pathlib import Path
 from typing import Any, Dict, Optional, Union
 
-import toml
 from sqlalchemy import func
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import Session
 
-from ...config.config_files import get_config_dir
-from ...config.config_files import settings as dynaconf_settings
+from ... import defaults
 from ..database.models import Setting, SettingType
 from ..models.settings import (
     AppSetting,
@@ -23,6 +22,25 @@ from ..models.settings import (
 logger = logging.getLogger(__name__)
 
 
+def check_env_setting(key: str) -> str | None:
+    """
+    Checks environment variables for a particular setting.
+
+    Args:
+        key: The database key for the setting.
+
+    Returns:
+        The setting from the environment variables, or None if the variable
+        is not set.
+
+    """
+    env_variable_name = f"LDR_{"_".join(key.split(".")).upper()}"
+    env_value = os.getenv(env_variable_name)
+    if env_value is not None:
+        logger.debug(f"Overriding {key} setting from environment variable.")
+    return env_value
+
+
 class SettingsManager:
     """
     Manager for handling application settings with database storage and file fallback.
@@ -37,30 +55,29 @@ class SettingsManager:
             db_session: SQLAlchemy session for database operations
         """
         self.db_session = db_session
-        self.config_dir = get_config_dir() / "config"
-        self.settings_file = self.config_dir / "settings.toml"
-        self.search_engines_file = self.config_dir / "search_engines.toml"
-        self.collections_file = self.config_dir / "local_collections.toml"
-        self.secrets_file = self.config_dir / ".secrets.toml"
         self.db_first = True  # Always prioritize DB settings
 
-        # In-memory cache for settings
-        self._settings_cache: Dict[str, Any] = {}
+        # Load default settings.
+        default_settings = pkg_resources.read_text(defaults, "default_settings.json")
+        self.default_settings = json.loads(default_settings)
 
-    def get_setting(self, key: str, default: Any = None) -> Any:
+    def get_setting(self, key: str, default: Any = None, check_env: bool = True) -> Any:
         """
         Get a setting value
 
         Args:
             key: Setting key
             default: Default value if setting is not found
+            check_env: If true, it will check the environment variable for
+                this setting before reading from the DB.
 
         Returns:
             Setting value or default if not found
         """
-        # Check in-memory cache first (highest priority)
-        if key in self._settings_cache:
-            return self._settings_cache[key]
+        if check_env:
+            env_value = check_env_setting(key)
+            if env_value is not None:
+                return env_value
 
         # If using database first approach and session available, check database
         if self.db_first and self.db_session:
@@ -73,7 +90,6 @@ class SettingsManager:
                 if len(settings) == 1:
                     # This is a bottom-level key.
                     value = settings[0].value
-                    self._settings_cache[key] = value
                     return value
                 elif len(settings) > 1:
                     # This is a higher-level key.
@@ -86,22 +102,6 @@ class SettingsManager:
             except SQLAlchemyError as e:
                 logger.error(f"Error retrieving setting {key} from database: {e}")
 
-        # Fall back to Dynaconf settings
-        try:
-            # Split the key into sections
-            parts = key.split(".")
-            if len(parts) == 2:
-                section, setting = parts
-                if hasattr(dynaconf_settings, section) and hasattr(
-                    getattr(dynaconf_settings, section), setting
-                ):
-                    value = getattr(getattr(dynaconf_settings, section), setting)
-                    # Update cache and return
-                    self._settings_cache[key] = value
-                    return value
-        except Exception as e:
-            logger.debug(f"Error retrieving setting {key} from Dynaconf: {e}")
-
         # Return default if not found
         return default
 
@@ -117,9 +117,6 @@ class SettingsManager:
         Returns:
             True if successful, False otherwise
         """
-        # Always update cache
-        self._settings_cache[key] = value
-
         # Always update database if available
         if self.db_session:
             try:
@@ -172,29 +169,36 @@ class SettingsManager:
         """
         result = {}
 
-        # Start with memory cache (highest priority)
-        result.update(self._settings_cache)
-
         # Add database settings if available
         if self.db_session:
             try:
                 for setting in self.db_session.query(Setting).all():
-                    result[setting.key] = setting.value
+                    result[setting.key] = dict(
+                        value=setting.value,
+                        type=setting.type.name,
+                        name=setting.name,
+                        description=setting.description,
+                        category=setting.category,
+                        ui_element=setting.ui_element,
+                        options=setting.options,
+                        min_value=setting.min_value,
+                        max_value=setting.max_value,
+                        step=setting.step,
+                        visible=setting.visible,
+                        editable=setting.editable,
+                    )
+
+                    # Override from the environment variables if needed.
+                    env_value = check_env_setting(setting.key)
+                    if env_value is not None:
+                        result[setting.key]["value"] = env_value
+                        # Mark it as non-editable, because changes to the DB
+                        # value have no effect as long as the environment
+                        # variable is set.
+                        result[setting.key]["editable"] = False
             except SQLAlchemyError as e:
                 logger.error(f"Error retrieving all settings from database: {e}")
 
-        # Fill in missing values from Dynaconf (lowest priority)
-        for section in ["llm", "search", "report", "app", "web"]:
-            if hasattr(dynaconf_settings, section):
-                section_obj = getattr(dynaconf_settings, section)
-                for key in dir(section_obj):
-                    if not key.startswith("_") and not callable(
-                        getattr(section_obj, key)
-                    ):
-                        full_key = f"{section}.{key}"
-                        if full_key not in result:
-                            result[full_key] = getattr(section_obj, key)
-
         return result
 
     def create_or_update_setting(
@@ -278,9 +282,6 @@ class SettingsManager:
                 )
                 self.db_session.add(db_setting)
 
-            # Update cache
-            self._settings_cache[setting_obj.key] = setting_obj.value
-
             if commit:
                 self.db_session.commit()
 
@@ -307,10 +308,6 @@ class SettingsManager:
             return False
 
         try:
-            # Remove from cache
-            if key in self._settings_cache:
-                del self._settings_cache[key]
-
             # Remove from database
             result = self.db_session.query(Setting).filter(Setting.key == key).delete()
 
@@ -323,181 +320,32 @@ class SettingsManager:
             self.db_session.rollback()
             return False
 
-    def export_to_file(self, setting_type: Optional[SettingType] = None) -> bool:
+    def load_from_defaults_file(self, commit: bool = True, **kwargs: Any) -> None:
         """
-        Export settings to file
+        Import settings from the defaults settings file.
 
         Args:
-            setting_type: Type of settings to export (or all if None)
+            commit: Whether to commit changes to database
+            **kwargs: Will be passed to `import_settings`.
 
-        Returns:
-            True if successful, False otherwise
         """
-        try:
-            # Get settings
-            settings = self.get_all_settings()
-
-            # Group by section
-            sections = {}
-            for key, value in settings.items():
-                # Split key into section and name
-                parts = key.split(".", 1)
-                if len(parts) == 2:
-                    section, name = parts
-                    if section not in sections:
-                        sections[section] = {}
-                    sections[section][name] = value
-
-            # Write to appropriate file
-            if setting_type == SettingType.LLM:
-                file_path = self.settings_file
-                section_name = "llm"
-            elif setting_type == SettingType.SEARCH:
-                file_path = self.search_engines_file
-                section_name = "search"
-            elif setting_type == SettingType.REPORT:
-                file_path = self.settings_file
-                section_name = "report"
-            else:
-                # Write all sections to appropriate files
-                for section_name, section_data in sections.items():
-                    if section_name == "search":
-                        self._write_section_to_file(
-                            self.search_engines_file, section_name, section_data
-                        )
-                    else:
-                        self._write_section_to_file(
-                            self.settings_file, section_name, section_data
-                        )
-                return True
-
-            # Write specific section
-            if section_name in sections:
-                return self._write_section_to_file(
-                    file_path, section_name, sections[section_name]
-                )
-
-            return False
-
-        except Exception as e:
-            logger.error(f"Error exporting settings to file: {e}")
-            return False
+        self.import_settings(self.default_settings, commit=commit, **kwargs)
 
-    def import_from_file(
-        self, setting_type: Optional[SettingType] = None, commit: bool = True
-    ) -> bool:
+    def db_version_matches_defaults(self) -> bool:
         """
-        Import settings from file
-
-        Args:
-            setting_type: Type of settings to import (or all if None)
-            commit: Whether to commit changes to database
-
         Returns:
-            True if successful, False otherwise
-        """
-        try:
-            # Determine file path
-            if (
-                setting_type == SettingType.LLM
-                or setting_type == SettingType.APP
-                or setting_type == SettingType.REPORT
-            ):
-                file_path = self.settings_file
-            elif setting_type == SettingType.SEARCH:
-                file_path = self.search_engines_file
-            else:
-                # Import from all files
-                success = True
-                success &= self.import_from_file(SettingType.LLM, commit=False)
-                success &= self.import_from_file(SettingType.SEARCH, commit=False)
-                success &= self.import_from_file(SettingType.REPORT, commit=False)
-                success &= self.import_from_file(SettingType.APP, commit=False)
-
-                # Commit all changes at once
-                if commit and self.db_session:
-                    self.db_session.commit()
-
-                return success
-
-            # Read from file
-            if not os.path.exists(file_path):
-                logger.warning(f"Settings file does not exist: {file_path}")
-                return False
-
-            # Parse TOML file
-            with open(file_path, "r") as f:
-                file_data = toml.load(f)
-
-            # Extract section based on setting type
-            section_name = setting_type.value.lower() if setting_type else None
-            if section_name and section_name in file_data:
-                section_data = file_data[section_name]
-            else:
-                section_data = file_data
-
-            # Import settings
-            for key, value in section_data.items():
-                if section_name:
-                    full_key = f"{section_name}.{key}"
-                else:
-                    # Try to determine section from key structure
-                    if "." in key:
-                        full_key = key
-                    else:
-                        # Assume it's an app setting
-                        full_key = f"app.{key}"
-
-                self.set_setting(full_key, value, commit=False)
-
-            # Commit if requested
-            if commit and self.db_session:
-                self.db_session.commit()
-
-            return True
+            True if the version saved in the DB matches that in the default
+            settings file.
 
-        except Exception as e:
-            logger.error(f"Error importing settings from file: {e}")
-            if self.db_session:
-                self.db_session.rollback()
-            return False
-
-    def _write_section_to_file(
-        self, file_path: Path, section: str, data: Dict[str, Any]
-    ) -> bool:
         """
-        Write a section of settings to a TOML file
-
-        Args:
-            file_path: Path to the file
-            section: Section name
-            data: Section data
+        db_version = self.get_setting("app.version")
+        default_version = self.default_settings["app.version"]["value"]
+        logger.debug(
+            f"App version saved in DB is {db_version}, have default "
+            f"settings from version {default_version}."
+        )
 
-        Returns:
-            True if successful, False otherwise
-        """
-        try:
-            # Create file if it doesn't exist
-            if not os.path.exists(file_path):
-                file_path.parent.mkdir(parents=True, exist_ok=True)
-                with open(file_path, "w") as f:
-                    f.write(f"[{section}]\n")
-
-            # Read existing file
-            with open(file_path, "r") as f:
-                file_data = toml.load(f)
-
-            # Update section
-            file_data[section] = data
-
-            # Write back to file
-            with open(file_path, "w") as f:
-                toml.dump(file_data, f)
-
-            return True
-        except Exception as e:
-            logger.error(f"Error writing section {section} to {file_path}: {e}")
-            return False
+        return db_version == default_version
 
     @classmethod
     def get_instance(cls, db_session: Optional[Session] = None) -> "SettingsManager":
@@ -518,100 +366,49 @@ class SettingsManager:
 
         return cls._instance
 
-    def import_default_settings(
-        self, main_settings_file, search_engines_file, collections_file
-    ):
+    def import_settings(
+        self,
+        settings_data: Dict[str, Any],
+        commit: bool = True,
+        overwrite: bool = True,
+        delete_extra: bool = False,
+    ) -> None:
         """
-        Import settings directly from default files
+        Import settings directly from the export format. This can be used to
+        re-import settings that have been exported with `get_all_settings()`.
 
         Args:
-            main_settings_file: Path to the main settings.toml file
-            search_engines_file: Path to the search_engines.toml file
-            collections_file: Path to the local_collections.toml file
+            settings_data: The raw settings data to import.
+            commit: Whether to commit the DB after loading the settings.
+            overwrite: If true, it will overwrite the value of settings that
+                are already in the database.
+            delete_extra: If true, it will delete any settings that are in
+                the database but don't have a corresponding entry in
+                `settings_data`.
 
-        Returns:
-            True if successful, False otherwise
         """
-        if not self.db_session:
-            logger.warning(
-                "No database session available, cannot import default settings"
-            )
-            return False
-
-        try:
-            # Import settings from main settings file
-            if os.path.exists(main_settings_file):
-                with open(main_settings_file, "r") as f:
-                    main_data = toml.load(f)
-
-                # Process each section in the main settings file
-                for section, values in main_data.items():
-                    if section in ["web", "llm", "general", "app"]:
-                        setting_type = None
-                        if section == "web" or section == "app":
-                            setting_type = SettingType.APP
-                            prefix = "app"
-                        elif section == "llm":
-                            setting_type = SettingType.LLM
-                            prefix = "llm"
-                        else:  # general section
-                            # Map general settings to appropriate types
-                            prefix = None
-                            for key, value in values.items():
-                                if key in [
-                                    "enable_fact_checking",
-                                    "knowledge_accumulation",
-                                    "knowledge_accumulation_context_limit",
-                                    "output_dir",
-                                ]:
-                                    self._create_setting(
-                                        f"report.{key}", value, SettingType.REPORT
-                                    )
-
-                        # Add settings with correct prefix
-                        if prefix:
-                            for key, value in values.items():
-                                self._create_setting(
-                                    f"{prefix}.{key}", value, setting_type
-                                )
-
-                    elif section == "search":
-                        # Search settings go to search type
-                        for key, value in values.items():
-                            self._create_setting(
-                                f"search.{key}", value, SettingType.SEARCH
-                            )
-
-                    elif section == "report":
-                        # Report settings
-                        for key, value in values.items():
-                            self._create_setting(
-                                f"report.{key}", value, SettingType.REPORT
-                            )
-
-            # Import settings from search engines file
-            if os.path.exists(search_engines_file):
-                with open(search_engines_file, "r") as f:
-                    search_data = toml.load(f)
-
-                # Find search section in search engines file
-                if "search" in search_data:
-                    for key, value in search_data["search"].items():
-                        # Skip complex sections that are nested
-                        if not isinstance(value, dict):
-                            self._create_setting(
-                                f"search.{key}", value, SettingType.SEARCH
-                            )
-
-            # Commit changes
+        for key, setting_values in settings_data.items():
+            if not overwrite:
+                existing_value = self.get_setting(key)
+                if existing_value is not None:
+                    # Preserve the value from this setting.
+                    setting_values["value"] = existing_value
+
+            # Delete any existing setting so we can completely overwrite it.
+            self.delete_setting(key, commit=False)
+
+            setting = Setting(key=key, **setting_values)
+            self.db_session.add(setting)
+
+        if delete_extra:
+            all_settings = self.get_all_settings()
+            for key in all_settings:
+                if key not in settings_data:
+                    logger.debug(f"Deleting extraneous setting: {key}")
+                    self.delete_setting(key, commit=False)
+
+        if commit:
             self.db_session.commit()
-            return True
-
-        except Exception as e:
-            logger.error(f"Error importing default settings: {e}")
-            if self.db_session:
-                self.db_session.rollback()
-            return False
 
     def _create_setting(self, key, value, setting_type):
         """Create a setting with appropriate metadata"""
@@ -662,8 +459,4 @@ class SettingsManager:
         }
 
         # Create the setting in the database
-        db_setting = self.create_or_update_setting(setting_dict, commit=False)
-
-        # Also update cache
-        if db_setting:
-            self._settings_cache[key] = value
+        self.create_or_update_setting(setting_dict, commit=False)

local_deep_research/web/services/settings_service.py

@@ -1,8 +1,7 @@
 import logging
-from pathlib import Path
 from typing import Any, Dict, Optional, Union
 
-from ..database.models import Setting, SettingType
+from ..database.models import Setting
 from .settings_manager import SettingsManager
 
 # Initialize logger
@@ -55,46 +54,18 @@ def set_setting(key: str, value: Any, commit: bool = True, db_session=None) -> b
     return manager.set_setting(key, value, commit)
 
 
-def get_all_settings(
-    setting_type: Optional[SettingType] = None, db_session=None
-) -> Dict[str, Any]:
+def get_all_settings(db_session=None) -> Dict[str, Any]:
     """
     Get all settings, optionally filtered by type
 
     Args:
-        setting_type: Optional filter by type
         db_session: Optional database session
 
     Returns:
         Dict[str, Any]: Dictionary of settings
     """
     manager = get_settings_manager(db_session)
-    return manager.get_all_settings(setting_type)
-
-
-def get_all_settings_as_dict(db_session=None) -> Dict[str, Dict[str, Any]]:
-    """
-    Get all settings as a structured dictionary
-
-    Args:
-        db_session: Optional database session
-
-    Returns:
-        Dict[str, Dict[str, Any]]: Dictionary of settings grouped by type
-    """
-    # Get settings manager
-    manager = get_settings_manager(db_session)
-
-    # Get all settings
-    all_settings = {}
-
-    for setting_type in SettingType:
-        type_key = setting_type.value.lower()
-        type_settings = manager.get_all_settings(setting_type)
-        if type_settings:
-            all_settings[type_key] = type_settings
-
-    return all_settings
+    return manager.get_all_settings()
 
 
 def create_or_update_setting(
@@ -147,30 +118,6 @@ def bulk_update_settings(
     return success
 
 
-def import_settings_from_file(
-    main_settings_file: Union[str, Path],
-    search_engines_file: Union[str, Path],
-    collections_file: Union[str, Path],
-    db_session=None,
-) -> bool:
-    """
-    Import settings from default configuration files
-
-    Args:
-        main_settings_file: Path to the main settings file
-        search_engines_file: Path to the search engines file
-        collections_file: Path to the collections file
-        db_session: Optional database session
-
-    Returns:
-        bool: True if import was successful
-    """
-    manager = get_settings_manager(db_session)
-    return manager.import_default_settings(
-        main_settings_file, search_engines_file, collections_file
-    )
-
-
 def validate_setting(setting: Setting, value: Any) -> tuple[bool, Optional[str]]:
     """
     Validate a setting value based on its type and constraints

local_deep_research/web/static/js/components/research.js

@@ -1043,7 +1043,7 @@
                 // Find the provider and model settings
                 const providerSetting = data.settings.value["provider"];
                 const modelSetting = data.settings.value["model"];
-                const customEndpointUrl = data.settings.value["openai_endpoint_url"];
+                const customEndpointUrl = data.settings.value["openai_endpoint.url"];
 
                 // Update provider dropdown if we have a valid provider
                 if (providerSetting && modelProviderSelect) {