local-deep-research 0.1.26__py3-none-any.whl → 0.2.0__py3-none-any.whl

local_deep_research/web/services/settings_manager.py

@@ -0,0 +1,669 @@
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+import toml
+from sqlalchemy import func
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
+
+from ...config.config_files import get_config_dir
+from ...config.config_files import settings as dynaconf_settings
+from ..database.models import Setting, SettingType
+from ..models.settings import (
+    AppSetting,
+    BaseSetting,
+    LLMSetting,
+    ReportSetting,
+    SearchSetting,
+)
+
+# Setup logging
+logger = logging.getLogger(__name__)
+
+
+class SettingsManager:
+    """
+    Manager for handling application settings with database storage and file fallback.
+    Provides methods to get and set settings, with the ability to override settings in memory.
+    """
+
+    def __init__(self, db_session: Session):
+        """
+        Initialize the settings manager
+
+        Args:
+            db_session: SQLAlchemy session for database operations
+        """
+        self.db_session = db_session
+        self.config_dir = get_config_dir() / "config"
+        self.settings_file = self.config_dir / "settings.toml"
+        self.search_engines_file = self.config_dir / "search_engines.toml"
+        self.collections_file = self.config_dir / "local_collections.toml"
+        self.secrets_file = self.config_dir / ".secrets.toml"
+        self.db_first = True  # Always prioritize DB settings
+
+        # In-memory cache for settings
+        self._settings_cache: Dict[str, Any] = {}
+
+    def get_setting(self, key: str, default: Any = None) -> Any:
+        """
+        Get a setting value
+
+        Args:
+            key: Setting key
+            default: Default value if setting is not found
+
+        Returns:
+            Setting value or default if not found
+        """
+        # Check in-memory cache first (highest priority)
+        if key in self._settings_cache:
+            return self._settings_cache[key]
+
+        # If using database first approach and session available, check database
+        if self.db_first and self.db_session:
+            try:
+                settings = (
+                    self.db_session.query(Setting)
+                    # This will find exact matches and any subkeys.
+                    .filter(Setting.key.startswith(key)).all()
+                )
+                if len(settings) == 1:
+                    # This is a bottom-level key.
+                    value = settings[0].value
+                    self._settings_cache[key] = value
+                    return value
+                elif len(settings) > 1:
+                    # This is a higher-level key.
+                    settings_map = {
+                        s.key.removeprefix(f"{key}."): s.value for s in settings
+                    }
+                    # We deliberately don't update the cache here to avoid
+                    # conflicts between low-level keys and their parent keys.
+                    return settings_map
+            except SQLAlchemyError as e:
+                logger.error(f"Error retrieving setting {key} from database: {e}")
+
+        # Fall back to Dynaconf settings
+        try:
+            # Split the key into sections
+            parts = key.split(".")
+            if len(parts) == 2:
+                section, setting = parts
+                if hasattr(dynaconf_settings, section) and hasattr(
+                    getattr(dynaconf_settings, section), setting
+                ):
+                    value = getattr(getattr(dynaconf_settings, section), setting)
+                    # Update cache and return
+                    self._settings_cache[key] = value
+                    return value
+        except Exception as e:
+            logger.debug(f"Error retrieving setting {key} from Dynaconf: {e}")
+
+        # Return default if not found
+        return default
+
+    def set_setting(self, key: str, value: Any, commit: bool = True) -> bool:
+        """
+        Set a setting value
+
+        Args:
+            key: Setting key
+            value: Setting value
+            commit: Whether to commit the change
+
+        Returns:
+            True if successful, False otherwise
+        """
+        # Always update cache
+        self._settings_cache[key] = value
+
+        # Always update database if available
+        if self.db_session:
+            try:
+                setting = (
+                    self.db_session.query(Setting).filter(Setting.key == key).first()
+                )
+                if setting:
+                    setting.value = value
+                    setting.updated_at = (
+                        func.now()
+                    )  # Explicitly set the current timestamp
+                else:
+                    # Determine setting type from key
+                    setting_type = SettingType.APP
+                    if key.startswith("llm."):
+                        setting_type = SettingType.LLM
+                    elif key.startswith("search."):
+                        setting_type = SettingType.SEARCH
+                    elif key.startswith("report."):
+                        setting_type = SettingType.REPORT
+
+                    # Create a new setting
+                    new_setting = Setting(
+                        key=key,
+                        value=value,
+                        type=setting_type,
+                        name=key.split(".")[-1].replace("_", " ").title(),
+                        description=f"Setting for {key}",
+                    )
+                    self.db_session.add(new_setting)
+
+                if commit:
+                    self.db_session.commit()
+
+                return True
+            except SQLAlchemyError as e:
+                logger.error(f"Error setting value for {key}: {e}")
+                self.db_session.rollback()
+                return False
+
+        # No database session, only update cache
+        return True
+
+    def get_all_settings(self) -> Dict[str, Any]:
+        """
+        Get all settings
+
+        Returns:
+            Dictionary of all settings
+        """
+        result = {}
+
+        # Start with memory cache (highest priority)
+        result.update(self._settings_cache)
+
+        # Add database settings if available
+        if self.db_session:
+            try:
+                for setting in self.db_session.query(Setting).all():
+                    result[setting.key] = setting.value
+            except SQLAlchemyError as e:
+                logger.error(f"Error retrieving all settings from database: {e}")
+
+        # Fill in missing values from Dynaconf (lowest priority)
+        for section in ["llm", "search", "report", "app", "web"]:
+            if hasattr(dynaconf_settings, section):
+                section_obj = getattr(dynaconf_settings, section)
+                for key in dir(section_obj):
+                    if not key.startswith("_") and not callable(
+                        getattr(section_obj, key)
+                    ):
+                        full_key = f"{section}.{key}"
+                        if full_key not in result:
+                            result[full_key] = getattr(section_obj, key)
+
+        return result
+
+    def create_or_update_setting(
+        self, setting: Union[BaseSetting, Dict[str, Any]], commit: bool = True
+    ) -> Optional[Setting]:
+        """
+        Create or update a setting
+
+        Args:
+            setting: Setting object or dictionary
+            commit: Whether to commit the change
+
+        Returns:
+            The created or updated Setting model, or None if failed
+        """
+        if not self.db_session:
+            logger.warning(
+                "No database session available, cannot create/update setting"
+            )
+            return None
+
+        # Convert dict to BaseSetting if needed
+        if isinstance(setting, dict):
+            # Determine type from key if not specified
+            if "type" not in setting and "key" in setting:
+                key = setting["key"]
+                if key.startswith("llm."):
+                    setting_obj = LLMSetting(**setting)
+                elif key.startswith("search."):
+                    setting_obj = SearchSetting(**setting)
+                elif key.startswith("report."):
+                    setting_obj = ReportSetting(**setting)
+                else:
+                    setting_obj = AppSetting(**setting)
+            else:
+                # Use generic BaseSetting
+                setting_obj = BaseSetting(**setting)
+        else:
+            setting_obj = setting
+
+        try:
+            # Check if setting exists
+            db_setting = (
+                self.db_session.query(Setting)
+                .filter(Setting.key == setting_obj.key)
+                .first()
+            )
+
+            if db_setting:
+                # Update existing setting
+                db_setting.value = setting_obj.value
+                db_setting.name = setting_obj.name
+                db_setting.description = setting_obj.description
+                db_setting.category = setting_obj.category
+                db_setting.ui_element = setting_obj.ui_element
+                db_setting.options = setting_obj.options
+                db_setting.min_value = setting_obj.min_value
+                db_setting.max_value = setting_obj.max_value
+                db_setting.step = setting_obj.step
+                db_setting.visible = setting_obj.visible
+                db_setting.editable = setting_obj.editable
+                db_setting.updated_at = (
+                    func.now()
+                )  # Explicitly set the current timestamp
+            else:
+                # Create new setting
+                db_setting = Setting(
+                    key=setting_obj.key,
+                    value=setting_obj.value,
+                    type=SettingType[setting_obj.type.upper()],
+                    name=setting_obj.name,
+                    description=setting_obj.description,
+                    category=setting_obj.category,
+                    ui_element=setting_obj.ui_element,
+                    options=setting_obj.options,
+                    min_value=setting_obj.min_value,
+                    max_value=setting_obj.max_value,
+                    step=setting_obj.step,
+                    visible=setting_obj.visible,
+                    editable=setting_obj.editable,
+                )
+                self.db_session.add(db_setting)
+
+            # Update cache
+            self._settings_cache[setting_obj.key] = setting_obj.value
+
+            if commit:
+                self.db_session.commit()
+
+            return db_setting
+
+        except SQLAlchemyError as e:
+            logger.error(f"Error creating/updating setting {setting_obj.key}: {e}")
+            self.db_session.rollback()
+            return None
+
+    def delete_setting(self, key: str, commit: bool = True) -> bool:
+        """
+        Delete a setting
+
+        Args:
+            key: Setting key
+            commit: Whether to commit the change
+
+        Returns:
+            True if successful, False otherwise
+        """
+        if not self.db_session:
+            logger.warning("No database session available, cannot delete setting")
+            return False
+
+        try:
+            # Remove from cache
+            if key in self._settings_cache:
+                del self._settings_cache[key]
+
+            # Remove from database
+            result = self.db_session.query(Setting).filter(Setting.key == key).delete()
+
+            if commit:
+                self.db_session.commit()
+
+            return result > 0
+        except SQLAlchemyError as e:
+            logger.error(f"Error deleting setting {key}: {e}")
+            self.db_session.rollback()
+            return False
+
+    def export_to_file(self, setting_type: Optional[SettingType] = None) -> bool:
+        """
+        Export settings to file
+
+        Args:
+            setting_type: Type of settings to export (or all if None)
+
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            # Get settings
+            settings = self.get_all_settings()
+
+            # Group by section
+            sections = {}
+            for key, value in settings.items():
+                # Split key into section and name
+                parts = key.split(".", 1)
+                if len(parts) == 2:
+                    section, name = parts
+                    if section not in sections:
+                        sections[section] = {}
+                    sections[section][name] = value
+
+            # Write to appropriate file
+            if setting_type == SettingType.LLM:
+                file_path = self.settings_file
+                section_name = "llm"
+            elif setting_type == SettingType.SEARCH:
+                file_path = self.search_engines_file
+                section_name = "search"
+            elif setting_type == SettingType.REPORT:
+                file_path = self.settings_file
+                section_name = "report"
+            else:
+                # Write all sections to appropriate files
+                for section_name, section_data in sections.items():
+                    if section_name == "search":
+                        self._write_section_to_file(
+                            self.search_engines_file, section_name, section_data
+                        )
+                    else:
+                        self._write_section_to_file(
+                            self.settings_file, section_name, section_data
+                        )
+                return True
+
+            # Write specific section
+            if section_name in sections:
+                return self._write_section_to_file(
+                    file_path, section_name, sections[section_name]
+                )
+
+            return False
+
+        except Exception as e:
+            logger.error(f"Error exporting settings to file: {e}")
+            return False
+
+    def import_from_file(
+        self, setting_type: Optional[SettingType] = None, commit: bool = True
+    ) -> bool:
+        """
+        Import settings from file
+
+        Args:
+            setting_type: Type of settings to import (or all if None)
+            commit: Whether to commit changes to database
+
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            # Determine file path
+            if (
+                setting_type == SettingType.LLM
+                or setting_type == SettingType.APP
+                or setting_type == SettingType.REPORT
+            ):
+                file_path = self.settings_file
+            elif setting_type == SettingType.SEARCH:
+                file_path = self.search_engines_file
+            else:
+                # Import from all files
+                success = True
+                success &= self.import_from_file(SettingType.LLM, commit=False)
+                success &= self.import_from_file(SettingType.SEARCH, commit=False)
+                success &= self.import_from_file(SettingType.REPORT, commit=False)
+                success &= self.import_from_file(SettingType.APP, commit=False)
+
+                # Commit all changes at once
+                if commit and self.db_session:
+                    self.db_session.commit()
+
+                return success
+
+            # Read from file
+            if not os.path.exists(file_path):
+                logger.warning(f"Settings file does not exist: {file_path}")
+                return False
+
+            # Parse TOML file
+            with open(file_path, "r") as f:
+                file_data = toml.load(f)
+
+            # Extract section based on setting type
+            section_name = setting_type.value.lower() if setting_type else None
+            if section_name and section_name in file_data:
+                section_data = file_data[section_name]
+            else:
+                section_data = file_data
+
+            # Import settings
+            for key, value in section_data.items():
+                if section_name:
+                    full_key = f"{section_name}.{key}"
+                else:
+                    # Try to determine section from key structure
+                    if "." in key:
+                        full_key = key
+                    else:
+                        # Assume it's an app setting
+                        full_key = f"app.{key}"
+
+                self.set_setting(full_key, value, commit=False)
+
+            # Commit if requested
+            if commit and self.db_session:
+                self.db_session.commit()
+
+            return True
+
+        except Exception as e:
+            logger.error(f"Error importing settings from file: {e}")
+            if self.db_session:
+                self.db_session.rollback()
+            return False
+
+    def _write_section_to_file(
+        self, file_path: Path, section: str, data: Dict[str, Any]
+    ) -> bool:
+        """
+        Write a section of settings to a TOML file
+
+        Args:
+            file_path: Path to the file
+            section: Section name
+            data: Section data
+
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            # Create file if it doesn't exist
+            if not os.path.exists(file_path):
+                file_path.parent.mkdir(parents=True, exist_ok=True)
+                with open(file_path, "w") as f:
+                    f.write(f"[{section}]\n")
+
+            # Read existing file
+            with open(file_path, "r") as f:
+                file_data = toml.load(f)
+
+            # Update section
+            file_data[section] = data
+
+            # Write back to file
+            with open(file_path, "w") as f:
+                toml.dump(file_data, f)
+
+            return True
+        except Exception as e:
+            logger.error(f"Error writing section {section} to {file_path}: {e}")
+            return False
+
+    @classmethod
+    def get_instance(cls, db_session: Optional[Session] = None) -> "SettingsManager":
+        """
+        Get a singleton instance of the settings manager
+
+        Args:
+            db_session: Optional database session
+
+        Returns:
+            SettingsManager instance
+        """
+        if not hasattr(cls, "_instance"):
+            cls._instance = cls(db_session)
+        elif db_session and not cls._instance.db_session:
+            # Update existing instance with a session
+            cls._instance.db_session = db_session
+
+        return cls._instance
+
+    def import_default_settings(
+        self, main_settings_file, search_engines_file, collections_file
+    ):
+        """
+        Import settings directly from default files
+
+        Args:
+            main_settings_file: Path to the main settings.toml file
+            search_engines_file: Path to the search_engines.toml file
+            collections_file: Path to the local_collections.toml file
+
+        Returns:
+            True if successful, False otherwise
+        """
+        if not self.db_session:
+            logger.warning(
+                "No database session available, cannot import default settings"
+            )
+            return False
+
+        try:
+            # Import settings from main settings file
+            if os.path.exists(main_settings_file):
+                with open(main_settings_file, "r") as f:
+                    main_data = toml.load(f)
+
+                # Process each section in the main settings file
+                for section, values in main_data.items():
+                    if section in ["web", "llm", "general", "app"]:
+                        setting_type = None
+                        if section == "web" or section == "app":
+                            setting_type = SettingType.APP
+                            prefix = "app"
+                        elif section == "llm":
+                            setting_type = SettingType.LLM
+                            prefix = "llm"
+                        else:  # general section
+                            # Map general settings to appropriate types
+                            prefix = None
+                            for key, value in values.items():
+                                if key in [
+                                    "enable_fact_checking",
+                                    "knowledge_accumulation",
+                                    "knowledge_accumulation_context_limit",
+                                    "output_dir",
+                                ]:
+                                    self._create_setting(
+                                        f"report.{key}", value, SettingType.REPORT
+                                    )
+
+                        # Add settings with correct prefix
+                        if prefix:
+                            for key, value in values.items():
+                                self._create_setting(
+                                    f"{prefix}.{key}", value, setting_type
+                                )
+
+                    elif section == "search":
+                        # Search settings go to search type
+                        for key, value in values.items():
+                            self._create_setting(
+                                f"search.{key}", value, SettingType.SEARCH
+                            )
+
+                    elif section == "report":
+                        # Report settings
+                        for key, value in values.items():
+                            self._create_setting(
+                                f"report.{key}", value, SettingType.REPORT
+                            )
+
+            # Import settings from search engines file
+            if os.path.exists(search_engines_file):
+                with open(search_engines_file, "r") as f:
+                    search_data = toml.load(f)
+
+                # Find search section in search engines file
+                if "search" in search_data:
+                    for key, value in search_data["search"].items():
+                        # Skip complex sections that are nested
+                        if not isinstance(value, dict):
+                            self._create_setting(
+                                f"search.{key}", value, SettingType.SEARCH
+                            )
+
+            # Commit changes
+            self.db_session.commit()
+            return True
+
+        except Exception as e:
+            logger.error(f"Error importing default settings: {e}")
+            if self.db_session:
+                self.db_session.rollback()
+            return False
+
+    def _create_setting(self, key, value, setting_type):
+        """Create a setting with appropriate metadata"""
+
+        # Determine appropriate category
+        category = None
+        ui_element = "text"
+
+        # Determine category based on key pattern
+        if key.startswith("app."):
+            category = "app_interface"
+        elif key.startswith("llm."):
+            if any(
+                param in key
+                for param in ["temperature", "max_tokens", "n_batch", "n_gpu_layers"]
+            ):
+                category = "llm_parameters"
+            else:
+                category = "llm_general"
+        elif key.startswith("search."):
+            if any(
+                param in key
+                for param in ["iterations", "questions", "results", "region"]
+            ):
+                category = "search_parameters"
+            else:
+                category = "search_general"
+        elif key.startswith("report."):
+            category = "report_parameters"
+
+        # Determine UI element type based on value
+        if isinstance(value, bool):
+            ui_element = "checkbox"
+        elif isinstance(value, (int, float)) and not isinstance(value, bool):
+            ui_element = "number"
+        elif isinstance(value, (dict, list)):
+            ui_element = "textarea"
+
+        # Build setting object
+        setting_dict = {
+            "key": key,
+            "value": value,
+            "type": setting_type.value.lower(),
+            "name": key.split(".")[-1].replace("_", " ").title(),
+            "description": f"Setting for {key}",
+            "category": category,
+            "ui_element": ui_element,
+        }
+
+        # Create the setting in the database
+        db_setting = self.create_or_update_setting(setting_dict, commit=False)
+
+        # Also update cache
+        if db_setting:
+            self._settings_cache[key] = value