local-deep-research 0.1.26__py3-none-any.whl → 0.2.0__py3-none-any.whl

local_deep_research/web/database/migrations.py

@@ -0,0 +1,447 @@
+import logging
+
+from sqlalchemy import inspect
+
+from ..services.settings_manager import SettingsManager
+from .models import Base, Setting, SettingType
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_settings_from_files(db_session):
+    """
+    Migrate settings from files to database
+    """
+    # Check if settings table is empty
+    settings_count = db_session.query(Setting).count()
+
+    if settings_count == 0:
+        logger.info("Settings table is empty, importing from files")
+
+        # Create settings manager and import settings
+        try:
+            settings_mgr = SettingsManager(db_session)
+            success = settings_mgr.import_from_file()
+            if success:
+                logger.info("Successfully imported settings from files")
+            else:
+                logger.warning("Failed to import some settings from files")
+        except Exception as e:
+            logger.error("Error importing settings from files: %s", e)
+    else:
+        logger.info(
+            "Settings table already has %s rows, skipping import", settings_count
+        )
+
+
+def run_migrations(engine, db_session=None):
+    """
+    Run any necessary database migrations
+
+    Args:
+        engine: SQLAlchemy engine
+        db_session: Optional SQLAlchemy session
+    """
+    # Create all tables if they don't exist
+    inspector = inspect(engine)
+    if not inspector.has_table("settings"):
+        logger.info("Creating settings table")
+        Base.metadata.create_all(engine, tables=[Setting.__table__])
+
+    # Import existing settings from files
+    if db_session:
+        migrate_settings_from_files(db_session)
+
+
+def setup_predefined_settings(db_session):
+    """
+    Set up predefined settings with UI metadata
+
+    Args:
+        db_session: SQLAlchemy session
+    """
+    # Define standard UI settings for LLM
+    llm_settings = [
+        {
+            "key": "llm.model",
+            "name": "LLM Model",
+            "description": "Language model to use for research and analysis",
+            "category": "llm_general",
+            "ui_element": "select",
+            "options": [
+                {"value": "gpt-4o", "label": "GPT-4o (OpenAI)"},
+                {"value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo (OpenAI)"},
+                {
+                    "value": "claude-3-5-sonnet-latest",
+                    "label": "Claude 3.5 Sonnet (Anthropic)",
+                },
+                {
+                    "value": "claude-3-opus-20240229",
+                    "label": "Claude 3 Opus (Anthropic)",
+                },
+                {"value": "llama3", "label": "Llama 3 (Meta)"},
+                {"value": "mistral", "label": "Mistral (Mistral AI)"},
+                {"value": "mixtral", "label": "Mixtral (Mistral AI)"},
+            ],
+            "value": "gpt-3.5-turbo",
+        },
+        {
+            "key": "llm.provider",
+            "name": "LLM Provider",
+            "description": "Service provider for the language model",
+            "category": "llm_general",
+            "ui_element": "select",
+            "options": [
+                {"value": "openai", "label": "OpenAI API"},
+                {"value": "anthropic", "label": "Anthropic API"},
+                {"value": "ollama", "label": "Ollama (Local)"},
+                {"value": "lmstudio", "label": "LM Studio (Local)"},
+                {"value": "vllm", "label": "vLLM (Local)"},
+                {"value": "openai_endpoint", "label": "Custom OpenAI-compatible API"},
+            ],
+            "value": "openai",
+        },
+        {
+            "key": "llm.temperature",
+            "name": "Temperature",
+            "description": "Controls randomness in model outputs (0.0 - 1.0)",
+            "category": "llm_parameters",
+            "ui_element": "slider",
+            "min_value": 0.0,
+            "max_value": 1.0,
+            "step": 0.05,
+            "value": 0.7,
+        },
+        {
+            "key": "llm.max_tokens",
+            "name": "Max Tokens",
+            "description": "Maximum number of tokens in model responses",
+            "category": "llm_parameters",
+            "ui_element": "number",
+            "min_value": 100,
+            "max_value": 4096,
+            "value": 1024,
+        },
+    ]
+
+    # Define standard UI settings for Search
+    search_settings = [
+        {
+            "key": "search.tool",
+            "name": "Search Engine",
+            "description": "Web search engine to use for research",
+            "category": "search_general",
+            "ui_element": "select",
+            "options": [
+                {"value": "auto", "label": "Auto (Default)"},
+                {"value": "arxiv", "label": "Arxiv"},
+                {"value": "wikipedia", "label": "Wikipedia"},
+                {"value": "pubmed", "label": "Pubmed"},
+                {"value": "github", "label": "Github"},
+                {"value": "serpapi", "label": "SerpAPI (Google)"},
+                {"value": "searxng", "label": "SearXNG (Self-hosted)"},
+                {"value": "google_pse", "label": "Google Programmable Search Engine"},
+                {"value": "duckduckgo", "label": "DuckDuckGo"},
+                {"value": "brave", "label": "Brave"},
+                {"value": "wayback", "label": "Wayback"},
+                {"value": "local_all", "label": "Local All"},
+            ],
+            "value": "auto",
+        },
+        {
+            "key": "search.max_results",
+            "name": "Max Results",
+            "description": "Maximum number of search results to retrieve",
+            "category": "search_parameters",
+            "ui_element": "number",
+            "min_value": 3,
+            "max_value": 50,
+            "value": 10,
+        },
+        {
+            "key": "search.region",
+            "name": "Search Region",
+            "description": "Geographic region for search results",
+            "category": "search_parameters",
+            "ui_element": "select",
+            "options": [
+                {"value": "us", "label": "United States"},
+                {"value": "uk", "label": "United Kingdom"},
+                {"value": "fr", "label": "France"},
+                {"value": "de", "label": "Germany"},
+                {"value": "jp", "label": "Japan"},
+                {"value": "wt-wt", "label": "No Region (Worldwide)"},
+            ],
+            "value": "us",
+        },
+        {
+            "key": "search.time_period",
+            "name": "Time Period",
+            "description": "Time period for search results",
+            "category": "search_parameters",
+            "ui_element": "select",
+            "options": [
+                {"value": "d", "label": "Past 24 hours"},
+                {"value": "w", "label": "Past week"},
+                {"value": "m", "label": "Past month"},
+                {"value": "y", "label": "Past year"},
+                {"value": "all", "label": "All time"},
+            ],
+            "value": "all",
+        },
+        {
+            "key": "search.snippets_only",
+            "name": "Snippets Only",
+            "description": "Only retrieve snippets instead of full search results",
+            "category": "search_parameters",
+            "ui_element": "checkbox",
+            "value": True,
+        },
+    ]
+
+    # Define standard UI settings for Report generation
+    report_settings = [
+        {
+            "key": "report.searches_per_section",
+            "name": "Searches Per Section",
+            "description": "Number of searches to run per report section",
+            "category": "report_parameters",
+            "ui_element": "number",
+            "min_value": 1,
+            "max_value": 5,
+            "value": 2,
+        },
+        {
+            "key": "report.enable_fact_checking",
+            "name": "Enable Fact Checking",
+            "description": "Enable fact checking for report contents",
+            "category": "report_parameters",
+            "ui_element": "checkbox",
+            "value": True,
+        },
+        {
+            "key": "report.detailed_citations",
+            "name": "Detailed Citations",
+            "description": "Include detailed citations in reports",
+            "category": "report_parameters",
+            "ui_element": "checkbox",
+            "value": True,
+        },
+    ]
+
+    # Define standard UI settings for App
+    app_settings = [
+        {
+            "key": "app.debug",
+            "name": "Debug Mode",
+            "description": "Enable debug mode for the web application",
+            "category": "app_interface",
+            "ui_element": "checkbox",
+            "value": True,
+        },
+        {
+            "key": "app.host",
+            "name": "Web Host",
+            "description": "Host address to bind the web server",
+            "category": "app_interface",
+            "ui_element": "text",
+            "value": "0.0.0.0",
+        },
+        {
+            "key": "app.port",
+            "name": "Web Port",
+            "description": "Port for the web server",
+            "category": "app_interface",
+            "ui_element": "number",
+            "min_value": 1,
+            "max_value": 65535,
+            "value": 5000,
+        },
+        {
+            "key": "app.enable_notifications",
+            "name": "Enable Notifications",
+            "description": "Enable browser notifications for research events",
+            "category": "app_interface",
+            "ui_element": "checkbox",
+            "value": True,
+        },
+        {
+            "key": "app.theme",
+            "name": "UI Theme",
+            "description": "User interface theme",
+            "category": "app_interface",
+            "ui_element": "select",
+            "options": [
+                {"value": "dark", "label": "Dark"},
+                {"value": "light", "label": "Light"},
+                {"value": "system", "label": "System Default"},
+            ],
+            "value": "dark",
+        },
+        {
+            "key": "app.web_interface",
+            "name": "Web Interface",
+            "description": "Enable the web interface",
+            "category": "app_interface",
+            "ui_element": "checkbox",
+            "value": True,
+        },
+        {
+            "key": "app.enable_web",
+            "name": "Enable Web Server",
+            "description": "Enable the web server",
+            "category": "app_interface",
+            "ui_element": "checkbox",
+            "value": True,
+        },
+    ]
+
+    # Update search settings with research-specific settings
+    search_settings.extend(
+        [
+            {
+                "key": "search.research_iterations",
+                "name": "Research Iterations",
+                "description": "Number of research cycles to perform",
+                "category": "search_parameters",
+                "ui_element": "number",
+                "min_value": 1,
+                "max_value": 5,
+                "value": 2,
+            },
+            {
+                "key": "search.questions_per_iteration",
+                "name": "Questions Per Iteration",
+                "description": "Number of questions to generate per research cycle",
+                "category": "search_parameters",
+                "ui_element": "number",
+                "min_value": 1,
+                "max_value": 10,
+                "value": 3,
+            },
+            {
+                "key": "search.searches_per_section",
+                "name": "Searches Per Section",
+                "description": "Number of searches to run per report section",
+                "category": "search_parameters",
+                "ui_element": "number",
+                "min_value": 1,
+                "max_value": 5,
+                "value": 2,
+            },
+            {
+                "key": "search.skip_relevance_filter",
+                "name": "Skip Relevance Filter",
+                "description": "Skip filtering search results for relevance",
+                "category": "search_parameters",
+                "ui_element": "checkbox",
+                "value": False,
+            },
+            {
+                "key": "search.safe_search",
+                "name": "Safe Search",
+                "description": "Enable safe search filtering",
+                "category": "search_parameters",
+                "ui_element": "checkbox",
+                "value": True,
+            },
+            {
+                "key": "search.search_language",
+                "name": "Search Language",
+                "description": "Language for search results",
+                "category": "search_parameters",
+                "ui_element": "select",
+                "options": [
+                    {"value": "English", "label": "English"},
+                    {"value": "French", "label": "French"},
+                    {"value": "German", "label": "German"},
+                    {"value": "Spanish", "label": "Spanish"},
+                    {"value": "Italian", "label": "Italian"},
+                    {"value": "Japanese", "label": "Japanese"},
+                    {"value": "Chinese", "label": "Chinese"},
+                ],
+                "value": "English",
+            },
+        ]
+    )
+
+    # Ensure these predefined settings exist in the database
+    # This will update existing settings with the same key
+    all_settings = llm_settings + search_settings + report_settings + app_settings
+
+    # Add/update each setting
+    for setting_dict in all_settings:
+        try:
+            # Convert to correct type based on key prefix
+            setting_type = None
+            key = setting_dict.get("key", "")
+
+            if key.startswith("llm."):
+                setting_type = SettingType.LLM
+            elif key.startswith("search."):
+                setting_type = SettingType.SEARCH
+            elif key.startswith("report."):
+                setting_type = SettingType.REPORT
+            elif key.startswith("app."):
+                setting_type = SettingType.APP
+
+            # Skip if no valid type
+            if not setting_type:
+                logger.warning("Skipping setting %s - unknown type", key)
+                continue
+
+            # Check if setting exists
+            existing = db_session.query(Setting).filter(Setting.key == key).first()
+
+            if existing:
+                # Update existing setting
+                logger.debug("Updating existing setting: %s", key)
+
+                # Only update metadata, not the value (to preserve user settings)
+                existing.name = setting_dict.get("name", existing.name)
+                existing.description = setting_dict.get(
+                    "description", existing.description
+                )
+                existing.category = setting_dict.get("category", existing.category)
+                existing.ui_element = setting_dict.get(
+                    "ui_element", existing.ui_element
+                )
+                existing.options = setting_dict.get("options", existing.options)
+                existing.min_value = setting_dict.get("min_value", existing.min_value)
+                existing.max_value = setting_dict.get("max_value", existing.max_value)
+                existing.step = setting_dict.get("step", existing.step)
+
+                # Only set value if it's not already set
+                if existing.value is None and "value" in setting_dict:
+                    existing.value = setting_dict["value"]
+            else:
+                # Create new setting
+                logger.info("Creating new setting: %s", key)
+                setting = Setting(
+                    key=key,
+                    value=setting_dict.get("value"),
+                    type=setting_type,
+                    name=setting_dict.get(
+                        "name", key.split(".")[-1].replace("_", " ").title()
+                    ),
+                    description=setting_dict.get("description", f"Setting for {key}"),
+                    category=setting_dict.get("category"),
+                    ui_element=setting_dict.get("ui_element", "text"),
+                    options=setting_dict.get("options"),
+                    min_value=setting_dict.get("min_value"),
+                    max_value=setting_dict.get("max_value"),
+                    step=setting_dict.get("step"),
+                    visible=setting_dict.get("visible", True),
+                    editable=setting_dict.get("editable", True),
+                )
+                db_session.add(setting)
+
+            # Commit after each successful setting
+            db_session.commit()
+
+        except Exception as e:
+            logger.error("Error ensuring setting %s: %s", setting_dict.get("key"), e)
+            db_session.rollback()
+
+    # Log completion
+    logger.info("Predefined settings setup complete")

local_deep_research/web/database/models.py

@@ -0,0 +1,117 @@
+import enum
+
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    Column,
+    DateTime,
+    Enum,
+    Float,
+    ForeignKey,
+    Integer,
+    String,
+    Text,
+    UniqueConstraint,
+)
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+from sqlalchemy.sql import func
+
+Base = declarative_base()
+
+
+class ResearchMode(enum.Enum):
+    QUICK = "quick"
+    DETAILED = "detailed"
+
+
+class ResearchStatus(enum.Enum):
+    PENDING = "pending"
+    IN_PROGRESS = "in_progress"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
+class Research(Base):
+    __tablename__ = "research"
+
+    id = Column(Integer, primary_key=True, index=True)
+    query = Column(String, nullable=False)
+    status = Column(
+        Enum(ResearchStatus), default=ResearchStatus.PENDING, nullable=False
+    )
+    mode = Column(Enum(ResearchMode), default=ResearchMode.QUICK, nullable=False)
+    created_at = Column(DateTime, server_default=func.now(), nullable=False)
+    updated_at = Column(
+        DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
+    )
+    progress = Column(Float, default=0.0, nullable=False)
+    start_time = Column(DateTime, nullable=True)
+    end_time = Column(DateTime, nullable=True)
+    error_message = Column(Text, nullable=True)
+
+    # Relationships
+    report = relationship(
+        "ResearchReport",
+        back_populates="research",
+        uselist=False,
+        cascade="all, delete-orphan",
+    )
+
+
+class ResearchReport(Base):
+    __tablename__ = "research_report"
+
+    id = Column(Integer, primary_key=True, index=True)
+    research_id = Column(
+        Integer,
+        ForeignKey("research.id", ondelete="CASCADE"),
+        nullable=False,
+        unique=True,
+    )
+    content = Column(Text, nullable=True)
+    created_at = Column(DateTime, server_default=func.now(), nullable=False)
+    updated_at = Column(
+        DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
+    )
+    report_metadata = Column(
+        JSON, nullable=True
+    )  # Additional metadata about the report
+
+    # Relationships
+    research = relationship("Research", back_populates="report")
+
+
+class SettingType(enum.Enum):
+    APP = "app"
+    LLM = "llm"
+    SEARCH = "search"
+    REPORT = "report"
+
+
+class Setting(Base):
+    """Database model for storing settings"""
+
+    __tablename__ = "settings"
+
+    id = Column(Integer, primary_key=True, index=True)
+    key = Column(String(255), nullable=False, unique=True, index=True)
+    value = Column(JSON, nullable=True)
+    type = Column(Enum(SettingType), nullable=False, index=True)
+    name = Column(String(255), nullable=False)
+    description = Column(Text, nullable=True)
+    category = Column(String(100), nullable=True, index=True)
+    ui_element = Column(String(50), default="text", nullable=False)
+    options = Column(JSON, nullable=True)  # For select elements
+    min_value = Column(Float, nullable=True)  # For numeric inputs
+    max_value = Column(Float, nullable=True)  # For numeric inputs
+    step = Column(Float, nullable=True)  # For sliders
+    visible = Column(Boolean, default=True, nullable=False)
+    editable = Column(Boolean, default=True, nullable=False)
+    created_at = Column(DateTime, server_default=func.now(), nullable=False)
+    updated_at = Column(
+        DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
+    )
+
+    __table_args__ = (UniqueConstraint("key", name="uix_settings_key"),)

local_deep_research/web/database/schema_upgrade.py

@@ -0,0 +1,107 @@
+"""
+Schema upgrade script for Local Deep Research database.
+Handles schema upgrades for existing ldr.db databases.
+"""
+
+import logging
+import os
+import sqlite3
+import sys
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Add the parent directory to sys.path to allow relative imports
+sys.path.append(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+)
+
+try:
+    from src.local_deep_research.web.models.database import DB_PATH
+except ImportError:
+    # Fallback path if import fails
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    project_root = os.path.abspath(os.path.join(current_dir, "..", "..", "..", ".."))
+    DB_PATH = os.path.join(project_root, "src", "data", "ldr.db")
+
+
+def check_table_exists(conn, table_name):
+    """
+    Check if a table exists in the database
+
+    Args:
+        conn: SQLite connection
+        table_name: Name of the table
+
+    Returns:
+        bool: True if table exists, False otherwise
+    """
+    cursor = conn.cursor()
+    cursor.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table_name,)
+    )
+    return cursor.fetchone() is not None
+
+
+def remove_research_log_table(conn):
+    """
+    Remove the redundant research_log table if it exists
+
+    Args:
+        conn: SQLite connection
+
+    Returns:
+        bool: True if operation was successful, False otherwise
+    """
+    try:
+        cursor = conn.cursor()
+
+        # Check if table exists
+        if check_table_exists(conn, "research_log"):
+            # For SQLite, DROP TABLE is the way to remove a table
+            cursor.execute("DROP TABLE research_log")
+            conn.commit()
+            logger.info("Successfully removed redundant 'research_log' table")
+            return True
+        else:
+            logger.info("Table 'research_log' does not exist, no action needed")
+            return True
+    except Exception as e:
+        logger.error(f"Error removing research_log table: {e}")
+        return False
+
+
+def run_schema_upgrades():
+    """
+    Run all schema upgrade operations on the database
+
+    Returns:
+        bool: True if all upgrades successful, False otherwise
+    """
+    # Check if database exists
+    if not os.path.exists(DB_PATH):
+        logger.warning(f"Database not found at {DB_PATH}, skipping schema upgrades")
+        return False
+
+    logger.info(f"Running schema upgrades on {DB_PATH}")
+
+    try:
+        # Connect to the database
+        conn = sqlite3.connect(DB_PATH)
+
+        # 1. Remove the redundant research_log table
+        remove_research_log_table(conn)
+
+        # Close connection
+        conn.close()
+
+        logger.info("Schema upgrades completed successfully")
+        return True
+    except Exception as e:
+        logger.error(f"Error during schema upgrades: {e}")
+        return False
+
+
+if __name__ == "__main__":
+    run_schema_upgrades()