local-deep-research 0.1.26__py3-none-any.whl → 0.2.0__py3-none-any.whl

local_deep_research/web/models/database.py

@@ -0,0 +1,294 @@
+import json
+import logging
+import os
+import sqlite3
+import traceback
+from datetime import datetime
+
+# Initialize logger
+logger = logging.getLogger(__name__)
+
+# Database path
+# Use unified database in data directory
+DATA_DIR = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "..", "..", "..", "data")
+)
+os.makedirs(DATA_DIR, exist_ok=True)
+DB_PATH = os.path.join(DATA_DIR, "ldr.db")
+
+# Legacy database paths (for migration)
+LEGACY_RESEARCH_HISTORY_DB = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "..", "..", "..", "research_history.db")
+)
+LEGACY_DEEP_RESEARCH_DB = os.path.join(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "data")
+    ),
+    "deep_research.db",
+)
+
+
+def get_db_connection():
+    """
+    Get a connection to the SQLite database.
+    Allows for custom row factory if needed.
+    """
+    conn = sqlite3.connect(DB_PATH)
+    return conn
+
+
+def init_db():
+    """Initialize the database with necessary tables."""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+
+    # Create the table if it doesn't exist
+    cursor.execute(
+        """
+    CREATE TABLE IF NOT EXISTS research_history (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        query TEXT NOT NULL,
+        mode TEXT NOT NULL,
+        status TEXT NOT NULL,
+        created_at TEXT NOT NULL,
+        completed_at TEXT,
+        duration_seconds INTEGER,
+        report_path TEXT,
+        metadata TEXT,
+        progress_log TEXT,
+        progress INTEGER,
+        title TEXT
+    )
+    """
+    )
+
+    # Create a dedicated table for research logs
+    cursor.execute(
+        """
+    CREATE TABLE IF NOT EXISTS research_logs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        research_id INTEGER NOT NULL,
+        timestamp TEXT NOT NULL,
+        message TEXT NOT NULL,
+        log_type TEXT NOT NULL,
+        progress INTEGER,
+        metadata TEXT,
+        FOREIGN KEY (research_id) REFERENCES research_history (id) ON DELETE CASCADE
+    )
+    """
+    )
+
+    # Create a dedicated table for research resources
+    cursor.execute(
+        """
+    CREATE TABLE IF NOT EXISTS research_resources (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        research_id INTEGER NOT NULL,
+        title TEXT,
+        url TEXT,
+        content_preview TEXT,
+        source_type TEXT,
+        metadata TEXT,
+        created_at TEXT NOT NULL,
+        FOREIGN KEY (research_id) REFERENCES research_history (id) ON DELETE CASCADE
+    )
+    """
+    )
+
+    # Check if the duration_seconds column exists, add it if missing
+    cursor.execute("PRAGMA table_info(research_history)")
+    columns = [column[1] for column in cursor.fetchall()]
+
+    if "duration_seconds" not in columns:
+        print("Adding missing 'duration_seconds' column to research_history table")
+        cursor.execute(
+            "ALTER TABLE research_history ADD COLUMN duration_seconds INTEGER"
+        )
+
+    # Check if the progress column exists, add it if missing
+    if "progress" not in columns:
+        print("Adding missing 'progress' column to research_history table")
+        cursor.execute("ALTER TABLE research_history ADD COLUMN progress INTEGER")
+
+    # Check if the title column exists, add it if missing
+    if "title" not in columns:
+        print("Adding missing 'title' column to research_history table")
+        cursor.execute("ALTER TABLE research_history ADD COLUMN title TEXT")
+
+    # Enable foreign key support
+    cursor.execute("PRAGMA foreign_keys = ON")
+
+    conn.commit()
+    conn.close()
+
+
+def calculate_duration(created_at_str, completed_at_str=None):
+    """
+    Calculate duration in seconds between created_at timestamp and completed_at or now.
+    Handles various timestamp formats and returns None if calculation fails.
+
+    Args:
+        created_at_str: The start timestamp
+        completed_at_str: Optional end timestamp, defaults to current time if None
+
+    Returns:
+        Duration in seconds or None if calculation fails
+    """
+    if not created_at_str:
+        return None
+
+    end_time = None
+    if completed_at_str:
+        # Use completed_at time if provided
+        try:
+            if "T" in completed_at_str:  # ISO format with T separator
+                end_time = datetime.fromisoformat(completed_at_str)
+            else:  # Older format without T
+                # Try different formats
+                try:
+                    end_time = datetime.strptime(
+                        completed_at_str, "%Y-%m-%d %H:%M:%S.%f"
+                    )
+                except ValueError:
+                    try:
+                        end_time = datetime.strptime(
+                            completed_at_str, "%Y-%m-%d %H:%M:%S"
+                        )
+                    except ValueError:
+                        # Last resort fallback
+                        end_time = datetime.fromisoformat(
+                            completed_at_str.replace(" ", "T")
+                        )
+        except Exception as e:
+            print(f"Error parsing completed_at timestamp: {str(e)}")
+            try:
+                from dateutil import parser
+
+                end_time = parser.parse(completed_at_str)
+            except Exception:
+                print(
+                    f"Fallback parsing also failed for completed_at: {completed_at_str}"
+                )
+                # Fall back to current time
+                end_time = datetime.utcnow()
+    else:
+        # Use current time if no completed_at provided
+        end_time = datetime.utcnow()
+
+    start_time = None
+    try:
+        # Proper parsing of ISO format
+        if "T" in created_at_str:  # ISO format with T separator
+            start_time = datetime.fromisoformat(created_at_str)
+        else:  # Older format without T
+            # Try different formats
+            try:
+                start_time = datetime.strptime(created_at_str, "%Y-%m-%d %H:%M:%S.%f")
+            except ValueError:
+                try:
+                    start_time = datetime.strptime(created_at_str, "%Y-%m-%d %H:%M:%S")
+                except ValueError:
+                    # Last resort fallback
+                    start_time = datetime.fromisoformat(
+                        created_at_str.replace(" ", "T")
+                    )
+    except Exception as e:
+        print(f"Error parsing created_at timestamp: {str(e)}")
+        # Fallback method if parsing fails
+        try:
+            from dateutil import parser
+
+            start_time = parser.parse(created_at_str)
+        except Exception:
+            print(f"Fallback parsing also failed for created_at: {created_at_str}")
+            return None
+
+    # Calculate duration if both timestamps are valid
+    if start_time and end_time:
+        try:
+            return int((end_time - start_time).total_seconds())
+        except Exception as e:
+            print(f"Error calculating duration: {str(e)}")
+
+    return None
+
+
+def add_log_to_db(research_id, message, log_type="info", progress=None, metadata=None):
+    """
+    Store a log entry in the database
+
+    Args:
+        research_id: ID of the research
+        message: Log message text
+        log_type: Type of log (info, error, milestone)
+        progress: Progress percentage (0-100)
+        metadata: Additional metadata as dictionary (will be stored as JSON)
+    """
+    try:
+        timestamp = datetime.utcnow().isoformat()
+        metadata_json = json.dumps(metadata) if metadata else None
+
+        conn = get_db_connection()
+        cursor = conn.cursor()
+        cursor.execute(
+            "INSERT INTO research_logs (research_id, timestamp, message, log_type, progress, metadata) "
+            "VALUES (?, ?, ?, ?, ?, ?)",
+            (research_id, timestamp, message, log_type, progress, metadata_json),
+        )
+        conn.commit()
+        conn.close()
+        return True
+    except Exception as e:
+        print(f"Error adding log to database: {str(e)}")
+        print(traceback.format_exc())
+        return False
+
+
+def get_logs_for_research(research_id):
+    """
+    Retrieve all logs for a specific research ID
+
+    Args:
+        research_id: ID of the research
+
+    Returns:
+        List of log entries as dictionaries
+    """
+    try:
+        conn = get_db_connection()
+        conn.row_factory = sqlite3.Row
+        cursor = conn.cursor()
+        cursor.execute(
+            "SELECT * FROM research_logs WHERE research_id = ? ORDER BY timestamp ASC",
+            (research_id,),
+        )
+        results = cursor.fetchall()
+        conn.close()
+
+        logs = []
+        for result in results:
+            log_entry = dict(result)
+            # Parse metadata JSON if it exists
+            if log_entry.get("metadata"):
+                try:
+                    log_entry["metadata"] = json.loads(log_entry["metadata"])
+                except Exception:
+                    log_entry["metadata"] = {}
+            else:
+                log_entry["metadata"] = {}
+
+            # Convert entry for frontend consumption
+            formatted_entry = {
+                "time": log_entry["timestamp"],
+                "message": log_entry["message"],
+                "progress": log_entry["progress"],
+                "metadata": log_entry["metadata"],
+                "type": log_entry["log_type"],
+            }
+            logs.append(formatted_entry)
+
+        return logs
+    except Exception as e:
+        print(f"Error retrieving logs from database: {str(e)}")
+        print(traceback.format_exc())
+        return []

local_deep_research/web/models/settings.py

@@ -0,0 +1,94 @@
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, field_validator
+
+
+class SettingType(str, Enum):
+    """Types of settings in the system"""
+
+    APP = "app"
+    LLM = "llm"
+    SEARCH = "search"
+    REPORT = "report"
+
+
+class BaseSetting(BaseModel):
+    """Base model for all settings"""
+
+    key: str
+    value: Any
+    type: SettingType
+    name: str
+    description: Optional[str] = None
+    category: Optional[str] = None
+    ui_element: Optional[str] = "text"  # text, select, checkbox, slider, etc.
+    options: Optional[List[Dict[str, Any]]] = None  # For select elements
+    min_value: Optional[float] = None  # For numeric inputs
+    max_value: Optional[float] = None  # For numeric inputs
+    step: Optional[float] = None  # For sliders
+    visible: bool = True
+    editable: bool = True
+
+    class Config:
+        from_attributes = True
+
+
+class LLMSetting(BaseSetting):
+    """LLM-specific settings"""
+
+    type: SettingType = SettingType.LLM
+
+    @field_validator("key")
+    def validate_llm_key(cls, v):
+        # Ensure LLM settings follow a convention
+        if not v.startswith("llm."):
+            return f"llm.{v}"
+        return v
+
+
+class SearchSetting(BaseSetting):
+    """Search-specific settings"""
+
+    type: SettingType = SettingType.SEARCH
+
+    @field_validator("key")
+    def validate_search_key(cls, v):
+        # Ensure search settings follow a convention
+        if not v.startswith("search."):
+            return f"search.{v}"
+        return v
+
+
+class ReportSetting(BaseSetting):
+    """Report generation settings"""
+
+    type: SettingType = SettingType.REPORT
+
+    @field_validator("key")
+    def validate_report_key(cls, v):
+        # Ensure report settings follow a convention
+        if not v.startswith("report."):
+            return f"report.{v}"
+        return v
+
+
+class AppSetting(BaseSetting):
+    """Application-wide settings"""
+
+    type: SettingType = SettingType.APP
+
+    @field_validator("key")
+    def validate_app_key(cls, v):
+        # Ensure app settings follow a convention
+        if not v.startswith("app."):
+            return f"app.{v}"
+        return v
+
+
+class SettingsGroup(BaseModel):
+    """A group of related settings"""
+
+    name: str
+    description: Optional[str] = None
+    settings: List[BaseSetting]