local-deep-research 0.1.26__py3-none-any.whl → 0.2.0__py3-none-any.whl

local_deep_research/web/app_factory.py

@@ -0,0 +1,289 @@
+import logging
+import os
+from importlib import resources as importlib_resources
+
+from flask import (
+    Flask,
+    jsonify,
+    make_response,
+    redirect,
+    request,
+    send_from_directory,
+    url_for,
+)
+from flask_socketio import SocketIO
+from flask_wtf.csrf import CSRFProtect
+
+from .models.database import DB_PATH, init_db
+
+# Initialize logger
+logger = logging.getLogger(__name__)
+
+
+def create_app():
+    """
+    Create and configure the Flask application.
+
+    Returns:
+        tuple: (app, socketio) - The configured Flask app and SocketIO instance
+    """
+    # Configure logging
+    logging.basicConfig(level=logging.INFO)
+
+    # Set Werkzeug logger to WARNING level to suppress Socket.IO polling logs
+    logging.getLogger("werkzeug").setLevel(logging.WARNING)
+
+    try:
+        # Get directories based on package installation
+        PACKAGE_DIR = importlib_resources.files("local_deep_research") / "web"
+        with importlib_resources.as_file(PACKAGE_DIR) as package_dir:
+            STATIC_DIR = (package_dir / "static").as_posix()
+            TEMPLATE_DIR = (package_dir / "templates").as_posix()
+
+        # Initialize Flask app with package directories
+        app = Flask(__name__, static_folder=STATIC_DIR, template_folder=TEMPLATE_DIR)
+        print(f"Using package static path: {STATIC_DIR}")
+        print(f"Using package template path: {TEMPLATE_DIR}")
+    except Exception as e:
+        # Fallback for development
+        print(f"Package directories not found, using fallback paths: {str(e)}")
+        app = Flask(
+            __name__,
+            static_folder=os.path.abspath("static"),
+            template_folder=os.path.abspath("templates"),
+        )
+
+    # App configuration
+    app.config["SECRET_KEY"] = "deep-research-secret-key"
+
+    # Initialize CSRF protection
+    csrf = CSRFProtect(app)
+    # Exempt Socket.IO from CSRF protection
+    csrf.exempt("research.socket_io")
+
+    # Database configuration - Use unified ldr.db from the database module
+    db_path = DB_PATH
+    app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{db_path}"
+    logger.info(f"Using database at {db_path}")
+    app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
+    app.config["SQLALCHEMY_ECHO"] = False
+
+    # Initialize extensions
+    socketio = SocketIO(
+        app,
+        cors_allowed_origins="*",
+        async_mode="threading",
+        path="/research/socket.io",
+        logger=False,
+        engineio_logger=False,
+        ping_timeout=20,
+        ping_interval=5,
+    )
+
+    # Initialize the database
+    create_database(app)
+    init_db()
+
+    # Register socket service
+    from .services.socket_service import set_socketio
+
+    set_socketio(socketio)
+
+    # Apply middleware
+    apply_middleware(app)
+
+    # Register blueprints
+    register_blueprints(app)
+
+    # Register error handlers
+    register_error_handlers(app)
+
+    # Register socket event handlers
+    register_socket_events(socketio)
+
+    return app, socketio
+
+
+def apply_middleware(app):
+    """Apply middleware to the Flask app."""
+
+    # Add Content Security Policy headers to allow Socket.IO to function
+    @app.after_request
+    def add_security_headers(response):
+        # Define a permissive CSP for development that allows Socket.IO to function
+        csp = (
+            "default-src 'self'; "
+            "connect-src 'self' ws: wss: http: https:; "
+            "script-src 'self' 'unsafe-inline' 'unsafe-eval' cdnjs.cloudflare.com cdn.jsdelivr.net unpkg.com; "
+            "style-src 'self' 'unsafe-inline' cdnjs.cloudflare.com; "
+            "font-src 'self' cdnjs.cloudflare.com; "
+            "img-src 'self' data:; "
+            "worker-src blob:; "
+            "frame-src 'self';"
+        )
+
+        response.headers["Content-Security-Policy"] = csp
+        response.headers["X-Content-Security-Policy"] = csp
+
+        # Add CORS headers for API requests
+        if request.path.startswith("/api/"):
+            response.headers["Access-Control-Allow-Origin"] = "*"
+            response.headers["Access-Control-Allow-Methods"] = (
+                "GET, POST, DELETE, OPTIONS"
+            )
+            response.headers["Access-Control-Allow-Headers"] = "Content-Type"
+
+        return response
+
+    # Add a middleware layer to handle abrupt disconnections
+    @app.before_request
+    def handle_websocket_requests():
+        if request.path.startswith("/research/socket.io"):
+            try:
+                if not request.environ.get("werkzeug.socket"):
+                    return
+            except Exception as e:
+                print(f"WebSocket preprocessing error: {e}")
+                # Return empty response to prevent further processing
+                return "", 200
+
+
+def register_blueprints(app):
+    """Register blueprints with the Flask app."""
+
+    # Import blueprints
+    from .routes.api_routes import api_bp  # Import the API blueprint
+    from .routes.history_routes import history_bp
+    from .routes.research_routes import research_bp
+    from .routes.settings_routes import settings_bp
+
+    # Register blueprints
+    app.register_blueprint(research_bp)
+    app.register_blueprint(history_bp, url_prefix="/research/api")
+    app.register_blueprint(settings_bp)
+    app.register_blueprint(
+        api_bp, url_prefix="/research/api"
+    )  # Register API blueprint with prefix
+
+    # Configure settings paths
+    # Import config inside the function to avoid circular dependencies
+    def configure_settings_routes():
+        try:
+            from ..config.config_files import SEARCH_ENGINES_FILE, get_config_dir
+            from .routes.settings_routes import set_config_paths
+
+            CONFIG_DIR = get_config_dir() / "config"
+            MAIN_CONFIG_FILE = CONFIG_DIR / "settings.toml"
+            LOCAL_COLLECTIONS_FILE = CONFIG_DIR / "local_collections.toml"
+
+            set_config_paths(
+                CONFIG_DIR,
+                SEARCH_ENGINES_FILE,
+                MAIN_CONFIG_FILE,
+                LOCAL_COLLECTIONS_FILE,
+            )
+        except Exception as e:
+            logger.error(f"Error configuring settings routes: {e}")
+
+    # Call this after all blueprints are registered
+    configure_settings_routes()
+
+    # Add root route redirect
+    @app.route("/")
+    def root_index():
+        return redirect(url_for("research.index"))
+
+    # Add favicon route
+    @app.route("/favicon.ico")
+    def favicon():
+        return send_from_directory(
+            app.static_folder, "favicon.ico", mimetype="image/x-icon"
+        )
+
+    # Add static route at the app level for compatibility
+    @app.route("/static/<path:path>")
+    def app_serve_static(path):
+        return send_from_directory(app.static_folder, path)
+
+
+def register_error_handlers(app):
+    """Register error handlers with the Flask app."""
+
+    @app.errorhandler(404)
+    def not_found(error):
+        return make_response(jsonify({"error": "Not found"}), 404)
+
+    @app.errorhandler(500)
+    def server_error(error):
+        return make_response(jsonify({"error": "Server error"}), 500)
+
+
+def register_socket_events(socketio):
+    """Register Socket.IO event handlers."""
+
+    from .routes.research_routes import get_globals
+    from .services.socket_service import (
+        handle_connect,
+        handle_default_error,
+        handle_disconnect,
+        handle_socket_error,
+        handle_subscribe,
+    )
+
+    @socketio.on("connect")
+    def on_connect():
+        handle_connect(request)
+
+    @socketio.on("disconnect")
+    def on_disconnect():
+        handle_disconnect(request)
+
+    @socketio.on("subscribe_to_research")
+    def on_subscribe(data):
+        globals_dict = get_globals()
+        active_research = globals_dict.get("active_research", {})
+        handle_subscribe(data, request, active_research)
+
+    @socketio.on_error
+    def on_error(e):
+        return handle_socket_error(e)
+
+    @socketio.on_error_default
+    def on_default_error(e):
+        return handle_default_error(e)
+
+
+def create_database(app):
+    """
+    Create the database and tables for the application.
+    """
+    from sqlalchemy import create_engine
+    from sqlalchemy.orm import scoped_session, sessionmaker
+
+    from .database.migrations import run_migrations, setup_predefined_settings
+    from .database.models import Base
+
+    # Configure SQLite to use URI mode, which allows for relative file paths
+    engine = create_engine(
+        app.config["SQLALCHEMY_DATABASE_URI"],
+        echo=app.config.get("SQLALCHEMY_ECHO", False),
+        connect_args={"check_same_thread": False},
+    )
+
+    app.engine = engine
+
+    # Create all tables
+    Base.metadata.create_all(engine)
+
+    # Configure session factory
+    session_factory = sessionmaker(bind=engine, autocommit=False, autoflush=False)
+    app.db_session = scoped_session(session_factory)
+
+    # Run migrations and setup predefined settings
+    run_migrations(engine, app.db_session)
+    setup_predefined_settings(app.db_session)
+
+    # Add teardown context
+    @app.teardown_appcontext
+    def remove_session(exception=None):
+        app.db_session.remove()

local_deep_research/web/database/README.md

@@ -0,0 +1,70 @@
+# Database Architecture
+
+## Overview
+
+Local Deep Research now uses a unified database architecture with a single SQLite database file (`ldr.db`) that replaces the previous split database approach (`deep_research.db` and `research_history.db`).
+
+The database is located at `src/data/ldr.db` within the project directory structure.
+
+## Database-First Settings
+
+The application now follows a "database-first" approach for settings:
+
+1. All settings are stored in the database, in the `settings` table
+2. Settings from TOML files are used only as fallbacks if a setting doesn't exist in the database
+3. The web UI settings page modifies the database values directly
+
+## Migration
+
+If you have existing data in the legacy databases, you need to migrate it to the new unified database.
+
+### Automatic Migration
+
+When you start the application for the first time after updating, it will check if migration is needed:
+
+1. If legacy databases exist and `ldr.db` doesn't exist, you'll see a warning message
+2. You can run migration using the command: `python -m src.local_deep_research.main --migrate-db`
+3. Alternatively, start the application with auto-migration: `python -m src.local_deep_research.main --auto-migrate`
+
+### Manual Migration
+
+If automatic migration doesn't work, you can:
+
+1. Run the migration script directly: `python -m src.local_deep_research.web.database.migrate_to_ldr_db`
+2. Check migration results in the log output
+
+### Schema Upgrades
+
+If you have already migrated your database but need to update its schema:
+
+1. The application automatically runs schema upgrades on startup
+2. You can manually run schema upgrades with: `python -m src.local_deep_research.main --schema-upgrade`
+3. Current schema upgrades include:
+   - Removing the redundant `research_log` table (consolidated into `research_logs`)
+
+## Database Schema
+
+The unified database contains:
+
+* `research_history` - Research history entries (from research_history.db)
+* `research_logs` - Consolidated logs for all research activities (merged from research_history.db)
+* `research_resources` - Resources found during research (from research_history.db)
+* `settings` - Application settings (from deep_research.db)
+* `research` - Research data (from deep_research.db)
+* `research_report` - Generated research reports (from deep_research.db)
+
+## Rollback
+
+If you need to roll back to the previous database architecture:
+
+1. Keep backup copies of your original `deep_research.db` and `research_history.db` files
+2. In case of issues, restore them and modify the database paths in the code
+
+## Troubleshooting
+
+If you encounter issues with database migration:
+
+1. Check the application logs for detailed error messages
+2. Ensure you have write permissions to the data directory
+3. Make sure SQLite is functioning properly
+4. If necessary, start with a fresh database by removing `ldr.db`

local_deep_research/web/database/migrate_to_ldr_db.py

@@ -0,0 +1,289 @@
+"""
+Migration script to merge deep_research.db and research_history.db into ldr.db
+"""
+
+# Standard library imports
+# import json  # Remove unused imports
+import logging
+import os
+import sqlite3
+import sys
+import traceback
+
+# from pathlib import Path  # Remove unused imports
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Add the parent directory to sys.path to allow relative imports
+sys.path.append(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+)
+
+# Import the database module after adding to sys path
+# pylint: disable=wrong-import-position
+from src.local_deep_research.web.models.database import (  # noqa: E402
+    DB_PATH,
+    LEGACY_DEEP_RESEARCH_DB,
+    LEGACY_RESEARCH_HISTORY_DB,
+)
+
+
+def migrate_to_ldr_db():
+    """
+    Migrates data from deep_research.db and research_history.db to ldr.db
+    """
+    # Ensure data directory exists
+    try:
+        from src.local_deep_research.setup_data_dir import setup_data_dir
+
+        setup_data_dir()
+    except ImportError:
+        # If we can't import directly, check the path manually
+        logger.info("Creating data directory manually")
+        data_dir = os.path.dirname(DB_PATH)
+        os.makedirs(data_dir, exist_ok=True)
+
+    logger.info(f"Using database path: {DB_PATH}")
+
+    # Check if ldr.db already exists
+    if os.path.exists(DB_PATH):
+        logger.info(f"Target database {DB_PATH} already exists")
+
+        # Ask for confirmation
+        if (
+            input(
+                f"Target database {DB_PATH} already exists. Do you want to continue migration? (y/n): "
+            ).lower()
+            != "y"
+        ):
+            logger.info("Migration aborted by user")
+            return False
+
+    # Connect to the target database
+    try:
+        ldr_conn = sqlite3.connect(DB_PATH)
+        ldr_cursor = ldr_conn.cursor()
+        logger.info(f"Connected to target database: {DB_PATH}")
+    except Exception as e:
+        logger.error(f"Failed to connect to target database: {e}")
+        return False
+
+    # Enable foreign keys
+    ldr_cursor.execute("PRAGMA foreign_keys = OFF")
+
+    # Initialize the database schema
+    try:
+        from src.local_deep_research.web.models.database import init_db
+
+        init_db()
+        logger.info("Initialized database schema")
+    except Exception as e:
+        logger.error(f"Failed to initialize database schema: {e}")
+        ldr_conn.close()
+        return False
+
+    # Migrate from research_history.db
+    migrated_research = migrate_research_history_db(
+        ldr_conn, LEGACY_RESEARCH_HISTORY_DB
+    )
+
+    # Migrate from deep_research.db
+    migrated_deep_research = migrate_deep_research_db(ldr_conn, LEGACY_DEEP_RESEARCH_DB)
+
+    # Re-enable foreign keys and commit
+    ldr_cursor.execute("PRAGMA foreign_keys = ON")
+    ldr_conn.commit()
+    ldr_conn.close()
+
+    logger.info(
+        f"Migration completed - Research History: {migrated_research}, Deep Research: {migrated_deep_research}"
+    )
+    return True
+
+
+def migrate_research_history_db(ldr_conn, legacy_path):
+    """
+    Migrates data from research_history.db to ldr.db
+
+    Args:
+        ldr_conn: Connection to the target ldr.db
+        legacy_path: Path to legacy research_history.db
+
+    Returns:
+        bool: True if migration was successful, False otherwise
+    """
+    if not os.path.exists(legacy_path):
+        logger.warning(f"Legacy database not found: {legacy_path}")
+        return False
+
+    try:
+        # Connect to legacy database
+        legacy_conn = sqlite3.connect(legacy_path)
+        legacy_cursor = legacy_conn.cursor()
+        ldr_cursor = ldr_conn.cursor()
+
+        logger.info(f"Connected to legacy database: {legacy_path}")
+
+        # Get tables from legacy database
+        legacy_cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        tables = [row[0] for row in legacy_cursor.fetchall()]
+
+        for table in tables:
+            # Skip sqlite internal tables
+            if table.startswith("sqlite_"):
+                continue
+
+            logger.info(f"Migrating table: {table}")
+
+            # Check if table exists in target database
+            ldr_cursor.execute(
+                f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table}'"
+            )
+            if not ldr_cursor.fetchone():
+                # Create the table in the target database
+                legacy_cursor.execute(
+                    f"SELECT sql FROM sqlite_master WHERE type='table' AND name='{table}'"
+                )
+                create_sql = legacy_cursor.fetchone()[0]
+                logger.info(f"Creating table {table} with SQL: {create_sql}")
+                ldr_cursor.execute(create_sql)
+                logger.info(f"Created table {table} in target database")
+
+            # Get column names
+            legacy_cursor.execute(f"PRAGMA table_info({table})")
+            columns = [row[1] for row in legacy_cursor.fetchall()]
+
+            # Get all data from legacy table
+            legacy_cursor.execute(f"SELECT * FROM {table}")
+            rows = legacy_cursor.fetchall()
+
+            logger.info(f"Found {len(rows)} rows in {table}")
+
+            if rows:
+                # Create placeholders for the SQL query
+                placeholders = ", ".join(["?" for _ in columns])
+                columns_str = ", ".join(columns)
+
+                # Insert data into target database
+                for row in rows:
+                    try:
+                        ldr_cursor.execute(
+                            f"INSERT OR IGNORE INTO {table} ({columns_str}) VALUES ({placeholders})",
+                            row,
+                        )
+                    except sqlite3.Error as e:
+                        logger.error(f"Error inserting into {table}: {e}")
+                        logger.error(f"Row data: {row}")
+                        continue
+
+                # Verify data was inserted
+                ldr_cursor.execute(f"SELECT COUNT(*) FROM {table}")
+                count = ldr_cursor.fetchone()[0]
+                logger.info(f"Migrated {count} rows to {table} (expected {len(rows)})")
+            else:
+                logger.info(f"No data to migrate from {table}")
+
+        legacy_conn.close()
+        return True
+
+    except Exception as e:
+        logger.error(f"Failed to migrate from {legacy_path}: {e}")
+        logger.error(f"Exception details: {traceback.format_exc()}")
+        return False
+
+
+def migrate_deep_research_db(ldr_conn, legacy_path):
+    """
+    Migrates data from deep_research.db to ldr.db
+
+    Args:
+        ldr_conn: Connection to the target ldr.db
+        legacy_path: Path to legacy deep_research.db
+
+    Returns:
+        bool: True if migration was successful, False otherwise
+    """
+    if not os.path.exists(legacy_path):
+        logger.warning(f"Legacy database not found: {legacy_path}")
+        return False
+
+    try:
+        # Connect to legacy database
+        legacy_conn = sqlite3.connect(legacy_path)
+        legacy_cursor = legacy_conn.cursor()
+        ldr_cursor = ldr_conn.cursor()
+
+        logger.info(f"Connected to legacy database: {legacy_path}")
+
+        # Get tables from legacy database
+        legacy_cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        tables = [row[0] for row in legacy_cursor.fetchall()]
+
+        # Migrate each table
+        for table in tables:
+            # Skip sqlite internal tables
+            if table.startswith("sqlite_"):
+                continue
+
+            # Skip the research_log table as it's redundant with research_logs
+            if table == "research_log":
+                logger.info(
+                    "Skipping redundant table 'research_log', using 'research_logs' instead"
+                )
+                continue
+
+            logger.info(f"Migrating table: {table}")
+
+            # Check if table exists in target database
+            ldr_cursor.execute(
+                f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table}'"
+            )
+            if not ldr_cursor.fetchone():
+                # Create the table in the target database
+                legacy_cursor.execute(
+                    f"SELECT sql FROM sqlite_master WHERE type='table' AND name='{table}'"
+                )
+                create_sql = legacy_cursor.fetchone()[0]
+                ldr_cursor.execute(create_sql)
+                logger.info(f"Created table {table} in target database")
+
+            # Get column names
+            legacy_cursor.execute(f"PRAGMA table_info({table})")
+            columns = [row[1] for row in legacy_cursor.fetchall()]
+
+            # Get all data from legacy table
+            legacy_cursor.execute(f"SELECT * FROM {table}")
+            rows = legacy_cursor.fetchall()
+
+            if rows:
+                # Create placeholders for the SQL query
+                placeholders = ", ".join(["?" for _ in columns])
+                columns_str = ", ".join(columns)
+
+                # Insert data into target database
+                for row in rows:
+                    try:
+                        ldr_cursor.execute(
+                            f"INSERT OR IGNORE INTO {table} ({columns_str}) VALUES ({placeholders})",
+                            row,
+                        )
+                    except sqlite3.Error as e:
+                        logger.error(f"Error inserting into {table}: {e}")
+                        continue
+
+                logger.info(f"Migrated {len(rows)} rows from {table}")
+            else:
+                logger.info(f"No data to migrate from {table}")
+
+        legacy_conn.close()
+        return True
+
+    except Exception as e:
+        logger.error(f"Failed to migrate from {legacy_path}: {e}")
+        return False
+
+
+if __name__ == "__main__":
+    migrate_to_ldr_db()