local-deep-research 0.1.26__py3-none-any.whl → 0.2.2__py3-none-any.whl

local_deep_research/web/routes/research_routes.py

@@ -0,0 +1,715 @@
+import json
+import logging
+import os
+import platform
+import subprocess
+from datetime import datetime
+
+from flask import (
+    Blueprint,
+    current_app,
+    jsonify,
+    redirect,
+    render_template,
+    request,
+    send_from_directory,
+    url_for,
+)
+
+from ..models.database import add_log_to_db, calculate_duration, get_db_connection
+from ..services.research_service import run_research_process, start_research_process
+
+# Initialize logger
+logger = logging.getLogger(__name__)
+
+# Create a Blueprint for the research application
+research_bp = Blueprint("research", __name__, url_prefix="/research")
+
+# Active research processes and socket subscriptions
+active_research = {}
+socket_subscriptions = {}
+
+# Add termination flags dictionary
+termination_flags = {}
+
+# Output directory for research results
+OUTPUT_DIR = "research_outputs"
+
+
+# Return reference to globals for other modules to access
+def get_globals():
+    return {
+        "active_research": active_research,
+        "socket_subscriptions": socket_subscriptions,
+        "termination_flags": termination_flags,
+    }
+
+
+# Route for index page - redirection
+@research_bp.route("/")
+def index():
+    return render_template("pages/research.html")
+
+
+# Add the missing static file serving route
+@research_bp.route("/static/<path:path>")
+def serve_static(path):
+    """Serve static files"""
+    return send_from_directory(
+        os.path.join(os.path.dirname(os.path.dirname(__file__)), "static"), path
+    )
+
+
+# Add static route at the root level
+@research_bp.route("/redirect-static/<path:path>")
+def redirect_static(path):
+    """Redirect old static URLs to new static URLs"""
+    return redirect(url_for("static", filename=path))
+
+
+@research_bp.route("/progress/<int:research_id>")
+def progress_page(research_id):
+    """Render the research progress page"""
+    return render_template("pages/progress.html")
+
+
+@research_bp.route("/details/<int:research_id>")
+def research_details_page(research_id):
+    """Render the research details page"""
+    return render_template("pages/details.html")
+
+
+@research_bp.route("/results/<int:research_id>")
+def results_page(research_id):
+    """Render the research results page"""
+    return render_template("pages/results.html")
+
+
+@research_bp.route("/history")
+def history_page():
+    """Render the history page"""
+    return render_template("pages/history.html")
+
+
+# Add missing settings routes
+@research_bp.route("/settings", methods=["GET"])
+def settings_page():
+    """Render the settings page"""
+    return render_template("settings_dashboard.html")
+
+
+@research_bp.route("/settings/main", methods=["GET"])
+def main_config_page():
+    """Render the main settings config page"""
+    return render_template("main_config.html")
+
+
+@research_bp.route("/settings/collections", methods=["GET"])
+def collections_config_page():
+    """Render the collections config page"""
+    return render_template("collections_config.html")
+
+
+@research_bp.route("/settings/api_keys", methods=["GET"])
+def api_keys_config_page():
+    """Render the API keys config page"""
+    return render_template("api_keys_config.html")
+
+
+@research_bp.route("/settings/search_engines", methods=["GET"])
+def search_engines_config_page():
+    """Render the search engines config page"""
+    return render_template("search_engines_config.html")
+
+
+@research_bp.route("/settings/llm", methods=["GET"])
+def llm_config_page():
+    """Render the LLM config page"""
+    return render_template("llm_config.html")
+
+
+@research_bp.route("/api/start_research", methods=["POST"])
+def start_research():
+    data = request.json
+    query = data.get("query")
+    mode = data.get("mode", "quick")
+
+    # Get model provider and model selections
+    model_provider = data.get("model_provider", "OLLAMA")
+    model = data.get("model")
+    custom_endpoint = data.get("custom_endpoint")
+    search_engine = data.get("search_engine") or data.get("search_tool")
+    max_results = data.get("max_results")
+    time_period = data.get("time_period")
+    iterations = data.get("iterations")
+    questions_per_iteration = data.get("questions_per_iteration")
+
+    # Log the selections for troubleshooting
+    logger.info(
+        f"Starting research with provider: {model_provider}, model: {model}, search engine: {search_engine}"
+    )
+    logger.info(
+        f"Additional parameters: max_results={max_results}, time_period={time_period}, iterations={iterations}, questions={questions_per_iteration}"
+    )
+
+    if not query:
+        return jsonify({"status": "error", "message": "Query is required"}), 400
+
+    # Validate required parameters based on provider
+    if model_provider == "OPENAI_ENDPOINT" and not custom_endpoint:
+        return (
+            jsonify(
+                {
+                    "status": "error",
+                    "message": "Custom endpoint URL is required for OpenAI endpoint provider",
+                }
+            ),
+            400,
+        )
+
+    if not model:
+        return jsonify({"status": "error", "message": "Model is required"}), 400
+
+    # Check if there's any active research that's actually still running
+    if active_research:
+        # Verify each active research is still valid
+        stale_research_ids = []
+        for research_id, research_data in list(active_research.items()):
+            # Check database status
+            conn = get_db_connection()
+            cursor = conn.cursor()
+            cursor.execute(
+                "SELECT status FROM research_history WHERE id = ?", (research_id,)
+            )
+            result = cursor.fetchone()
+            conn.close()
+
+            # If the research doesn't exist in DB or is not in_progress, it's stale
+            if not result or result[0] != "in_progress":
+                stale_research_ids.append(research_id)
+            # Also check if thread is still alive
+            elif (
+                not research_data.get("thread")
+                or not research_data.get("thread").is_alive()
+            ):
+                stale_research_ids.append(research_id)
+
+        # Clean up any stale research processes
+        for stale_id in stale_research_ids:
+            print(f"Cleaning up stale research process: {stale_id}")
+            if stale_id in active_research:
+                del active_research[stale_id]
+            if stale_id in termination_flags:
+                del termination_flags[stale_id]
+
+        # After cleanup, check if there's still active research
+        if active_research:
+            return (
+                jsonify(
+                    {
+                        "status": "error",
+                        "message": "Another research is already in progress. Please wait for it to complete.",
+                    }
+                ),
+                409,
+            )
+
+    # Create a record in the database with explicit UTC timestamp
+    created_at = datetime.utcnow().isoformat()
+    conn = get_db_connection()
+    cursor = conn.cursor()
+
+    # Save research settings in the metadata field
+    research_settings = {
+        "model_provider": model_provider,
+        "model": model,
+        "custom_endpoint": custom_endpoint,
+        "search_engine": search_engine,
+        "max_results": max_results,
+        "time_period": time_period,
+        "iterations": iterations,
+        "questions_per_iteration": questions_per_iteration,
+    }
+
+    cursor.execute(
+        "INSERT INTO research_history (query, mode, status, created_at, progress_log, metadata) VALUES (?, ?, ?, ?, ?, ?)",
+        (
+            query,
+            mode,
+            "in_progress",
+            created_at,
+            json.dumps(
+                [{"time": created_at, "message": "Research started", "progress": 0}]
+            ),
+            json.dumps(research_settings),
+        ),
+    )
+    research_id = cursor.lastrowid
+    conn.commit()
+    conn.close()
+
+    # Start the research process with the selected parameters
+    research_thread = start_research_process(
+        research_id,
+        query,
+        mode,
+        active_research,
+        termination_flags,
+        run_research_process,
+        model_provider=model_provider,
+        model=model,
+        custom_endpoint=custom_endpoint,
+        search_engine=search_engine,
+        max_results=max_results,
+        time_period=time_period,
+        iterations=iterations,
+        questions_per_iteration=questions_per_iteration,
+    )
+
+    # Store the thread reference in active_research
+    active_research[research_id]["thread"] = research_thread
+
+    return jsonify({"status": "success", "research_id": research_id})
+
+
+@research_bp.route("/api/terminate/<int:research_id>", methods=["POST"])
+def terminate_research(research_id):
+    """Terminate an in-progress research process"""
+
+    # Check if the research exists and is in progress
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    cursor.execute("SELECT status FROM research_history WHERE id = ?", (research_id,))
+    result = cursor.fetchone()
+
+    if not result:
+        conn.close()
+        return jsonify({"status": "error", "message": "Research not found"}), 404
+
+    status = result[0]
+
+    # If it's not in progress, return an error
+    if status != "in_progress":
+        conn.close()
+        return (
+            jsonify({"status": "error", "message": "Research is not in progress"}),
+            400,
+        )
+
+    # Check if it's in the active_research dict
+    if research_id not in active_research:
+        # Update the status in the database
+        cursor.execute(
+            "UPDATE research_history SET status = ? WHERE id = ?",
+            ("suspended", research_id),
+        )
+        conn.commit()
+        conn.close()
+        return jsonify({"status": "success", "message": "Research terminated"})
+
+    # Set the termination flag
+    termination_flags[research_id] = True
+
+    # Log the termination request - using UTC timestamp
+    timestamp = datetime.utcnow().isoformat()
+    termination_message = "Research termination requested by user"
+    current_progress = active_research[research_id]["progress"]
+
+    # Create log entry
+    log_entry = {
+        "time": timestamp,
+        "message": termination_message,
+        "progress": current_progress,
+        "metadata": {"phase": "termination"},
+    }
+
+    # Add to in-memory log
+    active_research[research_id]["log"].append(log_entry)
+
+    # Add to database log
+    add_log_to_db(
+        research_id,
+        termination_message,
+        log_type="milestone",
+        progress=current_progress,
+        metadata={"phase": "termination"},
+    )
+
+    # Update the log in the database (old way for backward compatibility)
+    cursor.execute(
+        "SELECT progress_log FROM research_history WHERE id = ?", (research_id,)
+    )
+    log_result = cursor.fetchone()
+    if log_result:
+        try:
+            current_log = json.loads(log_result[0])
+        except Exception:
+            current_log = []
+        current_log.append(log_entry)
+        cursor.execute(
+            "UPDATE research_history SET progress_log = ? WHERE id = ?",
+            (json.dumps(current_log), research_id),
+        )
+
+    # IMMEDIATELY update the status to 'suspended' to avoid race conditions
+    cursor.execute(
+        "UPDATE research_history SET status = ? WHERE id = ?",
+        ("suspended", research_id),
+    )
+    conn.commit()
+    conn.close()
+
+    # Emit a socket event for the termination request
+    try:
+        event_data = {
+            "status": "suspended",  # Changed from 'terminating' to 'suspended'
+            "message": "Research was suspended by user request",
+        }
+
+        from ..services.socket_service import emit_socket_event
+
+        emit_socket_event(f"research_progress_{research_id}", event_data)
+
+    except Exception as socket_error:
+        print(f"Socket emit error (non-critical): {str(socket_error)}")
+
+    return jsonify({"status": "success", "message": "Research termination requested"})
+
+
+@research_bp.route("/api/delete/<int:research_id>", methods=["DELETE"])
+def delete_research(research_id):
+    """Delete a research record"""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+
+    # First check if the research exists and is not in progress
+    cursor.execute(
+        "SELECT status, report_path FROM research_history WHERE id = ?", (research_id,)
+    )
+    result = cursor.fetchone()
+
+    if not result:
+        conn.close()
+        return jsonify({"status": "error", "message": "Research not found"}), 404
+
+    status, report_path = result
+
+    # Don't allow deleting research in progress
+    if status == "in_progress" and research_id in active_research:
+        conn.close()
+        return (
+            jsonify(
+                {
+                    "status": "error",
+                    "message": "Cannot delete research that is in progress",
+                }
+            ),
+            400,
+        )
+
+    # Delete report file if it exists
+    if report_path and os.path.exists(report_path):
+        try:
+            os.remove(report_path)
+        except Exception as e:
+            print(f"Error removing report file: {str(e)}")
+
+    # Delete the database record
+    cursor.execute("DELETE FROM research_history WHERE id = ?", (research_id,))
+    conn.commit()
+    conn.close()
+
+    return jsonify({"status": "success"})
+
+
+@research_bp.route("/api/clear_history", methods=["POST"])
+def clear_history():
+    """Clear all research history"""
+    try:
+        conn = get_db_connection()
+        cursor = conn.cursor()
+
+        # Get all research IDs first to clean up files
+        cursor.execute("SELECT id, report_path FROM research_history")
+        research_records = cursor.fetchall()
+
+        # Clean up report files
+        for research_id, report_path in research_records:
+            # Skip active research
+            if research_id in active_research:
+                continue
+
+            # Delete report file if it exists
+            if report_path and os.path.exists(report_path):
+                try:
+                    os.remove(report_path)
+                except Exception as e:
+                    print(f"Error removing report file: {str(e)}")
+
+        # Delete records from the database, except active research
+        placeholders = ", ".join(["?"] * len(active_research))
+        if active_research:
+            cursor.execute(
+                f"DELETE FROM research_history WHERE id NOT IN ({placeholders})",
+                list(active_research.keys()),
+            )
+        else:
+            cursor.execute("DELETE FROM research_history")
+
+        conn.commit()
+        conn.close()
+
+        return jsonify({"status": "success"})
+    except Exception as e:
+        return jsonify({"status": "error", "message": str(e)}), 500
+
+
+@research_bp.route("/open_file_location", methods=["POST"])
+def open_file_location():
+    """Open a file location in the system file explorer"""
+    data = request.json
+    file_path = data.get("path")
+
+    if not file_path:
+        return jsonify({"status": "error", "message": "Path is required"}), 400
+
+    # Convert to absolute path if needed
+    if not os.path.isabs(file_path):
+        file_path = os.path.abspath(file_path)
+
+    # Check if path exists
+    if not os.path.exists(file_path):
+        return jsonify({"status": "error", "message": "Path does not exist"}), 404
+
+    try:
+        if platform.system() == "Windows":
+            # On Windows, open the folder and select the file
+            if os.path.isfile(file_path):
+                subprocess.run(["explorer", "/select,", file_path], check=True)
+            else:
+                # If it's a directory, just open it
+                subprocess.run(["explorer", file_path], check=True)
+        elif platform.system() == "Darwin":  # macOS
+            subprocess.run(["open", file_path], check=True)
+        else:  # Linux and others
+            subprocess.run(["xdg-open", os.path.dirname(file_path)], check=True)
+
+        return jsonify({"status": "success"})
+    except Exception as e:
+        return jsonify({"status": "error", "message": str(e)}), 500
+
+
+@research_bp.route("/api/save_raw_config", methods=["POST"])
+def save_raw_config():
+    """Save raw configuration"""
+    data = request.json
+    raw_config = data.get("raw_config")
+
+    if not raw_config:
+        return (
+            jsonify({"success": False, "error": "Raw configuration is required"}),
+            400,
+        )
+
+    try:
+        # Get the config file path
+        config_dir = os.path.join(os.path.expanduser("~"), ".local_deep_research")
+        os.makedirs(config_dir, exist_ok=True)
+        config_path = os.path.join(config_dir, "config.toml")
+
+        # Write the configuration to file
+        with open(config_path, "w", encoding="utf-8") as f:
+            f.write(raw_config)
+
+        return jsonify({"success": True})
+    except Exception as e:
+        return jsonify({"success": False, "error": str(e)}), 500
+
+
+@research_bp.route("/api/history", methods=["GET"])
+def get_history():
+    """Get research history"""
+    try:
+        conn = get_db_connection()
+        cursor = conn.cursor()
+
+        # Check if title column exists in the database
+        cursor.execute("PRAGMA table_info(research_history)")
+        columns = [column[1] for column in cursor.fetchall()]
+
+        # Build query based on existing columns
+        select_columns = [
+            "id",
+            "query",
+            "mode",
+            "status",
+            "created_at",
+            "completed_at",
+            "report_path",
+        ]
+
+        # Optionally include title if it exists
+        if "title" in columns:
+            select_columns.append("title")
+
+        # Construct query
+        select_query = f"SELECT {', '.join(select_columns)} FROM research_history ORDER BY created_at DESC"
+
+        # Execute query
+        cursor.execute(select_query)
+
+        history_items = []
+        for row in cursor.fetchall():
+            # Extract values
+            row_data = dict(zip(select_columns, row))
+            research_id = row_data["id"]
+            query = row_data["query"]
+            mode = row_data["mode"]
+            status = row_data["status"]
+            created_at = row_data["created_at"]
+            completed_at = row_data["completed_at"]
+            report_path = row_data["report_path"]
+            title = row_data.get(
+                "title", None
+            )  # Use get to handle title not being present
+
+            # Calculate duration if completed
+            duration_seconds = None
+            if completed_at and created_at:
+                try:
+                    duration_seconds = calculate_duration(created_at, completed_at)
+                except Exception as e:
+                    print(f"Error calculating duration: {e}")
+
+            # Create a history item
+            item = {
+                "id": research_id,
+                "query": query,
+                "mode": mode,
+                "status": status,
+                "created_at": created_at,
+                "completed_at": completed_at,
+                "duration_seconds": duration_seconds,
+                "report_path": report_path,
+            }
+
+            # Add title if not None
+            if title is not None:
+                item["title"] = title
+
+            history_items.append(item)
+
+        conn.close()
+        return jsonify({"status": "success", "items": history_items})
+    except Exception as e:
+        # Import traceback only when needed
+        import traceback
+
+        print(f"Error getting history: {e}")
+        print(traceback.format_exc())
+        return jsonify({"status": "error", "message": str(e)}), 500
+
+
+@research_bp.route("/api/research/<research_id>/status")
+def get_research_status(research_id):
+    """Get the status of a research process"""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    cursor.execute(
+        "SELECT status, progress, completed_at, report_path, metadata FROM research_history WHERE id = ?",
+        (research_id,),
+    )
+    result = cursor.fetchone()
+
+    if result is None:
+        conn.close()
+        return jsonify({"error": "Research not found"}), 404
+
+    status, progress, completed_at, report_path, metadata_str = result
+
+    # Parse metadata if it exists
+    metadata = {}
+    if metadata_str:
+        try:
+            metadata = json.loads(metadata_str)
+        except json.JSONDecodeError:
+            current_app.logger.warning(
+                f"Invalid JSON in metadata for research {research_id}"
+            )
+
+    # Extract and format error information for better UI display
+    error_info = {}
+    if metadata and "error" in metadata:
+        error_msg = metadata["error"]
+        error_type = "unknown"
+
+        # Detect specific error types
+        if "timeout" in error_msg.lower():
+            error_type = "timeout"
+            error_info = {
+                "type": "timeout",
+                "message": "LLM service timed out during synthesis. This may be due to high server load or connectivity issues.",
+                "suggestion": "Try again later or use a smaller query scope.",
+            }
+        elif (
+            "token limit" in error_msg.lower() or "context length" in error_msg.lower()
+        ):
+            error_type = "token_limit"
+            error_info = {
+                "type": "token_limit",
+                "message": "The research query exceeded the AI model's token limit during synthesis.",
+                "suggestion": "Try using a more specific query or reduce the research scope.",
+            }
+        elif (
+            "final answer synthesis fail" in error_msg.lower()
+            or "llm error" in error_msg.lower()
+        ):
+            error_type = "llm_error"
+            error_info = {
+                "type": "llm_error",
+                "message": "The AI model encountered an error during final answer synthesis.",
+                "suggestion": "Check that your LLM service is running correctly or try a different model.",
+            }
+        elif "ollama" in error_msg.lower():
+            error_type = "ollama_error"
+            error_info = {
+                "type": "ollama_error",
+                "message": "The Ollama service is not responding properly.",
+                "suggestion": "Make sure Ollama is running with 'ollama serve' and the model is downloaded.",
+            }
+        elif "connection" in error_msg.lower():
+            error_type = "connection"
+            error_info = {
+                "type": "connection",
+                "message": "Connection error with the AI service.",
+                "suggestion": "Check your internet connection and AI service status.",
+            }
+        elif metadata.get("solution"):
+            # Use the solution provided in metadata if available
+            error_info = {
+                "type": error_type,
+                "message": error_msg,
+                "suggestion": metadata.get("solution"),
+            }
+        else:
+            # Generic error with the original message
+            error_info = {
+                "type": error_type,
+                "message": error_msg,
+                "suggestion": "Try again with a different query or check the application logs.",
+            }
+
+    # Add error_info to the response if it exists
+    if error_info:
+        metadata["error_info"] = error_info
+
+    conn.close()
+    return jsonify(
+        {
+            "status": status,
+            "progress": progress,
+            "completed_at": completed_at,
+            "report_path": report_path,
+            "metadata": metadata,
+        }
+    )