local-deep-research 0.1.0__py3-none-any.whl

local_deep_research/web/app.py

@@ -0,0 +1,1209 @@
+import os
+import json
+import time
+import sqlite3
+import threading
+from datetime import datetime
+from flask import Flask, render_template, request, jsonify, send_from_directory, Response, make_response, current_app, Blueprint, redirect, url_for, flash
+from flask_socketio import SocketIO, emit
+from local_deep_research.search_system import AdvancedSearchSystem
+from local_deep_research.report_generator import IntegratedReportGenerator
+# Move this import up to ensure it's available globally
+from dateutil import parser
+import traceback
+import pkg_resources
+# Import the new configuration manager
+from local_deep_research.config import get_config_dir 
+import logging
+logger = logging.getLogger(__name__)
+
+CONFIG_DIR = get_config_dir() / "config"
+MAIN_CONFIG_FILE = CONFIG_DIR / "settings.toml"
+LLM_CONFIG_FILE = CONFIG_DIR / "llm_config.py"
+LOCAL_COLLECTIONS_FILE = CONFIG_DIR / "local_collections.toml"
+import toml
+
+# Set flag for tracking OpenAI availability - we'll check it only when needed
+OPENAI_AVAILABLE = False
+
+# Initialize Flask app
+try:
+    import os
+    import logging
+    from local_deep_research.utilties.setup_utils import setup_user_directories
+    
+    # Configure logging
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__name__)
+    
+    # Explicitly run setup
+    logger.info("Initializing configuration...")
+    setup_user_directories()
+    
+    # Get directories based on package installation
+    PACKAGE_DIR = pkg_resources.resource_filename('local_deep_research', 'web')
+    STATIC_DIR = os.path.join(PACKAGE_DIR, 'static')
+    TEMPLATE_DIR = os.path.join(PACKAGE_DIR, 'templates')
+    
+    # Setup logging
+    logging.basicConfig(level=logging.INFO)
+    
+    # Create directories and default configs if needed
+    setup_user_directories()
+
+    # Initialize Flask app with package directories
+    app = Flask(__name__, 
+                static_folder=STATIC_DIR,
+                template_folder=TEMPLATE_DIR)
+    print(f"Using package static path: {STATIC_DIR}")
+    print(f"Using package template path: {TEMPLATE_DIR}")
+except Exception as e:
+    # Fallback for development
+    print(f"Package directories not found, using fallback paths: {str(e)}")
+    app = Flask(__name__, 
+                static_folder=os.path.abspath('static'),
+                template_folder=os.path.abspath('templates'))
+app.config['SECRET_KEY'] = 'deep-research-secret-key'
+
+# Create a Blueprint for the research application
+research_bp = Blueprint('research', __name__, url_prefix='/research')
+
+# Add improved Socket.IO configuration with better error handling
+socketio = SocketIO(
+    app, 
+    cors_allowed_origins="*",
+    async_mode='threading',
+    path='/research/socket.io',
+    logger=True,
+    engineio_logger=True,
+    ping_timeout=20,
+    ping_interval=5
+)
+
+# Active research processes and socket subscriptions
+active_research = {}
+socket_subscriptions = {}
+
+# Add termination flags dictionary
+termination_flags = {}
+
+# Database setup
+DB_PATH = 'research_history.db'
+
+# Add Content Security Policy headers to allow Socket.IO to function
+@app.after_request
+def add_security_headers(response):
+    # Define a permissive CSP for development that allows Socket.IO to function
+    csp = (
+        "default-src 'self'; "
+        "connect-src 'self' ws: wss: http: https:; " 
+        "script-src 'self' 'unsafe-inline' 'unsafe-eval' cdnjs.cloudflare.com cdn.jsdelivr.net unpkg.com; "
+        "style-src 'self' 'unsafe-inline' cdnjs.cloudflare.com; "
+        "font-src 'self' cdnjs.cloudflare.com; "
+        "img-src 'self' data:; "
+        "worker-src blob:; "
+        "frame-src 'self';"
+    )
+    
+    response.headers['Content-Security-Policy'] = csp
+    response.headers['X-Content-Security-Policy'] = csp
+    
+    # Add CORS headers for API requests
+    if request.path.startswith('/api/'):
+        response.headers['Access-Control-Allow-Origin'] = '*'
+        response.headers['Access-Control-Allow-Methods'] = 'GET, POST, DELETE, OPTIONS'
+        response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
+    
+    return response
+
+# Add a middleware layer to handle abrupt disconnections
+@app.before_request
+def handle_websocket_requests():
+    if request.path.startswith('/research/socket.io'):
+        try:
+            if not request.environ.get('werkzeug.socket'):
+                return
+        except Exception as e:
+            print(f"WebSocket preprocessing error: {e}")
+            # Return empty response to prevent further processing
+            return '', 200
+
+# Initialize the database
+def init_db():
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    
+    # Create the table if it doesn't exist
+    cursor.execute('''
+    CREATE TABLE IF NOT EXISTS research_history (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        query TEXT NOT NULL,
+        mode TEXT NOT NULL,
+        status TEXT NOT NULL,
+        created_at TEXT NOT NULL,
+        completed_at TEXT,
+        duration_seconds INTEGER,
+        report_path TEXT,
+        metadata TEXT,
+        progress_log TEXT
+    )
+    ''')
+    
+    # Check if the duration_seconds column exists, add it if missing
+    cursor.execute('PRAGMA table_info(research_history)')
+    columns = [column[1] for column in cursor.fetchall()]
+    
+    if 'duration_seconds' not in columns:
+        print("Adding missing 'duration_seconds' column to research_history table")
+        cursor.execute('ALTER TABLE research_history ADD COLUMN duration_seconds INTEGER')
+    
+    conn.commit()
+    conn.close()
+
+# Helper function to calculate duration between created_at and completed_at timestamps
+def calculate_duration(created_at_str):
+    """
+    Calculate duration in seconds between created_at timestamp and now.
+    Handles various timestamp formats and returns None if calculation fails.
+    """
+    if not created_at_str:
+        return None
+        
+    now = datetime.utcnow()
+    duration_seconds = None
+    
+    try:
+        # Proper parsing of ISO format
+        if 'T' in created_at_str:  # ISO format with T separator
+            start_time = datetime.fromisoformat(created_at_str)
+        else:  # Older format without T
+            # Try different formats
+            try:
+                start_time = datetime.strptime(created_at_str, '%Y-%m-%d %H:%M:%S.%f')
+            except ValueError:
+                try:
+                    start_time = datetime.strptime(created_at_str, '%Y-%m-%d %H:%M:%S')
+                except ValueError:
+                    # Last resort fallback
+                    start_time = datetime.fromisoformat(created_at_str.replace(' ', 'T'))
+        
+        # Ensure we're comparing UTC times
+        duration_seconds = int((now - start_time).total_seconds())
+    except Exception as e:
+        print(f"Error calculating duration: {str(e)}")
+        # Fallback method if parsing fails
+        try:
+            start_time_fallback = parser.parse(created_at_str)
+            duration_seconds = int((now - start_time_fallback).total_seconds())
+        except:
+            print(f"Fallback duration calculation also failed for timestamp: {created_at_str}")
+    
+    return duration_seconds
+
+# Initialize the database on startup
+def initialize():
+    init_db()
+
+# Call initialize immediately when app is created
+initialize()
+
+# Route for index page - keep this at root level for easy access
+@app.route('/')
+def root_index():
+    return redirect(url_for('research.index'))
+
+# Update all routes with the research prefix
+@research_bp.route('/')
+def index():
+    return render_template('index.html')
+
+@research_bp.route('/static/<path:path>')
+def serve_static(path):
+    try:
+        print(f"Serving static file: {path}")
+        print(f"Static folder path: {app.static_folder}")
+        return send_from_directory(app.static_folder, path)
+    except Exception as e:
+        print(f"Error serving static file {path}: {str(e)}")
+        return f"Error serving file: {str(e)}", 404
+
+@research_bp.route('/api/history', methods=['GET'])
+def get_history():
+    """Get the research history"""
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        conn.row_factory = sqlite3.Row
+        cursor = conn.cursor()
+        
+        # Get all history records ordered by latest first
+        cursor.execute('SELECT * FROM research_history ORDER BY created_at DESC')
+        results = cursor.fetchall()
+        conn.close()
+        
+        # Convert to list of dicts
+        history = []
+        for result in results:
+            item = dict(result)
+            
+            # Ensure all keys exist with default values
+            if 'id' not in item:
+                item['id'] = None
+            if 'query' not in item:
+                item['query'] = 'Untitled Research'
+            if 'mode' not in item:
+                item['mode'] = 'quick'
+            if 'status' not in item:
+                item['status'] = 'unknown'
+            if 'created_at' not in item:
+                item['created_at'] = None
+            if 'completed_at' not in item:
+                item['completed_at'] = None
+            if 'duration_seconds' not in item:
+                item['duration_seconds'] = None
+            if 'report_path' not in item:
+                item['report_path'] = None
+            if 'metadata' not in item:
+                item['metadata'] = '{}'
+            if 'progress_log' not in item:
+                item['progress_log'] = '[]'
+            
+            # Ensure timestamps are in ISO format
+            if item['created_at'] and 'T' not in item['created_at']:
+                try:
+                    # Convert to ISO format if it's not already
+                    dt = parser.parse(item['created_at'])
+                    item['created_at'] = dt.isoformat()
+                except:
+                    pass
+                
+            if item['completed_at'] and 'T' not in item['completed_at']:
+                try:
+                    # Convert to ISO format if it's not already
+                    dt = parser.parse(item['completed_at'])
+                    item['completed_at'] = dt.isoformat()
+                except:
+                    pass
+                
+            # Recalculate duration based on timestamps if it's null but both timestamps exist
+            if item['duration_seconds'] is None and item['created_at'] and item['completed_at']:
+                try:
+                    start_time = parser.parse(item['created_at'])
+                    end_time = parser.parse(item['completed_at'])
+                    item['duration_seconds'] = int((end_time - start_time).total_seconds())
+                except Exception as e:
+                    print(f"Error recalculating duration: {str(e)}")
+            
+            history.append(item)
+        
+        # Add CORS headers
+        response = make_response(jsonify(history))
+        response.headers.add('Access-Control-Allow-Origin', '*')
+        response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
+        response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
+        return response
+    except Exception as e:
+        print(f"Error getting history: {str(e)}")
+        print(traceback.format_exc())
+        # Return empty array with CORS headers
+        response = make_response(jsonify([]))
+        response.headers.add('Access-Control-Allow-Origin', '*')
+        response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
+        response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
+        return response
+
+@research_bp.route('/api/start_research', methods=['POST'])
+def start_research():
+    data = request.json
+    query = data.get('query')
+    mode = data.get('mode', 'quick')
+    
+    if not query:
+        return jsonify({'status': 'error', 'message': 'Query is required'}), 400
+        
+    # Check if there's any active research
+    if active_research:
+        return jsonify({
+            'status': 'error', 
+            'message': 'Another research is already in progress. Please wait for it to complete.'
+        }), 409
+        
+    # Create a record in the database with explicit UTC timestamp
+    created_at = datetime.utcnow().isoformat()
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute(
+        'INSERT INTO research_history (query, mode, status, created_at, progress_log) VALUES (?, ?, ?, ?, ?)',
+        (query, mode, 'in_progress', created_at, json.dumps([{"time": created_at, "message": "Research started", "progress": 0}]))
+    )
+    research_id = cursor.lastrowid
+    conn.commit()
+    conn.close()
+    
+    # Start research process in a background thread
+    thread = threading.Thread(
+        target=run_research_process,
+        args=(research_id, query, mode)
+    )
+    thread.daemon = True
+    thread.start()
+    
+    active_research[research_id] = {
+        'thread': thread,
+        'progress': 0,
+        'status': 'in_progress',
+        'log': [{"time": created_at, "message": "Research started", "progress": 0}]
+    }
+    
+    return jsonify({
+        'status': 'success',
+        'research_id': research_id
+    })
+
+@research_bp.route('/api/research/<int:research_id>')
+def get_research_status(research_id):
+    conn = sqlite3.connect(DB_PATH)
+    conn.row_factory = sqlite3.Row
+    cursor = conn.cursor()
+    cursor.execute('SELECT * FROM research_history WHERE id = ?', (research_id,))
+    result = dict(cursor.fetchone() or {})
+    conn.close()
+    
+    if not result:
+        return jsonify({'status': 'error', 'message': 'Research not found'}), 404
+        
+    # Add progress information
+    if research_id in active_research:
+        result['progress'] = active_research[research_id]['progress']
+        result['log'] = active_research[research_id]['log']
+    elif result.get('status') == 'completed':
+        result['progress'] = 100
+        try:
+            result['log'] = json.loads(result.get('progress_log', '[]'))
+        except:
+            result['log'] = []
+    else:
+        result['progress'] = 0
+        try:
+            result['log'] = json.loads(result.get('progress_log', '[]'))
+        except:
+            result['log'] = []
+        
+    return jsonify(result)
+
+@research_bp.route('/api/research/<int:research_id>/details')
+def get_research_details(research_id):
+    """Get detailed progress log for a specific research"""
+    conn = sqlite3.connect(DB_PATH)
+    conn.row_factory = sqlite3.Row
+    cursor = conn.cursor()
+    cursor.execute('SELECT * FROM research_history WHERE id = ?', (research_id,))
+    result = dict(cursor.fetchone() or {})
+    conn.close()
+    
+    if not result:
+        return jsonify({'status': 'error', 'message': 'Research not found'}), 404
+    
+    try:
+        # Get the progress log
+        progress_log = json.loads(result.get('progress_log', '[]'))
+    except:
+        progress_log = []
+        
+    # If this is an active research, get the latest log
+    if research_id in active_research:
+        progress_log = active_research[research_id]['log']
+    
+    return jsonify({
+        'status': 'success',
+        'research_id': research_id,
+        'query': result.get('query'),
+        'mode': result.get('mode'),
+        'status': result.get('status'),
+        'progress': active_research.get(research_id, {}).get('progress', 100 if result.get('status') == 'completed' else 0),
+        'created_at': result.get('created_at'),
+        'completed_at': result.get('completed_at'),
+        'log': progress_log
+    })
+
+@research_bp.route('/api/report/<int:research_id>')
+def get_report(research_id):
+    conn = sqlite3.connect(DB_PATH)
+    conn.row_factory = sqlite3.Row
+    cursor = conn.cursor()
+    cursor.execute('SELECT * FROM research_history WHERE id = ?', (research_id,))
+    result = dict(cursor.fetchone() or {})
+    conn.close()
+    
+    if not result or not result.get('report_path'):
+        return jsonify({'status': 'error', 'message': 'Report not found'}), 404
+        
+    try:
+        with open(result['report_path'], 'r', encoding='utf-8') as f:
+            content = f.read()
+        return jsonify({
+            'status': 'success',
+            'content': content,
+            'metadata': json.loads(result.get('metadata', '{}'))
+        })
+    except Exception as e:
+        return jsonify({'status': 'error', 'message': str(e)}), 500
+
+@research_bp.route('/research/details/<int:research_id>')
+def research_details_page(research_id):
+    """Render the research details page"""
+    return render_template('index.html')
+
+@socketio.on('connect')
+def handle_connect():
+    print(f"Client connected: {request.sid}")
+
+@socketio.on('disconnect')
+def handle_disconnect():
+    try:
+        print(f"Client disconnected: {request.sid}")
+        # Clean up subscriptions for this client
+        for research_id, subscribers in list(socket_subscriptions.items()):
+            if request.sid in subscribers:
+                subscribers.remove(request.sid)
+            if not subscribers:
+                socket_subscriptions.pop(research_id, None)
+    except Exception as e:
+        print(f"Error handling disconnect: {e}")
+
+@socketio.on('subscribe_to_research')
+def handle_subscribe(data):
+    research_id = data.get('research_id')
+    if research_id:
+        if research_id not in socket_subscriptions:
+            socket_subscriptions[research_id] = set()
+        socket_subscriptions[research_id].add(request.sid)
+        print(f"Client {request.sid} subscribed to research {research_id}")
+        
+        # Send current status immediately if available
+        if research_id in active_research:
+            progress = active_research[research_id]['progress']
+            latest_log = active_research[research_id]['log'][-1] if active_research[research_id]['log'] else None
+            
+            if latest_log:
+                emit(f'research_progress_{research_id}', {
+                    'progress': progress,
+                    'message': latest_log.get('message', 'Processing...'),
+                    'status': 'in_progress',
+                    'log_entry': latest_log
+                })
+
+@socketio.on_error
+def handle_socket_error(e):
+    print(f"Socket.IO error: {str(e)}")
+    # Don't propagate exceptions to avoid crashing the server
+    return False
+
+@socketio.on_error_default
+def handle_default_error(e):
+    print(f"Unhandled Socket.IO error: {str(e)}")
+    # Don't propagate exceptions to avoid crashing the server
+    return False
+
+def run_research_process(research_id, query, mode):
+    try:
+        system = AdvancedSearchSystem()
+        
+        # Set up progress callback
+        def progress_callback(message, progress_percent, metadata):
+            timestamp = datetime.utcnow().isoformat()
+            log_entry = {
+                "time": timestamp,
+                "message": message,
+                "progress": progress_percent,
+                "metadata": metadata
+            }
+            
+            # Check if termination was requested
+            if research_id in termination_flags and termination_flags[research_id]:
+                # Clean up and exit
+                raise Exception("Research was terminated by user")
+            
+            # Update active research record
+            if research_id in active_research:
+                active_research[research_id]['log'].append(log_entry)
+                if progress_percent is not None:
+                    active_research[research_id]['progress'] = progress_percent
+                
+                # Save to database (but not too frequently)
+                if progress_percent is None or progress_percent % 10 == 0 or metadata.get('phase') in ['complete', 'iteration_complete']:
+                    conn = sqlite3.connect(DB_PATH)
+                    cursor = conn.cursor()
+                    cursor.execute(
+                        'SELECT progress_log FROM research_history WHERE id = ?',
+                        (research_id,)
+                    )
+                    result = cursor.fetchone()
+                    if result:
+                        try:
+                            current_log = json.loads(result[0])
+                        except:
+                            current_log = []
+                        current_log.append(log_entry)
+                        cursor.execute(
+                            'UPDATE research_history SET progress_log = ? WHERE id = ?',
+                            (json.dumps(current_log), research_id)
+                        )
+                        conn.commit()
+                    conn.close()
+                
+                # Emit socket event with try/except block to handle connection issues
+                try:
+                    event_data = {
+                        'progress': progress_percent,
+                        'message': message,
+                        'status': 'in_progress',
+                        'log_entry': log_entry
+                    }
+                    
+                    # Emit to the specific research channel
+                    socketio.emit(f'research_progress_{research_id}', event_data)
+                    
+                    # Also emit to specific subscribers if available
+                    if research_id in socket_subscriptions and socket_subscriptions[research_id]:
+                        for sid in socket_subscriptions[research_id]:
+                            try:
+                                socketio.emit(
+                                    f'research_progress_{research_id}', 
+                                    event_data,
+                                    room=sid
+                                )
+                            except Exception as sub_err:
+                                print(f"Error emitting to subscriber {sid}: {str(sub_err)}")
+                    
+                except Exception as socket_error:
+                    # Log socket error but continue with the research process
+                    print(f"Socket emit error (non-critical): {str(socket_error)}")
+        
+        # Set the progress callback in the system
+        system.set_progress_callback(progress_callback)
+        
+        # Run the search
+        progress_callback("Starting research process", 5, {"phase": "init"})
+        results = system.analyze_topic(query)
+        progress_callback("Search complete, generating output", 80, {"phase": "output_generation"})
+        
+        # Generate output based on mode
+        if mode == 'quick':
+            # Quick Summary
+            if results.get('findings'):
+                #initial_analysis = [finding['content'] for finding in results['findings']]
+                summary = ""
+                raw_formatted_findings = results['formatted_findings']
+                
+                # ADDED CODE: Convert debug output to clean markdown
+                clean_markdown = convert_debug_to_markdown(raw_formatted_findings, query)
+                
+                # Save as markdown file
+                output_dir = "research_outputs"
+                if not os.path.exists(output_dir):
+                    os.makedirs(output_dir)
+                    
+                safe_query = "".join(x for x in query if x.isalnum() or x in [" ", "-", "_"])[:50]
+                safe_query = safe_query.replace(" ", "_").lower()
+                report_path = os.path.join(output_dir, f"quick_summary_{safe_query}.md")
+                
+                with open(report_path, "w", encoding="utf-8") as f:
+                    f.write("# Quick Research Summary\n\n")
+                    f.write(f"Query: {query}\n\n")
+                    f.write(clean_markdown)  # Use clean markdown instead of raw findings
+                    f.write("\n\n## Research Metrics\n")
+                    f.write(f"- Search Iterations: {results['iterations']}\n")
+                    f.write(f"- Generated at: {datetime.utcnow().isoformat()}\n")
+                
+                # Update database
+                metadata = {
+                    'iterations': results['iterations'],
+                    'generated_at': datetime.utcnow().isoformat()
+                }
+                
+                # Calculate duration in seconds - using UTC consistently
+                now = datetime.utcnow()
+                completed_at = now.isoformat()
+                
+                # Get the start time from the database
+                conn = sqlite3.connect(DB_PATH)
+                cursor = conn.cursor()
+                cursor.execute('SELECT created_at FROM research_history WHERE id = ?', (research_id,))
+                result = cursor.fetchone()
+                
+                # Use the helper function for consistent duration calculation
+                duration_seconds = calculate_duration(result[0])
+                
+                # Update the record
+                cursor.execute(
+                    'UPDATE research_history SET status = ?, completed_at = ?, duration_seconds = ?, report_path = ?, metadata = ? WHERE id = ?',
+                    ('completed', completed_at, duration_seconds, report_path, json.dumps(metadata), research_id)
+                )
+                conn.commit()
+                conn.close()
+                
+                progress_callback("Research completed successfully", 100, {"phase": "complete", "report_path": report_path})
+        else:
+            # Full Report
+            progress_callback("Generating detailed report...", 85, {"phase": "report_generation"})
+            report_generator = IntegratedReportGenerator()
+            final_report = report_generator.generate_report(results, query)
+            progress_callback("Report generation complete", 95, {"phase": "report_complete"})
+            
+            # Save as markdown file
+            output_dir = "research_outputs"
+            if not os.path.exists(output_dir):
+                os.makedirs(output_dir)
+                
+            safe_query = "".join(x for x in query if x.isalnum() or x in [" ", "-", "_"])[:50]
+            safe_query = safe_query.replace(" ", "_").lower()
+            report_path = os.path.join(output_dir, f"detailed_report_{safe_query}.md")
+            
+            with open(report_path, "w", encoding="utf-8") as f:
+                f.write(final_report['content'])
+            
+            # Update database
+            metadata = final_report['metadata']
+            metadata['iterations'] = results['iterations']
+            
+            # Calculate duration in seconds - using UTC consistently
+            now = datetime.utcnow()
+            completed_at = now.isoformat()
+            
+            # Get the start time from the database
+            conn = sqlite3.connect(DB_PATH)
+            cursor = conn.cursor()
+            cursor.execute('SELECT created_at FROM research_history WHERE id = ?', (research_id,))
+            result = cursor.fetchone()
+            
+            # Use the helper function for consistent duration calculation
+            duration_seconds = calculate_duration(result[0])
+            
+            cursor.execute(
+                'UPDATE research_history SET status = ?, completed_at = ?, duration_seconds = ?, report_path = ?, metadata = ? WHERE id = ?',
+                ('completed', completed_at, duration_seconds, report_path, json.dumps(metadata), research_id)
+            )
+            conn.commit()
+            conn.close()
+            
+            progress_callback("Research completed successfully", 100, {"phase": "complete", "report_path": report_path})
+            
+        # Clean up
+        if research_id in active_research:
+            del active_research[research_id]
+            
+    except Exception as e:
+        # Handle error
+        error_message = f"Research failed: {str(e)}"
+        print(f"Research error: {error_message}")
+        try:
+            progress_callback(error_message, None, {"phase": "error", "error": str(e)})
+        
+            conn = sqlite3.connect(DB_PATH)
+            cursor = conn.cursor()
+            
+            # If termination was requested, mark as suspended instead of failed
+            status = 'suspended' if (research_id in termination_flags and termination_flags[research_id]) else 'failed'
+            message = "Research was terminated by user" if status == 'suspended' else str(e)
+            
+            # Calculate duration up to termination point - using UTC consistently
+            now = datetime.utcnow()
+            completed_at = now.isoformat()
+            
+            # Get the start time from the database
+            duration_seconds = None
+            cursor.execute('SELECT created_at FROM research_history WHERE id = ?', (research_id,))
+            result = cursor.fetchone()
+            
+            # Use the helper function for consistent duration calculation
+            if result and result[0]:
+                duration_seconds = calculate_duration(result[0])
+            
+            cursor.execute(
+                'UPDATE research_history SET status = ?, completed_at = ?, duration_seconds = ?, metadata = ? WHERE id = ?',
+                (status, completed_at, duration_seconds, json.dumps({'error': message}), research_id)
+            )
+            conn.commit()
+            conn.close()
+            
+            try:
+                socketio.emit(f'research_progress_{research_id}', {
+                    'status': status,
+                    'error': message
+                })
+                
+                # Also notify specific subscribers
+                if research_id in socket_subscriptions and socket_subscriptions[research_id]:
+                    for sid in socket_subscriptions[research_id]:
+                        try:
+                            socketio.emit(
+                                f'research_progress_{research_id}', 
+                                {'status': status, 'error': message},
+                                room=sid
+                            )
+                        except Exception as sub_err:
+                            print(f"Error emitting to subscriber {sid}: {str(sub_err)}")
+                            
+            except Exception as socket_error:
+                print(f"Failed to emit error via socket: {str(socket_error)}")
+        except Exception as inner_e:
+            print(f"Error in error handler: {str(inner_e)}")
+        
+        # Clean up resources
+        if research_id in active_research:
+            del active_research[research_id]
+        if research_id in termination_flags:
+            del termination_flags[research_id]
+
+@research_bp.route('/api/research/<int:research_id>/terminate', methods=['POST'])
+def terminate_research(research_id):
+    """Terminate an in-progress research process"""
+    
+    # Check if the research exists and is in progress
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute('SELECT status FROM research_history WHERE id = ?', (research_id,))
+    result = cursor.fetchone()
+    
+    if not result:
+        conn.close()
+        return jsonify({'status': 'error', 'message': 'Research not found'}), 404
+    
+    status = result[0]
+    
+    # If it's not in progress, return an error
+    if status != 'in_progress':
+        conn.close()
+        return jsonify({'status': 'error', 'message': 'Research is not in progress'}), 400
+    
+    # Check if it's in the active_research dict
+    if research_id not in active_research:
+        # Update the status in the database
+        cursor.execute('UPDATE research_history SET status = ? WHERE id = ?', ('suspended', research_id))
+        conn.commit()
+        conn.close()
+        return jsonify({'status': 'success', 'message': 'Research terminated'})
+    
+    # Set the termination flag
+    termination_flags[research_id] = True
+    
+    # Log the termination request - using UTC timestamp
+    timestamp = datetime.utcnow().isoformat()
+    log_entry = {
+        "time": timestamp,
+        "message": "Research termination requested by user",
+        "progress": active_research[research_id]['progress'],
+        "metadata": {"phase": "termination"}
+    }
+    
+    active_research[research_id]['log'].append(log_entry)
+    
+    # Update the log in the database
+    cursor.execute('SELECT progress_log FROM research_history WHERE id = ?', (research_id,))
+    log_result = cursor.fetchone()
+    if log_result:
+        try:
+            current_log = json.loads(log_result[0])
+        except:
+            current_log = []
+        current_log.append(log_entry)
+        cursor.execute(
+            'UPDATE research_history SET progress_log = ? WHERE id = ?',
+            (json.dumps(current_log), research_id)
+        )
+    
+    conn.commit()
+    conn.close()
+    
+    # Emit a socket event for the termination request
+    try:
+        event_data = {
+            'status': 'terminating',
+            'message': 'Research termination requested by user'
+        }
+        
+        socketio.emit(f'research_progress_{research_id}', event_data)
+        
+        if research_id in socket_subscriptions and socket_subscriptions[research_id]:
+            for sid in socket_subscriptions[research_id]:
+                try:
+                    socketio.emit(
+                        f'research_progress_{research_id}', 
+                        event_data,
+                        room=sid
+                    )
+                except Exception as err:
+                    print(f"Error emitting to subscriber {sid}: {str(err)}")
+                    
+    except Exception as socket_error:
+        print(f"Socket emit error (non-critical): {str(socket_error)}")
+    
+    return jsonify({'status': 'success', 'message': 'Research termination requested'})
+
+@research_bp.route('/api/research/<int:research_id>/delete', methods=['DELETE'])
+def delete_research(research_id):
+    """Delete a research record"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    
+    # First check if the research exists and is not in progress
+    cursor.execute('SELECT status, report_path FROM research_history WHERE id = ?', (research_id,))
+    result = cursor.fetchone()
+    
+    if not result:
+        conn.close()
+        return jsonify({'status': 'error', 'message': 'Research not found'}), 404
+    
+    status, report_path = result
+    
+    # Don't allow deleting research in progress
+    if status == 'in_progress' and research_id in active_research:
+        conn.close()
+        return jsonify({
+            'status': 'error', 
+            'message': 'Cannot delete research that is in progress'
+        }), 400
+    
+    # Delete report file if it exists
+    if report_path and os.path.exists(report_path):
+        try:
+            os.remove(report_path)
+        except Exception as e:
+            print(f"Error removing report file: {str(e)}")
+    
+    # Delete the database record
+    cursor.execute('DELETE FROM research_history WHERE id = ?', (research_id,))
+    conn.commit()
+    conn.close()
+    
+    return jsonify({'status': 'success'})
+
+# Main settings page that links to specialized config pages
+@research_bp.route('/settings', methods=['GET'])
+def settings_page():
+    """Main settings dashboard with links to specialized config pages"""
+    return render_template('settings_dashboard.html')
+
+@research_bp.route('/settings/main', methods=['GET'])
+def main_config_page():
+    """Edit main configuration with search parameters"""
+    return render_template('main_config.html', main_file_path=MAIN_CONFIG_FILE)
+
+@research_bp.route('/settings/llm', methods=['GET'])
+def llm_config_page():
+    """Edit LLM configuration using raw file editor"""
+    return render_template('llm_config.html', llm_file_path=LLM_CONFIG_FILE)
+
+@research_bp.route('/settings/collections', methods=['GET']) 
+def collections_config_page():
+    """Edit local collections configuration using raw file editor"""
+    return render_template('collections_config.html', collections_file_path=LOCAL_COLLECTIONS_FILE)
+
+@research_bp.route('/settings/api_keys', methods=['GET'])
+def api_keys_config_page():
+    """Edit API keys configuration"""
+    # Get the secrets file path
+    secrets_file = CONFIG_DIR / ".secrets.toml"
+    
+    return render_template('api_keys_config.html', secrets_file_path=secrets_file)
+# Add to the imports section
+from local_deep_research.config import SEARCH_ENGINES_FILE
+
+# Add a new route for search engines configuration page
+@research_bp.route('/settings/search_engines', methods=['GET'])
+def search_engines_config_page():
+    """Edit search engines configuration using raw file editor"""
+    # Read the current config file
+    raw_config = ""
+    try:
+        with open(SEARCH_ENGINES_FILE, 'r') as f:
+            raw_config = f.read()
+    except Exception as e:
+        flash(f'Error reading search engines configuration: {str(e)}', 'error')
+        raw_config = "# Error reading configuration file"
+    
+    # Get list of engine names for display
+    engine_names = []
+    try:
+        from local_deep_research.web_search_engines.search_engines_config import SEARCH_ENGINES
+        engine_names = list(SEARCH_ENGINES.keys())
+        engine_names.sort()  # Alphabetical order
+    except Exception as e:
+        logger.error(f"Error getting engine names: {e}")
+    
+    return render_template('search_engines_config.html', 
+                          search_engines_file_path=SEARCH_ENGINES_FILE,
+                          raw_config=raw_config,
+                          engine_names=engine_names)
+
+# Add a route to save search engines configuration
+@research_bp.route('/api/save_search_engines_config', methods=['POST'])
+def save_search_engines_config():
+    try:
+        data = request.get_json()
+        raw_config = data.get('raw_config', '')
+        
+        # Validate TOML syntax
+        try:
+            toml.loads(raw_config)
+        except toml.TomlDecodeError as e:
+            return jsonify({'success': False, 'error': f'TOML syntax error: {str(e)}'})
+        
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(SEARCH_ENGINES_FILE), exist_ok=True)
+        
+        # Create a backup first
+        backup_path = f"{SEARCH_ENGINES_FILE}.bak"
+        if os.path.exists(SEARCH_ENGINES_FILE):
+            import shutil
+            shutil.copy2(SEARCH_ENGINES_FILE, backup_path)
+        
+        # Write new config
+        with open(SEARCH_ENGINES_FILE, 'w') as f:
+            f.write(raw_config)
+            
+        return jsonify({'success': True})
+    except Exception as e:
+        return jsonify({'success': False, 'error': str(e)})
+
+
+# API endpoint to save raw LLM config
+@research_bp.route('/api/save_llm_config', methods=['POST'])
+def save_llm_config():
+    try:
+        data = request.get_json()
+        raw_config = data.get('raw_config', '')
+        
+        # Validate Python syntax
+        try:
+            compile(raw_config, '<string>', 'exec')
+        except SyntaxError as e:
+            return jsonify({'success': False, 'error': f'Syntax error: {str(e)}'})
+        
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(LLM_CONFIG_FILE), exist_ok=True)
+        
+        # Create a backup first
+        backup_path = f"{LLM_CONFIG_FILE}.bak"
+        if os.path.exists(LLM_CONFIG_FILE):
+            import shutil
+            shutil.copy2(LLM_CONFIG_FILE, backup_path)
+        
+        # Write new config
+        with open(LLM_CONFIG_FILE, 'w') as f:
+            f.write(raw_config)
+            
+        return jsonify({'success': True})
+    except Exception as e:
+        return jsonify({'success': False, 'error': str(e)})
+
+# API endpoint to save raw collections config
+@research_bp.route('/api/save_collections_config', methods=['POST'])
+def save_collections_config():
+    try:
+        data = request.get_json()
+        raw_config = data.get('raw_config', '')
+        
+        # Validate TOML syntax
+        try:
+            toml.loads(raw_config)
+        except toml.TomlDecodeError as e:
+            return jsonify({'success': False, 'error': f'TOML syntax error: {str(e)}'})
+        
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(LOCAL_COLLECTIONS_FILE), exist_ok=True)
+        
+        # Create a backup first
+        backup_path = f"{LOCAL_COLLECTIONS_FILE}.bak"
+        if os.path.exists(LOCAL_COLLECTIONS_FILE):
+            import shutil
+            shutil.copy2(LOCAL_COLLECTIONS_FILE, backup_path)
+        
+        # Write new config
+        with open(LOCAL_COLLECTIONS_FILE, 'w') as f:
+            f.write(raw_config)
+        
+        # Also trigger a reload in the collections system
+        try:
+            load_local_collections(reload=True)
+        except Exception as reload_error:
+            return jsonify({'success': True, 'warning': f'Config saved, but error reloading: {str(reload_error)}'})
+            
+        return jsonify({'success': True})
+    except Exception as e:
+        return jsonify({'success': False, 'error': str(e)})
+
+# API endpoint to save raw main config 
+@research_bp.route('/api/save_main_config', methods=['POST'])
+def save_raw_main_config():
+    try:
+        data = request.get_json()
+        raw_config = data.get('raw_config', '')
+        
+        # Validate TOML syntax
+        try:
+            toml.loads(raw_config)
+        except toml.TomlDecodeError as e:
+            return jsonify({'success': False, 'error': f'TOML syntax error: {str(e)}'})
+        
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(MAIN_CONFIG_FILE), exist_ok=True)
+        
+        # Create a backup first
+        backup_path = f"{MAIN_CONFIG_FILE}.bak"
+        if os.path.exists(MAIN_CONFIG_FILE):
+            import shutil
+            shutil.copy2(MAIN_CONFIG_FILE, backup_path)
+        
+        # Write new config
+        with open(MAIN_CONFIG_FILE, 'w') as f:
+            f.write(raw_config)
+            
+        return jsonify({'success': True})
+    except Exception as e:
+        return jsonify({'success': False, 'error': str(e)})
+@research_bp.route('/raw_config')
+def get_raw_config():
+    """Return the raw configuration file content"""
+    try:
+        # Determine which config file to load based on a query parameter
+        config_type = request.args.get('type', 'main')
+        
+        if config_type == 'main':
+            config_path = os.path.join(app.config['CONFIG_DIR'], 'config.toml')
+            with open(config_path, 'r') as f:
+                return f.read()
+        elif config_type == 'llm':
+            config_path = os.path.join(app.config['CONFIG_DIR'], 'llm_config.py')
+            with open(config_path, 'r') as f:
+                return f.read()
+        elif config_type == 'collections':
+            config_path = os.path.join(app.config['CONFIG_DIR'], 'collections.toml')
+            with open(config_path, 'r') as f:
+                return f.read()
+        else:
+            return "Unknown configuration type", 400
+    except Exception as e:
+        return str(e), 500
+import os
+import subprocess
+import platform
+
+@research_bp.route('/open_file_location', methods=['POST'])
+def open_file_location():
+    file_path = request.form.get('file_path')
+    
+    if not file_path:
+        flash('No file path provided', 'error')
+        return redirect(url_for('research.settings_page'))
+    
+    # Get the directory containing the file
+    dir_path = os.path.dirname(os.path.abspath(file_path))
+    
+    # Open the directory in the file explorer
+    try:
+        if platform.system() == "Windows":
+            subprocess.Popen(f'explorer "{dir_path}"')
+        elif platform.system() == "Darwin":  # macOS
+            subprocess.Popen(["open", dir_path])
+        else:  # Linux
+            subprocess.Popen(["xdg-open", dir_path])
+        
+        flash(f'Opening folder: {dir_path}', 'success')
+    except Exception as e:
+        flash(f'Error opening folder: {str(e)}', 'error')
+    
+    # Redirect back to the settings page
+    if 'llm' in file_path:
+        return redirect(url_for('research.llm_config_page'))
+    elif 'collections' in file_path:
+        return redirect(url_for('research.collections_config_page'))
+    else:
+        return redirect(url_for('research.main_config_page'))
+# Register the blueprint
+app.register_blueprint(research_bp)
+
+# Also add the static route at the app level for compatibility
+@app.route('/static/<path:path>')
+def app_serve_static(path):
+    return send_from_directory(app.static_folder, path)
+
+# Add favicon route to prevent 404 errors
+@app.route('/favicon.ico')
+def favicon():
+    return send_from_directory(app.static_folder, 'favicon.ico', mimetype='image/x-icon')
+
+
+# Add this function to app.py
+def convert_debug_to_markdown(raw_text, query):
+    """
+    Convert the debug-formatted text to clean markdown.
+    
+    Args:
+        raw_text: The raw formatted findings with debug symbols
+        query: Original research query
+    
+    Returns:
+        Clean markdown formatted text
+    """
+    # If there's a "DETAILED FINDINGS:" section, extract everything after it
+    if "DETAILED FINDINGS:" in raw_text:
+        detailed_index = raw_text.index("DETAILED FINDINGS:")
+        content = raw_text[detailed_index + len("DETAILED FINDINGS:"):].strip()
+    else:
+        content = raw_text
+    
+    # Remove divider lines with === symbols
+    content = "\n".join([line for line in content.split("\n") 
+                        if not line.strip().startswith("===") and not line.strip() == "="*80])
+    
+    # If COMPLETE RESEARCH OUTPUT exists, remove that section
+    if "COMPLETE RESEARCH OUTPUT" in content:
+        content = content.split("COMPLETE RESEARCH OUTPUT")[0].strip()
+    
+    # Remove SEARCH QUESTIONS BY ITERATION section
+    if "SEARCH QUESTIONS BY ITERATION:" in content:
+        search_index = content.index("SEARCH QUESTIONS BY ITERATION:")
+        next_major_section = -1
+        for marker in ["DETAILED FINDINGS:", "COMPLETE RESEARCH:"]:
+            if marker in content[search_index:]:
+                marker_pos = content.index(marker, search_index)
+                if next_major_section == -1 or marker_pos < next_major_section:
+                    next_major_section = marker_pos
+        
+        if next_major_section != -1:
+            content = content[:search_index] + content[next_major_section:]
+        else:
+            # If no later section, just remove everything from SEARCH QUESTIONS onwards
+            content = content[:search_index].strip()
+    
+    return content.strip()
+def main():
+    """
+    Entry point for the web application when run as a command.
+    This function is needed for the package's entry point to work properly.
+    """
+
+    # Check for OpenAI availability but don't import it unless necessary
+    try:
+        import os
+        api_key = os.environ.get("OPENAI_API_KEY")
+        if api_key:
+            try:
+                # Only try to import if we have an API key
+                import openai
+                openai.api_key = api_key
+                OPENAI_AVAILABLE = True
+                print("OpenAI integration is available")
+            except ImportError:
+                print("OpenAI package not installed, integration disabled")
+        else:
+            print("OPENAI_API_KEY not found in environment variables, OpenAI integration disabled")
+    except Exception as e:
+        print(f"Error checking OpenAI availability: {e}")
+        
+
+    socketio.run(app, debug=True, host='0.0.0.0', port=5000, allow_unsafe_werkzeug=True)
+
+if __name__ == '__main__':
+    main()