local-deep-research 0.5.9__py3-none-any.whl → 0.6.1__py3-none-any.whl

local_deep_research/web/routes/api_routes.py

@@ -5,6 +5,7 @@ import requests
 from flask import Blueprint, current_app, jsonify, request
 
 from ...utilities.url_utils import normalize_url
+from ...utilities.db_utils import get_db_session
 from ..models.database import get_db_connection
 from ..routes.research_routes import active_research, termination_flags
 from ..services.research_service import (
@@ -17,12 +18,50 @@ from ..services.resource_service import (
     delete_resource,
     get_resources_for_research,
 )
+from ..services.settings_manager import SettingsManager
 
 # Create blueprint
 api_bp = Blueprint("api", __name__)
 logger = logging.getLogger(__name__)
 
 
+@api_bp.route("/settings/current-config", methods=["GET"])
+def get_current_config():
+    """Get the current configuration from database settings."""
+    try:
+        session = get_db_session()
+        settings_manager = SettingsManager(db_session=session)
+
+        config = {
+            "provider": settings_manager.get_setting(
+                "llm.provider", "Not configured"
+            ),
+            "model": settings_manager.get_setting(
+                "llm.model", "Not configured"
+            ),
+            "search_tool": settings_manager.get_setting(
+                "search.tool", "searxng"
+            ),
+            "iterations": settings_manager.get_setting("search.iterations", 8),
+            "questions_per_iteration": settings_manager.get_setting(
+                "search.questions_per_iteration", 5
+            ),
+            "search_strategy": settings_manager.get_setting(
+                "search.search_strategy", "focused_iteration"
+            ),
+        }
+
+        session.close()
+
+        return jsonify({"success": True, "config": config})
+
+    except Exception:
+        logger.exception("Error getting current config")
+        return jsonify(
+            {"success": False, "error": "An internal error occurred"}
+        ), 500
+
+
 # API Routes
 @api_bp.route("/start", methods=["POST"])
 def api_start_research():
@@ -53,7 +92,7 @@ def api_start_research():
         }
 
         cursor.execute(
-            "INSERT INTO research_history (query, mode, status, created_at, progress_log, metadata) VALUES (?, ?, ?, ?, ?, ?)",
+            "INSERT INTO research_history (query, mode, status, created_at, progress_log, research_meta) VALUES (?, ?, ?, ?, ?, ?)",
             (
                 query,
                 mode,
@@ -100,39 +139,31 @@ def api_research_status(research_id):
     Get the status of a research process
     """
     try:
-        conn = get_db_connection()
-        cursor = conn.cursor()
-        cursor.execute(
-            "SELECT status, progress, completed_at, report_path, metadata FROM research_history WHERE id = ?",
-            (research_id,),
-        )
-        result = cursor.fetchone()
-        conn.close()
-
-        if result is None:
-            return jsonify({"error": "Research not found"}), 404
+        from ..database.models import ResearchHistory
+
+        db_session = get_db_session()
+        with db_session:
+            research = (
+                db_session.query(ResearchHistory)
+                .filter_by(id=research_id)
+                .first()
+            )
 
-        status, progress, completed_at, report_path, metadata_str = result
+            if research is None:
+                return jsonify({"error": "Research not found"}), 404
 
-        # Parse metadata if it exists
-        metadata = {}
-        if metadata_str:
-            try:
-                metadata = json.loads(metadata_str)
-            except json.JSONDecodeError:
-                logger.warning(
-                    f"Invalid JSON in metadata for research {research_id}"
-                )
+            # Get metadata
+            metadata = research.research_meta or {}
 
-        return jsonify(
-            {
-                "status": status,
-                "progress": progress,
-                "completed_at": completed_at,
-                "report_path": report_path,
-                "metadata": metadata,
-            }
-        )
+            return jsonify(
+                {
+                    "status": research.status,
+                    "progress": research.progress,
+                    "completed_at": research.completed_at,
+                    "report_path": research.report_path,
+                    "metadata": metadata,
+                }
+            )
     except Exception as e:
         logger.error(f"Error getting research status: {str(e)}")
         return jsonify({"status": "error", "message": str(e)}), 500

local_deep_research/web/routes/history_routes.py

@@ -4,6 +4,7 @@ import traceback
 from pathlib import Path
 
 from flask import Blueprint, jsonify, make_response
+from ..utils.templates import render_template_with_defaults
 
 from ..models.database import (
     get_db_connection,
@@ -17,7 +18,7 @@ from ..services.research_service import get_research_strategy
 logger = logging.getLogger(__name__)
 
 # Create a Blueprint for the history routes
-history_bp = Blueprint("history", __name__)
+history_bp = Blueprint("history", __name__, url_prefix="/history")
 
 
 def resolve_report_path(report_path: str) -> Path:
@@ -34,9 +35,15 @@ def resolve_report_path(report_path: str) -> Path:
     return project_root / path
 
 
-@history_bp.route("/history", methods=["GET"])
+@history_bp.route("/")
+def history_page():
+    """Render the history page"""
+    return render_template_with_defaults("pages/history.html")
+
+
+@history_bp.route("/api", methods=["GET"])
 def get_history():
-    """Get the research history"""
+    """Get the research history JSON data"""
     try:
         conn = get_db_connection()
         conn.row_factory = lambda cursor, row: {
@@ -73,8 +80,8 @@ def get_history():
                 item["duration_seconds"] = None
             if "report_path" not in item:
                 item["report_path"] = None
-            if "metadata" not in item:
-                item["metadata"] = "{}"
+            if "research_meta" not in item:
+                item["research_meta"] = "{}"
             if "progress_log" not in item:
                 item["progress_log"] = "[]"
 
@@ -287,7 +294,7 @@ def get_report(research_id):
         }
 
         # Also include any stored metadata
-        stored_metadata = json.loads(result.get("metadata", "{}"))
+        stored_metadata = json.loads(result.get("research_meta", "{}"))
         if stored_metadata and isinstance(stored_metadata, dict):
             enhanced_metadata.update(stored_metadata)
 

local_deep_research/web/routes/metrics_routes.py

@@ -11,7 +11,13 @@ from ...metrics.db_models import ResearchRating, TokenUsage
 from ...metrics.query_utils import get_time_filter_condition
 from ...metrics.search_tracker import get_search_tracker
 from ...utilities.db_utils import get_db_session
-from ..database.models import Research, ResearchStrategy
+from ...web_search_engines.rate_limiting import get_tracker
+from ..database.models import (
+    Research,
+    ResearchStrategy,
+    RateLimitAttempt,
+    RateLimitEstimate,
+)
 from ..utils.templates import render_template_with_defaults
 
 # Create a Blueprint for metrics
@@ -296,6 +302,161 @@ def get_strategy_analytics(period="30d"):
         }
 
 
+def get_rate_limiting_analytics(period="30d"):
+    """Get rate limiting analytics for the specified period."""
+    try:
+        session = get_db_session()
+
+        # Calculate date range
+        days_map = {"7d": 7, "30d": 30, "90d": 90, "365d": 365, "all": None}
+        days = days_map.get(period, 30)
+
+        try:
+            # Get current rate limit estimates
+            estimates = session.query(RateLimitEstimate).all()
+
+            # Get recent attempts for analytics
+            attempts_query = session.query(RateLimitAttempt)
+            if days:
+                cutoff_timestamp = (
+                    datetime.now() - timedelta(days=days)
+                ).timestamp()
+                attempts_query = attempts_query.filter(
+                    RateLimitAttempt.timestamp >= cutoff_timestamp
+                )
+
+            attempts = attempts_query.all()
+
+            # Calculate analytics
+            total_attempts = len(attempts)
+            successful_attempts = len([a for a in attempts if a.success])
+            failed_attempts = total_attempts - successful_attempts
+
+            # Engine-specific analytics
+            engine_stats = {}
+            for estimate in estimates:
+                engine_attempts = [
+                    a for a in attempts if a.engine_type == estimate.engine_type
+                ]
+                engine_success = len([a for a in engine_attempts if a.success])
+                engine_total = len(engine_attempts)
+
+                engine_stats[estimate.engine_type] = {
+                    "base_wait_seconds": round(estimate.base_wait_seconds, 2),
+                    "min_wait_seconds": round(estimate.min_wait_seconds, 2),
+                    "max_wait_seconds": round(estimate.max_wait_seconds, 2),
+                    "success_rate": round(estimate.success_rate * 100, 1),
+                    "total_attempts": estimate.total_attempts,
+                    "recent_attempts": engine_total,
+                    "recent_success_rate": round(
+                        (engine_success / engine_total * 100)
+                        if engine_total > 0
+                        else 0,
+                        1,
+                    ),
+                    "last_updated": datetime.fromtimestamp(
+                        estimate.last_updated
+                    ).strftime("%Y-%m-%d %H:%M:%S"),
+                    "status": "healthy"
+                    if estimate.success_rate > 0.8
+                    else "degraded"
+                    if estimate.success_rate > 0.5
+                    else "poor",
+                }
+
+            # Overall success rate trend
+            success_rate = round(
+                (successful_attempts / total_attempts * 100)
+                if total_attempts > 0
+                else 0,
+                1,
+            )
+
+            # Rate limiting events (failures)
+            rate_limit_events = len(
+                [
+                    a
+                    for a in attempts
+                    if not a.success and a.error_type == "RateLimitError"
+                ]
+            )
+
+            # Calculate average wait times
+            if attempts:
+                avg_wait_time = sum(a.wait_time for a in attempts) / len(
+                    attempts
+                )
+                successful_wait_times = [
+                    a.wait_time for a in attempts if a.success
+                ]
+                avg_successful_wait = (
+                    sum(successful_wait_times) / len(successful_wait_times)
+                    if successful_wait_times
+                    else 0
+                )
+            else:
+                avg_wait_time = 0
+                avg_successful_wait = 0
+
+            return {
+                "rate_limiting_analytics": {
+                    "total_attempts": total_attempts,
+                    "successful_attempts": successful_attempts,
+                    "failed_attempts": failed_attempts,
+                    "success_rate": success_rate,
+                    "rate_limit_events": rate_limit_events,
+                    "avg_wait_time": round(avg_wait_time, 2),
+                    "avg_successful_wait": round(avg_successful_wait, 2),
+                    "engine_stats": engine_stats,
+                    "total_engines_tracked": len(estimates),
+                    "healthy_engines": len(
+                        [
+                            s
+                            for s in engine_stats.values()
+                            if s["status"] == "healthy"
+                        ]
+                    ),
+                    "degraded_engines": len(
+                        [
+                            s
+                            for s in engine_stats.values()
+                            if s["status"] == "degraded"
+                        ]
+                    ),
+                    "poor_engines": len(
+                        [
+                            s
+                            for s in engine_stats.values()
+                            if s["status"] == "poor"
+                        ]
+                    ),
+                }
+            }
+
+        finally:
+            session.close()
+
+    except Exception as e:
+        logger.exception(f"Error getting rate limiting analytics: {e}")
+        return {
+            "rate_limiting_analytics": {
+                "total_attempts": 0,
+                "successful_attempts": 0,
+                "failed_attempts": 0,
+                "success_rate": 0,
+                "rate_limit_events": 0,
+                "avg_wait_time": 0,
+                "avg_successful_wait": 0,
+                "engine_stats": {},
+                "total_engines_tracked": 0,
+                "healthy_engines": 0,
+                "degraded_engines": 0,
+                "poor_engines": 0,
+                "error": "An internal error occurred while processing the request.",
+            }
+        }
+
+
 @metrics_bp.route("/")
 def metrics_dashboard():
     """Render the metrics dashboard page."""
@@ -355,11 +516,15 @@ def api_metrics():
         # Get strategy analytics
         strategy_data = get_strategy_analytics(period)
 
+        # Get rate limiting analytics
+        rate_limiting_data = get_rate_limiting_analytics(period)
+
         # Combine metrics
         combined_metrics = {
             **token_metrics,
             **search_metrics,
             **strategy_data,
+            **rate_limiting_data,
             "user_satisfaction": user_satisfaction,
         }
 
@@ -384,6 +549,80 @@ def api_metrics():
         )
 
 
+@metrics_bp.route("/api/rate-limiting")
+def api_rate_limiting_metrics():
+    """Get detailed rate limiting metrics."""
+    try:
+        period = request.args.get("period", "30d")
+        rate_limiting_data = get_rate_limiting_analytics(period)
+
+        return jsonify(
+            {"status": "success", "data": rate_limiting_data, "period": period}
+        )
+    except Exception as e:
+        logger.exception(f"Error getting rate limiting metrics: {e}")
+        return jsonify(
+            {
+                "status": "error",
+                "message": "Failed to retrieve rate limiting metrics",
+            }
+        ), 500
+
+
+@metrics_bp.route("/api/rate-limiting/current")
+def api_current_rate_limits():
+    """Get current rate limit estimates for all engines."""
+    try:
+        tracker = get_tracker()
+        stats = tracker.get_stats()
+
+        current_limits = []
+        for stat in stats:
+            (
+                engine_type,
+                base_wait,
+                min_wait,
+                max_wait,
+                last_updated,
+                total_attempts,
+                success_rate,
+            ) = stat
+            current_limits.append(
+                {
+                    "engine_type": engine_type,
+                    "base_wait_seconds": round(base_wait, 2),
+                    "min_wait_seconds": round(min_wait, 2),
+                    "max_wait_seconds": round(max_wait, 2),
+                    "success_rate": round(success_rate * 100, 1),
+                    "total_attempts": total_attempts,
+                    "last_updated": datetime.fromtimestamp(
+                        last_updated
+                    ).strftime("%Y-%m-%d %H:%M:%S"),
+                    "status": "healthy"
+                    if success_rate > 0.8
+                    else "degraded"
+                    if success_rate > 0.5
+                    else "poor",
+                }
+            )
+
+        return jsonify(
+            {
+                "status": "success",
+                "current_limits": current_limits,
+                "timestamp": datetime.now().isoformat(),
+            }
+        )
+    except Exception as e:
+        logger.exception(f"Error getting current rate limits: {e}")
+        return jsonify(
+            {
+                "status": "error",
+                "message": "Failed to retrieve current rate limits",
+            }
+        ), 500
+
+
 @metrics_bp.route("/api/metrics/research/<int:research_id>")
 def api_research_metrics(research_id):
     """Get metrics for a specific research."""
@@ -1030,18 +1269,39 @@ def api_cost_analytics():
             if time_condition is not None:
                 query = query.filter(time_condition)
 
-            usage_records = query.all()
+            # First check if we have any records to avoid expensive queries
+            record_count = query.count()
 
-            if not usage_records:
+            if record_count == 0:
                 return jsonify(
                     {
                         "status": "success",
                         "period": period,
-                        "total_cost": 0.0,
+                        "overview": {
+                            "total_cost": 0.0,
+                            "total_tokens": 0,
+                            "prompt_tokens": 0,
+                            "completion_tokens": 0,
+                        },
+                        "top_expensive_research": [],
+                        "research_count": 0,
                         "message": "No token usage data found for this period",
                     }
                 )
 
+            # If we have too many records, limit to recent ones to avoid timeout
+            if record_count > 1000:
+                logger.warning(
+                    f"Large dataset detected ({record_count} records), limiting to recent 1000 for performance"
+                )
+                usage_records = (
+                    query.order_by(TokenUsage.timestamp.desc())
+                    .limit(1000)
+                    .all()
+                )
+            else:
+                usage_records = query.all()
+
             # Convert to dict format
             usage_data = []
             for record in usage_records:

local_deep_research/web/routes/research_routes.py

@@ -29,18 +29,12 @@ from ..database.models import ResearchHistory, ResearchLog
 from ...utilities.db_utils import get_db_session
 
 # Create a Blueprint for the research application
-research_bp = Blueprint("research", __name__, url_prefix="/research")
+research_bp = Blueprint("research", __name__)
 
 # Output directory for research results
 OUTPUT_DIR = "research_outputs"
 
 
-# Route for index page - redirection
-@research_bp.route("/")
-def index():
-    return render_template_with_defaults("pages/research.html")
-
-
 # Add the missing static file serving route
 @research_bp.route("/static/<path:path>")
 def serve_static(path):
@@ -325,7 +319,7 @@ def terminate_research(research_id):
     active_research[research_id]["log"].append(log_entry)
 
     # Add to database log
-    logger.log("milestone", "Research ended: {}", termination_message)
+    logger.log("MILESTONE", f"Research ended: {termination_message}")
 
     # Update the log in the database (old way for backward compatibility)
     cursor.execute(
@@ -749,7 +743,7 @@ def get_research_status(research_id):
     conn = get_db_connection()
     cursor = conn.cursor()
     cursor.execute(
-        "SELECT status, progress, completed_at, report_path, metadata FROM research_history WHERE id = ?",
+        "SELECT status, progress, completed_at, report_path, research_meta FROM research_history WHERE id = ?",
         (research_id,),
     )
     result = cursor.fetchone()
@@ -837,13 +831,46 @@ def get_research_status(research_id):
     if error_info:
         metadata["error_info"] = error_info
 
+    # Get the latest milestone log for this research
+    latest_milestone = None
+    try:
+        db_session = get_db_session()
+        with db_session:
+            milestone_log = (
+                db_session.query(ResearchLog)
+                .filter_by(research_id=research_id, level="MILESTONE")
+                .order_by(ResearchLog.timestamp.desc())
+                .first()
+            )
+            if milestone_log:
+                latest_milestone = {
+                    "message": milestone_log.message,
+                    "time": milestone_log.timestamp.isoformat()
+                    if milestone_log.timestamp
+                    else None,
+                    "type": "MILESTONE",
+                }
+                logger.debug(
+                    f"Found latest milestone for research {research_id}: {milestone_log.message}"
+                )
+            else:
+                logger.debug(
+                    f"No milestone logs found for research {research_id}"
+                )
+    except Exception as e:
+        logger.warning(f"Error fetching latest milestone: {str(e)}")
+
     conn.close()
-    return jsonify(
-        {
-            "status": status,
-            "progress": progress,
-            "completed_at": completed_at,
-            "report_path": report_path,
-            "metadata": metadata,
-        }
-    )
+    response_data = {
+        "status": status,
+        "progress": progress,
+        "completed_at": completed_at,
+        "report_path": report_path,
+        "metadata": metadata,
+    }
+
+    # Include latest milestone as a log_entry for frontend compatibility
+    if latest_milestone:
+        response_data["log_entry"] = latest_milestone
+
+    return jsonify(response_data)