local-deep-research 0.5.9__py3-none-any.whl → 0.6.0__py3-none-any.whl

local_deep_research/web/templates/pages/metrics.html

@@ -682,6 +682,77 @@
             </div>
         </div>
 
+        <!-- Rate Limiting Analytics -->
+        <div class="card" style="margin-top: 2rem;">
+            <div class="card-header">
+                <h2><i class="fas fa-clock"></i> Rate Limiting Analytics</h2>
+            </div>
+            <div class="card-content">
+                <!-- Rate Limiting Overview Cards -->
+                <div class="metrics-grid" style="margin-bottom: 2rem;">
+                    <div class="metric-card">
+                        <div class="metric-icon">
+                            <i class="fas fa-stopwatch"></i>
+                        </div>
+                        <div class="metric-label tooltip" data-tooltip="Percentage of retry attempts that succeeded after rate limiting">
+                            Rate Limit Success Rate
+                            <i class="fas fa-info-circle info-icon"></i>
+                        </div>
+                        <div class="metric-value" id="rate-limit-success-rate">0%</div>
+                    </div>
+
+                    <div class="metric-card">
+                        <div class="metric-icon">
+                            <i class="fas fa-exclamation-triangle"></i>
+                        </div>
+                        <div class="metric-label tooltip" data-tooltip="Number of times search engines were rate limited">
+                            Rate Limit Events
+                            <i class="fas fa-info-circle info-icon"></i>
+                        </div>
+                        <div class="metric-value" id="rate-limit-events">0</div>
+                    </div>
+
+                    <div class="metric-card">
+                        <div class="metric-icon">
+                            <i class="fas fa-hourglass-half"></i>
+                        </div>
+                        <div class="metric-label tooltip" data-tooltip="Average time waited before retrying after rate limits">
+                            Avg Wait Time
+                            <i class="fas fa-info-circle info-icon"></i>
+                        </div>
+                        <div class="metric-value" id="avg-wait-time">0s</div>
+                    </div>
+
+                    <div class="metric-card">
+                        <div class="metric-icon">
+                            <i class="fas fa-server"></i>
+                        </div>
+                        <div class="metric-label tooltip" data-tooltip="Number of search engines being tracked for rate limiting">
+                            Engines Tracked
+                            <i class="fas fa-info-circle info-icon"></i>
+                        </div>
+                        <div class="metric-value" id="engines-tracked">0</div>
+                    </div>
+                </div>
+
+                <!-- Engine Status Grid -->
+                <div style="margin-bottom: 2rem;">
+                    <h3><i class="fas fa-tachometer-alt"></i> Search Engine Status</h3>
+                    <div id="engine-status-grid" style="display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 1rem; margin-top: 1rem;">
+                        <!-- Populated dynamically -->
+                    </div>
+                </div>
+
+                <!-- Rate Limiting Chart -->
+                <div style="margin-top: 2rem;">
+                    <h3><i class="fas fa-chart-line"></i> Rate Limiting Activity Over Time</h3>
+                    <div class="chart-container" style="height: 300px;">
+                        <canvas id="rate-limiting-chart"></canvas>
+                    </div>
+                </div>
+            </div>
+        </div>
+
         <!-- Developer Insights -->
         <div class="card" style="margin-top: 2rem;">
             <div class="card-header">
@@ -817,7 +888,6 @@
 <script>
 // Metrics Dashboard JavaScript
 (function() {
-    console.log("=== METRICS SCRIPT STARTED ===");
 
     let metricsData = null;
     let tokenChart = null;
@@ -871,40 +941,30 @@
 
     // Load metrics data for current time period
     async function loadMetrics(period = currentPeriod) {
-        console.log('=== STARTING LOADMETRICS ===');
-        console.log('Period:', period, 'Mode:', currentMode);
 
         try {
             // Show loading state
-            console.log('Setting loading state...');
             document.getElementById('loading').style.display = 'block';
             document.getElementById('metrics-content').style.display = 'none';
             document.getElementById('error').style.display = 'none';
 
-            console.log('Making basic API call...');
             const basicResponse = await fetch(`/metrics/api/metrics?period=${period}&mode=${currentMode}`);
-            console.log('Basic response status:', basicResponse.status);
 
             if (!basicResponse.ok) {
                 throw new Error(`Basic API failed: ${basicResponse.status}`);
             }
 
             const basicData = await basicResponse.json();
-            console.log('Basic data received:', basicData);
 
-            console.log('Making enhanced API call...');
             const enhancedResponse = await fetch(`/metrics/api/metrics/enhanced?period=${period}&mode=${currentMode}`);
-            console.log('Enhanced response status:', enhancedResponse.status);
 
             if (!enhancedResponse.ok) {
                 throw new Error(`Enhanced API failed: ${enhancedResponse.status}`);
             }
 
             const enhancedData = await enhancedResponse.json();
-            console.log('Enhanced data received:', enhancedData);
 
             if (basicData.status === 'success') {
-                console.log('Basic data success, setting metricsData');
                 metricsData = basicData.metrics;
 
                 // Check if we have any data at all
@@ -913,15 +973,12 @@
                                (metricsData.by_model && metricsData.by_model.length > 0) ||
                                (metricsData.recent_researches && metricsData.recent_researches.length > 0);
 
-                console.log('Has data check:', hasData, 'tokens:', metricsData.total_tokens, 'researches:', metricsData.total_researches);
 
                 if (hasData) {
-                    console.log('Displaying metrics...');
                     // We have data, display normally
                     displayMetrics();
 
                     if (enhancedData.status === 'success') {
-                        console.log('Displaying enhanced metrics...');
                         displayEnhancedMetrics(enhancedData.metrics);
                         createTimeSeriesChart(enhancedData.metrics.time_series_data);
                         createSearchActivityChart(enhancedData.metrics.search_time_series);
@@ -932,40 +989,35 @@
                     // Load cost analytics
                     loadCostAnalytics(period);
 
+                    // Load rate limiting analytics
+                    loadRateLimitingAnalytics(period);
+
                     document.getElementById('loading').style.display = 'none';
                     document.getElementById('metrics-content').style.display = 'block';
-                    console.log('Metrics display completed');
-                } else {
-                    console.log('No data, showing empty state');
+                    } else {
                     // No data yet, show empty state with helpful messages
                     showEmptyState();
 
                     // Still try to display enhanced metrics in case there's some enhanced data
                     if (enhancedData.status === 'success') {
-                        console.log('Displaying enhanced metrics in empty state...');
                         displayEnhancedMetrics(enhancedData.metrics);
                         // Re-setup tooltips for any new content
                         setTimeout(setupTooltipPositioning, 100);
                     }
                 }
             } else {
-                console.log('Basic data failed, showing error');
                 showError();
             }
         } catch (error) {
-            console.error('Error loading metrics:', error);
             showError();
         }
     }
 
     // Display metrics on the page
     function displayMetrics() {
-        console.log('displayMetrics called with metricsData:', metricsData);
 
         // Update summary cards
-        console.log('Updating total tokens:', metricsData.total_tokens);
         document.getElementById('total-tokens').textContent = formatNumber(metricsData.total_tokens);
-        console.log('Updating total researches:', metricsData.total_researches);
         document.getElementById('total-researches').textContent = formatNumber(metricsData.total_researches);
 
         // Update token breakdown details
@@ -984,15 +1036,12 @@
         document.getElementById('success-rate').textContent = '0%';
 
         // Update user satisfaction rating
-        console.log('User satisfaction data:', metricsData.user_satisfaction);
         if (metricsData.user_satisfaction && metricsData.user_satisfaction.avg_rating !== null) {
             const rating = metricsData.user_satisfaction.avg_rating;
             const stars = '⭐'.repeat(Math.floor(rating));
             const displayText = `${rating} ${stars}`;
-            console.log('Setting user rating to:', displayText);
             document.getElementById('avg-user-rating').textContent = displayText;
         } else {
-            console.log('No user satisfaction data available');
             document.getElementById('avg-user-rating').textContent = '-';
         }
 
@@ -1120,7 +1169,7 @@
                 <div class="research-tokens">${formatNumber(research.tokens)} tokens</div>
             `;
             item.onclick = () => {
-                window.location.href = `/research/details/${research.id}`;
+                window.location.href = `/details/${research.id}`;
             };
             container.appendChild(item);
         });
@@ -1194,8 +1243,6 @@
         // Get strategy data from metricsData (added by our API)
         const strategyData = metricsData.strategy_analytics;
 
-        // Log strategy data for debugging
-        console.log('Strategy data received:', strategyData);
 
         if (strategyData && strategyData.strategy_usage && strategyData.strategy_usage.length > 0) {
             strategyData.strategy_usage.forEach(strategy => {
@@ -1219,7 +1266,7 @@
             noDataMsg.style.color = 'var(--text-secondary)';
             noDataMsg.style.padding = '1rem';
 
-            let message = 'No strategy data yet. <a href="/research/" style="color: var(--accent-tertiary);">Start a research</a> to track strategies!';
+            let message = 'No strategy data yet. <a href="/" style="color: var(--accent-tertiary);">Start a research</a> to track strategies!';
 
             // Show different message based on what data we have
             if (strategyData) {
@@ -1285,7 +1332,7 @@
             noDataMsg.style.textAlign = 'center';
             noDataMsg.style.color = 'var(--text-secondary)';
             noDataMsg.style.padding = '1rem';
-            noDataMsg.innerHTML = 'No ratings yet. <a href="/research/" style="color: var(--accent-tertiary);">Start a research</a> and rate it!';
+            noDataMsg.innerHTML = 'No ratings yet. <a href="/" style="color: var(--accent-tertiary);">Start a research</a> and rate it!';
             ratingContainer.appendChild(noDataMsg);
         }
 
@@ -1322,7 +1369,7 @@
             // Add click handler to navigate to research details
             if (item.research_id) {
                 row.onclick = () => {
-                    window.location.href = `/research/details/${item.research_id}`;
+                    window.location.href = `/details/${item.research_id}`;
                 };
                 row.title = `Click to view research details`;
             }
@@ -1651,26 +1698,20 @@
     // Load cost analytics data
     async function loadCostAnalytics(period = currentPeriod) {
         try {
-            console.log('Loading cost analytics for period:', period);
-
             const response = await fetch(`/metrics/api/cost-analytics?period=${period}`);
             if (!response.ok) {
-                console.warn('Cost analytics API failed:', response.status);
                 displayCostData(null);
                 return;
             }
 
             const data = await response.json();
-            console.log('Cost analytics data received:', data);
 
             if (data.status === 'success') {
                 displayCostData(data);
             } else {
-                console.warn('Cost analytics returned error:', data.message);
                 displayCostData(null);
             }
         } catch (error) {
-            console.error('Error loading cost analytics:', error);
             displayCostData(null);
         }
     }
@@ -1700,7 +1741,133 @@
         document.getElementById('total-output-cost').textContent = formatCurrency(overview.completion_cost);
         document.getElementById('total-cost-breakdown').textContent = formatCurrency(overview.total_cost);
 
-        console.log('Cost data displayed successfully');
+    }
+
+    // Load rate limiting analytics
+    async function loadRateLimitingAnalytics(period = currentPeriod) {
+        try {
+            const response = await fetch(`/metrics/api/rate-limiting?period=${period}`);
+            if (!response.ok) {
+                displayRateLimitingData(null);
+                return;
+            }
+
+            const data = await response.json();
+
+            if (data.status === 'success') {
+                displayRateLimitingData(data.data);
+            } else {
+                displayRateLimitingData(null);
+            }
+        } catch (error) {
+            displayRateLimitingData(null);
+        }
+    }
+
+    // Display rate limiting data in the dashboard
+    function displayRateLimitingData(rateLimitData) {
+        if (!rateLimitData || !rateLimitData.rate_limiting_analytics) {
+            // No rate limiting data available, show zeros
+            document.getElementById('rate-limit-success-rate').textContent = '0%';
+            document.getElementById('rate-limit-events').textContent = '0';
+            document.getElementById('avg-wait-time').textContent = '0s';
+            document.getElementById('engines-tracked').textContent = '0';
+
+            // Clear engine status grid
+            const statusGrid = document.getElementById('engine-status-grid');
+            statusGrid.innerHTML = '<div style="grid-column: 1 / -1; text-align: center; padding: 2rem; color: var(--text-secondary);"><i class="fas fa-clock fa-2x" style="margin-bottom: 1rem;"></i><p>No rate limiting data yet</p><p style="font-size: 0.875rem;">Rate limiting will appear after search engines are used</p></div>';
+            return;
+        }
+
+        const analytics = rateLimitData.rate_limiting_analytics;
+
+        // Update overview cards
+        document.getElementById('rate-limit-success-rate').textContent = `${analytics.success_rate}%`;
+        document.getElementById('rate-limit-events').textContent = formatNumber(analytics.rate_limit_events);
+        document.getElementById('avg-wait-time').textContent = `${analytics.avg_wait_time}s`;
+        document.getElementById('engines-tracked').textContent = formatNumber(analytics.total_engines_tracked);
+
+        // Update engine status grid
+        const statusGrid = document.getElementById('engine-status-grid');
+        statusGrid.innerHTML = '';
+
+        if (Object.keys(analytics.engine_stats).length === 0) {
+            statusGrid.innerHTML = '<div style="grid-column: 1 / -1; text-align: center; padding: 2rem; color: var(--text-secondary);"><i class="fas fa-clock fa-2x" style="margin-bottom: 1rem;"></i><p>No engine data yet</p><p style="font-size: 0.875rem;">Search engine metrics will appear after rate limiting occurs</p></div>';
+        } else {
+            Object.entries(analytics.engine_stats).forEach(([engineType, stats]) => {
+                const statusCard = document.createElement('div');
+                statusCard.className = 'metric-card';
+                statusCard.style.borderLeft = `4px solid ${getStatusColor(stats.status)}`;
+
+                statusCard.innerHTML = `
+                    <div class="metric-header" style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 1rem;">
+                        <h4 style="margin: 0; color: var(--text-primary);">${engineType.replace('SearchEngine', '')}</h4>
+                        <span class="status-badge" style="
+                            background: ${getStatusColor(stats.status)};
+                            color: white;
+                            padding: 2px 8px;
+                            border-radius: 12px;
+                            font-size: 0.75rem;
+                            font-weight: 600;
+                            text-transform: uppercase;
+                        ">${stats.status}</span>
+                    </div>
+                    <div class="engine-stats-grid" style="display: grid; grid-template-columns: 1fr 1fr; gap: 0.5rem; font-size: 0.875rem;">
+                        <div>
+                            <div style="color: var(--text-secondary);">Base Wait:</div>
+                            <div style="font-weight: 600;">${stats.base_wait_seconds}s</div>
+                        </div>
+                        <div>
+                            <div style="color: var(--text-secondary);">Success Rate:</div>
+                            <div style="font-weight: 600; color: ${stats.success_rate > 80 ? '#4CAF50' : stats.success_rate > 50 ? '#FF9800' : '#F44336'};">${stats.success_rate}%</div>
+                        </div>
+                        <div>
+                            <div style="color: var(--text-secondary);">Total Attempts:</div>
+                            <div style="font-weight: 600;">${formatNumber(stats.total_attempts)}</div>
+                        </div>
+                        <div>
+                            <div style="color: var(--text-secondary);">Last Updated:</div>
+                            <div style="font-weight: 600;">${formatTime(stats.last_updated)}</div>
+                        </div>
+                    </div>
+                    <div style="margin-top: 0.75rem; font-size: 0.75rem; color: var(--text-secondary);">
+                        Range: ${stats.min_wait_seconds}s - ${stats.max_wait_seconds}s
+                    </div>
+                `;
+
+                statusGrid.appendChild(statusCard);
+            });
+        }
+
+    }
+
+    // Get status color based on engine health
+    function getStatusColor(status) {
+        switch (status) {
+            case 'healthy': return '#4CAF50';
+            case 'degraded': return '#FF9800';
+            case 'poor': return '#F44336';
+            default: return '#9E9E9E';
+        }
+    }
+
+    // Format time for display
+    function formatTime(timeString) {
+        try {
+            const date = new Date(timeString);
+            const now = new Date();
+            const diffHours = (now - date) / (1000 * 60 * 60);
+
+            if (diffHours < 1) {
+                return 'Just now';
+            } else if (diffHours < 24) {
+                return `${Math.floor(diffHours)}h ago`;
+            } else {
+                return `${Math.floor(diffHours / 24)}d ago`;
+            }
+        } catch {
+            return timeString;
+        }
     }
 
     // Show error message
@@ -1736,6 +1903,12 @@
         document.getElementById('total-output-cost').textContent = '-';
         document.getElementById('total-cost-breakdown').textContent = '-';
 
+        // Reset rate limiting metrics
+        document.getElementById('rate-limit-success-rate').textContent = '0%';
+        document.getElementById('rate-limit-events').textContent = '0';
+        document.getElementById('avg-wait-time').textContent = '0s';
+        document.getElementById('engines-tracked').textContent = '0';
+
         // Show helpful message in charts
         showEmptyChartsAndTables();
     }

local_deep_research/web/templates/pages/research.html

@@ -81,7 +81,7 @@
                         <!-- Model Provider Selection -->
                         <div class="form-group half">
                             <label for="model_provider">Model Provider</label>
-                            <select id="model_provider" name="model_provider" class="form-control">
+                            <select id="model_provider" name="model_provider" class="form-control" data-initial-value="{{ settings.llm_provider }}">
                                 <!-- Will be populated dynamically -->
                                 <option value="">Loading providers...</option>
                             </select>
@@ -91,7 +91,7 @@
                         <!-- Custom Endpoint (hidden by default) -->
                         <div class="form-group half" id="endpoint_container" style="display: none;">
                             <label for="custom_endpoint">Custom Endpoint</label>
-                            <input type="text" id="custom_endpoint" name="custom_endpoint" class="form-control" placeholder="https://your-endpoint-url/v1">
+                            <input type="text" id="custom_endpoint" name="custom_endpoint" class="form-control" placeholder="https://your-endpoint-url/v1" value="{{ settings.llm_openai_endpoint_url }}">
                             <span class="input-help">Enter the OpenAI-compatible API endpoint URL</span>
                         </div>
                     </div>
@@ -106,7 +106,8 @@
                                 label="Language Model",
                                 help_text="Select or enter a custom model name",
                                 show_refresh=True,
-                                refresh_aria_label="Refresh model list"
+                                refresh_aria_label="Refresh model list",
+                                data_initial_value=settings.llm_model
                             ) }}
                         </div>
 
@@ -119,7 +120,8 @@
                                 label="Search Engine",
                                 help_text="Select the search engine to use for research",
                                 show_refresh=True,
-                                refresh_aria_label="Refresh search engine list"
+                                refresh_aria_label="Refresh search engine list",
+                                data_initial_value=settings.search_tool
                             ) }}
                         </div>
                     </div>
@@ -128,14 +130,14 @@
                         <!-- Search Iterations -->
                         <div class="form-group half">
                             <label for="iterations">Search Iterations</label>
-                            <input type="number" id="iterations" name="iterations" class="form-control" min="1" max="5" value="2">
+                            <input type="number" id="iterations" name="iterations" class="form-control" min="1" max="5" value="{{ settings.search_iterations }}">
                             <span class="input-help">Number of research cycles to perform</span>
                         </div>
 
                         <!-- Questions Per Iteration -->
                         <div class="form-group half">
                             <label for="questions_per_iteration">Questions Per Iteration</label>
-                            <input type="number" id="questions_per_iteration" name="questions_per_iteration" class="form-control" min="1" max="10" value="3">
+                            <input type="number" id="questions_per_iteration" name="questions_per_iteration" class="form-control" min="1" max="10" value="{{ settings.search_questions_per_iteration }}">
                             <span class="input-help">Follow-up questions in each cycle</span>
                         </div>
                     </div>

local_deep_research/web/templates/pages/star_reviews.html

@@ -782,7 +782,7 @@ function updateRecentRatings(recentRatings) {
         item.innerHTML = `
             <div class="rating-details">
                 <div class="rating-query">
-                    <a href="/research/results/${rating.research_id}" class="rating-link">
+                    <a href="/results/${rating.research_id}" class="rating-link">
                         ${rating.query}
                     </a>
                 </div>

local_deep_research/web_search_engines/engines/search_engine_arxiv.py

@@ -9,6 +9,7 @@ from ...advanced_search_system.filters.journal_reputation_filter import (
 )
 from ...config import search_config
 from ..search_engine_base import BaseSearchEngine
+from ..rate_limiting import RateLimitError
 
 
 class ArXivSearchEngine(BaseSearchEngine):
@@ -155,8 +156,20 @@ class ArXivSearchEngine(BaseSearchEngine):
 
             return previews
 
-        except Exception:
+        except Exception as e:
+            error_msg = str(e)
             logger.exception("Error getting arXiv previews")
+
+            # Check for rate limiting patterns
+            if (
+                "429" in error_msg
+                or "too many requests" in error_msg.lower()
+                or "rate limit" in error_msg.lower()
+                or "service unavailable" in error_msg.lower()
+                or "503" in error_msg
+            ):
+                raise RateLimitError(f"arXiv rate limit hit: {error_msg}")
+
             return []
 
     def _get_full_content(

local_deep_research/web_search_engines/engines/search_engine_brave.py

@@ -7,6 +7,7 @@ from langchain_core.language_models import BaseLLM
 
 from ...config import search_config
 from ..search_engine_base import BaseSearchEngine
+from ..rate_limiting import RateLimitError
 
 logger = logging.getLogger(__name__)
 
@@ -174,7 +175,20 @@ class BraveSearchEngine(BaseSearchEngine):
             return previews
 
         except Exception as e:
-            logger.error(f"Error getting Brave Search results: {e}")
+            error_msg = str(e)
+            logger.error(f"Error getting Brave Search results: {error_msg}")
+
+            # Check for rate limit patterns
+            if (
+                "429" in error_msg
+                or "too many requests" in error_msg.lower()
+                or "rate limit" in error_msg.lower()
+                or "quota" in error_msg.lower()
+            ):
+                raise RateLimitError(
+                    f"Brave Search rate limit hit: {error_msg}"
+                )
+
             return []
 
     def _get_full_content(

local_deep_research/web_search_engines/engines/search_engine_ddg.py

@@ -5,6 +5,7 @@ from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
 from langchain_core.language_models import BaseLLM
 
 from ..search_engine_base import BaseSearchEngine
+from ..rate_limiting import RateLimitError
 from .full_search import FullSearchResults  # Import the FullSearchResults class
 
 logger = logging.getLogger(__name__)
@@ -114,7 +115,25 @@ class DuckDuckGoSearchEngine(BaseSearchEngine):
             return previews
 
         except Exception as e:
-            logger.error(f"Error getting DuckDuckGo previews: {e}")
+            error_msg = str(e)
+            logger.error(f"Error getting DuckDuckGo previews: {error_msg}")
+
+            # Check for known rate limit patterns
+            if "202 Ratelimit" in error_msg or "ratelimit" in error_msg.lower():
+                raise RateLimitError(f"DuckDuckGo rate limit hit: {error_msg}")
+            elif "403" in error_msg or "forbidden" in error_msg.lower():
+                raise RateLimitError(
+                    f"DuckDuckGo access forbidden (possible rate limit): {error_msg}"
+                )
+            elif (
+                "timeout" in error_msg.lower()
+                or "timed out" in error_msg.lower()
+            ):
+                # Timeouts can sometimes indicate rate limiting
+                raise RateLimitError(
+                    f"DuckDuckGo timeout (possible rate limit): {error_msg}"
+                )
+
             return []
 
     def _get_full_content(

local_deep_research/web_search_engines/engines/search_engine_google_pse.py

@@ -8,6 +8,7 @@ from langchain_core.language_models import BaseLLM
 from requests.exceptions import RequestException
 
 from ..search_engine_base import BaseSearchEngine
+from ..rate_limiting import RateLimitError
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -220,20 +221,43 @@ class GooglePSESearchEngine(BaseSearchEngine):
                 return response.json()
 
             except RequestException as e:
+                error_msg = str(e)
                 logger.warning(
                     "Request error on attempt %s / %s: %s",
                     attempt + 1,
                     self.max_retries,
-                    str(e),
+                    error_msg,
                 )
+
+                # Check for rate limiting patterns
+                if (
+                    "quota" in error_msg.lower()
+                    or "quotaExceeded" in error_msg
+                    or "dailyLimitExceeded" in error_msg
+                    or "rateLimitExceeded" in error_msg
+                    or "429" in error_msg
+                    or "403" in error_msg
+                ):
+                    raise RateLimitError(
+                        f"Google PSE rate limit/quota exceeded: {error_msg}"
+                    )
+
                 last_exception = e
             except Exception as e:
+                error_msg = str(e)
                 logger.warning(
                     "Error on attempt %s / %s: %s",
                     attempt + 1,
                     self.max_retries,
-                    str(e),
+                    error_msg,
                 )
+
+                # Check for rate limiting patterns in general errors
+                if "quota" in error_msg.lower() or "limit" in error_msg.lower():
+                    raise RateLimitError(
+                        f"Google PSE error (possible rate limit): {error_msg}"
+                    )
+
                 last_exception = e
 
             attempt += 1

local_deep_research/web_search_engines/engines/search_engine_pubmed.py

@@ -9,6 +9,7 @@ from langchain_core.language_models import BaseLLM
 
 from ...config import search_config
 from ..search_engine_base import BaseSearchEngine
+from ..rate_limiting import RateLimitError
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
@@ -639,7 +640,20 @@ The default assumption should be that medical and scientific queries want RECENT
             return summaries
 
         except Exception as e:
-            logger.error(f"Error getting article summaries: {e}")
+            error_msg = str(e)
+            logger.error(f"Error getting article summaries: {error_msg}")
+
+            # Check for rate limiting patterns
+            if (
+                "429" in error_msg
+                or "too many requests" in error_msg.lower()
+                or "rate limit" in error_msg.lower()
+                or "service unavailable" in error_msg.lower()
+                or "503" in error_msg
+                or "403" in error_msg
+            ):
+                raise RateLimitError(f"PubMed rate limit hit: {error_msg}")
+
             return []
 
     def _get_article_abstracts(self, id_list: List[str]) -> Dict[str, str]: