local-deep-research 0.4.4__py3-none-any.whl → 0.5.0__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_serpapi.py

@@ -45,7 +45,9 @@ class SerpAPISearchEngine(BaseSearchEngine):
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
         )
         self.include_full_content = include_full_content
 
@@ -68,7 +70,9 @@ class SerpAPISearchEngine(BaseSearchEngine):
 
         serpapi_api_key = api_key
         if not serpapi_api_key:
-            serpapi_api_key = get_db_setting("search.engine.web.serpapi.api_key")
+            serpapi_api_key = get_db_setting(
+                "search.engine.web.serpapi.api_key"
+            )
 
         if not serpapi_api_key:
             raise ValueError(
@@ -126,13 +130,17 @@ class SerpAPISearchEngine(BaseSearchEngine):
 
         try:
             # Get search results from SerpAPI
-            organic_results = self.engine.results(query).get("organic_results", [])
+            organic_results = self.engine.results(query).get(
+                "organic_results", []
+            )
 
             # Format results as previews
             previews = []
             for result in organic_results:
                 preview = {
-                    "id": result.get("position", len(previews)),  # Use position as ID
+                    "id": result.get(
+                        "position", len(previews)
+                    ),  # Use position as ID
                     "title": result.get("title", ""),
                     "link": result.get("link", ""),
                     "snippet": result.get("snippet", ""),

local_deep_research/web_search_engines/engines/search_engine_wayback.py

@@ -43,7 +43,9 @@ class WaybackSearchEngine(BaseSearchEngine):
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
         )
         self.max_snapshots_per_url = max_snapshots_per_url
         self.language = language
@@ -79,7 +81,9 @@ class WaybackSearchEngine(BaseSearchEngine):
             return [f"http://{query}"]
 
         # For non-URL queries, use DuckDuckGo to find relevant URLs
-        logger.info("Query is not a URL, using DuckDuckGo to find relevant URLs")
+        logger.info(
+            "Query is not a URL, using DuckDuckGo to find relevant URLs"
+        )
         try:
             # Import DuckDuckGo search engine
             from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
@@ -91,9 +95,13 @@ class WaybackSearchEngine(BaseSearchEngine):
             results = ddg.results(query, url_search_limit)
 
             # Extract URLs from results
-            ddg_urls = [result.get("link") for result in results if result.get("link")]
+            ddg_urls = [
+                result.get("link") for result in results if result.get("link")
+            ]
             if ddg_urls:
-                logger.info(f"Found {len(ddg_urls)} URLs from DuckDuckGo search")
+                logger.info(
+                    f"Found {len(ddg_urls)} URLs from DuckDuckGo search"
+                )
                 return ddg_urls
         except Exception as e:
             logger.error(f"Error using DuckDuckGo for URL discovery: {e}")
@@ -185,12 +193,16 @@ class WaybackSearchEngine(BaseSearchEngine):
                         snapshot = dict(zip(headers, item))
                         timestamp = snapshot.get("timestamp", "")
 
-                        wayback_url = f"https://web.archive.org/web/{timestamp}/{url}"
+                        wayback_url = (
+                            f"https://web.archive.org/web/{timestamp}/{url}"
+                        )
 
                         snapshots.append(
                             {
                                 "timestamp": timestamp,
-                                "formatted_date": self._format_timestamp(timestamp),
+                                "formatted_date": self._format_timestamp(
+                                    timestamp
+                                ),
                                 "url": wayback_url,
                                 "original_url": url,
                                 "available": True,
@@ -265,8 +277,12 @@ class WaybackSearchEngine(BaseSearchEngine):
         if not html or not html.strip():
             return ""
         try:
-            paragraphs = justext.justext(html, justext.get_stoplist(self.language))
-            cleaned = "\n".join([p.text for p in paragraphs if not p.is_boilerplate])
+            paragraphs = justext.justext(
+                html, justext.get_stoplist(self.language)
+            )
+            cleaned = "\n".join(
+                [p.text for p in paragraphs if not p.is_boilerplate]
+            )
             return cleaned
         except Exception as e:
             logger.error(f"Error removing boilerplate: {e}")
@@ -471,7 +487,10 @@ class WaybackSearchEngine(BaseSearchEngine):
             response = requests.get(self.available_api, params={"url": url})
             data = response.json()
 
-            if "archived_snapshots" in data and "closest" in data["archived_snapshots"]:
+            if (
+                "archived_snapshots" in data
+                and "closest" in data["archived_snapshots"]
+            ):
                 snapshot = data["archived_snapshots"]["closest"]
                 timestamp = snapshot["timestamp"]
                 wayback_url = snapshot["url"]
@@ -491,7 +510,9 @@ class WaybackSearchEngine(BaseSearchEngine):
                     not hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
                     or not search_config.SEARCH_SNIPPETS_ONLY
                 ):
-                    raw_html, full_content = self._get_wayback_content(wayback_url)
+                    raw_html, full_content = self._get_wayback_content(
+                        wayback_url
+                    )
                     result["raw_html"] = raw_html
                     result["full_content"] = full_content
 

local_deep_research/web_search_engines/engines/search_engine_wikipedia.py

@@ -38,7 +38,9 @@ class WikipediaSearchEngine(BaseSearchEngine):
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
         )
         self.include_content = include_content
         self.sentences = sentences
@@ -113,6 +115,7 @@ class WikipediaSearchEngine(BaseSearchEngine):
                             "title": title,
                             "snippet": summary,
                             "link": f"https://en.wikipedia.org/wiki/{title.replace(' ', '_')}",
+                            "source": "Wikipedia",
                         }
 
                         previews.append(preview)
@@ -128,7 +131,9 @@ class WikipediaSearchEngine(BaseSearchEngine):
                     logger.error(f"Unexpected error for '{title}': {e}")
                     continue
 
-            logger.info(f"Successfully created {len(previews)} previews from Wikipedia")
+            logger.info(
+                f"Successfully created {len(previews)} previews from Wikipedia"
+            )
             return previews
 
         except Exception as e:
@@ -176,6 +181,7 @@ class WikipediaSearchEngine(BaseSearchEngine):
                     "title": page.title,
                     "link": page.url,
                     "snippet": item.get("snippet", ""),  # Keep existing snippet
+                    "source": "Wikipedia",
                 }
 
                 # Add additional information
@@ -218,7 +224,9 @@ class WikipediaSearchEngine(BaseSearchEngine):
         """
         sentences = sentences or self.sentences
         try:
-            return wikipedia.summary(title, sentences=sentences, auto_suggest=False)
+            return wikipedia.summary(
+                title, sentences=sentences, auto_suggest=False
+            )
         except wikipedia.exceptions.DisambiguationError as e:
             if e.options and len(e.options) > 0:
                 return wikipedia.summary(
@@ -250,6 +258,7 @@ class WikipediaSearchEngine(BaseSearchEngine):
                 "title": page.title,
                 "link": page.url,
                 "snippet": self.get_summary(title, self.sentences),
+                "source": "Wikipedia",
             }
 
             # Add additional information if requested

local_deep_research/web_search_engines/search_engine_base.py

@@ -1,4 +1,5 @@
 import json
+import time
 from abc import ABC, abstractmethod
 from datetime import datetime
 from typing import Any, Dict, List, Optional
@@ -7,7 +8,8 @@ from langchain_core.language_models import BaseLLM
 from loguru import logger
 
 from ..advanced_search_system.filters.base_filter import BaseFilter
-from ..config import search_config
+from ..metrics.search_tracker import get_search_tracker
+from ..utilities.db_utils import get_db_setting
 
 
 class BaseSearchEngine(ABC):
@@ -50,8 +52,12 @@ class BaseSearchEngine(ABC):
             self._content_filters = []
 
         self.llm = llm  # LLM for relevance filtering
-        self._max_filtered_results = int(max_filtered_results)  # Ensure it's an integer
-        self._max_results = max(1, int(max_results))  # Ensure it's a positive integer
+        self._max_filtered_results = int(
+            max_filtered_results
+        )  # Ensure it's an integer
+        self._max_results = max(
+            1, int(max_results)
+        )  # Ensure it's a positive integer
 
     @property
     def max_filtered_results(self) -> int:
@@ -92,44 +98,80 @@ class BaseSearchEngine(ABC):
         Returns:
             List of search results with full content (if available)
         """
-        # Ensure we're measuring time correctly for citation tracking
+        # Track search call for metrics
+        tracker = get_search_tracker()
+        engine_name = self.__class__.__name__.replace(
+            "SearchEngine", ""
+        ).lower()
+        start_time = time.time()
 
-        # Step 1: Get preview information for items
-        previews = self._get_previews(query)
-        if not previews:
-            logger.info(
-                f"Search engine {self.__class__.__name__} returned no preview results for query: {query}"
-            )
-            return []
+        success = True
+        error_message = None
+        results_count = 0
+
+        try:
+            # Step 1: Get preview information for items
+            previews = self._get_previews(query)
+            if not previews:
+                logger.info(
+                    f"Search engine {self.__class__.__name__} returned no preview results for query: {query}"
+                )
+                results_count = 0
+                return []
 
-        for preview_filter in self._preview_filters:
-            previews = preview_filter.filter_results(previews, query)
+            for preview_filter in self._preview_filters:
+                previews = preview_filter.filter_results(previews, query)
 
-        # Step 2: Filter previews for relevance with LLM
-        filtered_items = self._filter_for_relevance(previews, query)
-        if not filtered_items:
+            # Step 2: Filter previews for relevance with LLM
+            # TEMPORARILY DISABLED: Skip LLM relevance filtering
+            filtered_items = previews
             logger.info(
-                f"All preview results were filtered out as irrelevant for query: {query}"
+                f"LLM relevance filtering disabled - returning all {len(previews)} previews"
             )
-            # Do not fall back to previews, return empty list instead
-            return []
 
-        # Step 3: Get full content for filtered items
-        # Import config inside the method to avoid circular import
+            # # Original filtering code (disabled):
+            # filtered_items = self._filter_for_relevance(previews, query)
+            # if not filtered_items:
+            #     logger.info(
+            #         f"All preview results were filtered out as irrelevant for query: {query}"
+            #     )
+            #     # Do not fall back to previews, return empty list instead
+            #     results_count = 0
+            #     return []
+
+            # Step 3: Get full content for filtered items
+            # Import config inside the method to avoid circular import
+
+            if get_db_setting("search.snippets_only", True):
+                logger.info("Returning snippet-only results as per config")
+                results = filtered_items
+            else:
+                results = self._get_full_content(filtered_items)
+
+            for content_filter in self._content_filters:
+                results = content_filter.filter_results(results, query)
 
-        if (
-            hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
-            and search_config.SEARCH_SNIPPETS_ONLY
-        ):
-            logger.info("Returning snippet-only results as per config")
-            results = filtered_items
-        else:
-            results = self._get_full_content(filtered_items)
+            results_count = len(results)
+            return results
 
-        for content_filter in self._content_filters:
-            results = content_filter.filter_results(results, query)
+        except Exception as e:
+            success = False
+            error_message = str(e)
+            logger.error(f"Search engine {self.__class__.__name__} failed: {e}")
+            results_count = 0
+            return []
 
-        return results
+        finally:
+            # Record search metrics
+            response_time_ms = int((time.time() - start_time) * 1000)
+            tracker.record_search(
+                engine_name=engine_name,
+                query=query,
+                results_count=results_count,
+                response_time_ms=response_time_ms,
+                success=success,
+                error_message=error_message,
+            )
 
     def invoke(self, query: str) -> List[Dict[str, Any]]:
         """Compatibility method for LangChain tools"""
@@ -229,7 +271,9 @@ Respond with ONLY the JSON array, no other text."""
                         if idx < len(previews):
                             ranked_results.append(previews[idx])
                         else:
-                            logger.warning(f"Index {idx} out of range, skipping")
+                            logger.warning(
+                                f"Index {idx} out of range, skipping"
+                            )
 
                     # Limit to max_filtered_results if specified
                     if (
@@ -244,14 +288,18 @@ Respond with ONLY the JSON array, no other text."""
                     return ranked_results
 
                 except json.JSONDecodeError as e:
-                    logger.warning(f"Failed to parse JSON from LLM response: {e}")
+                    logger.warning(
+                        f"Failed to parse JSON from LLM response: {e}"
+                    )
                     logger.debug(f"Problematic JSON text: {array_text}")
                     return []
             else:
                 logger.warning(
                     "Could not find JSON array in response, returning original previews"
                 )
-                logger.debug(f"Response text without JSON array: {response_text}")
+                logger.debug(
+                    f"Response text without JSON array: {response_text}"
+                )
                 return previews[: min(5, len(previews))]
 
         except Exception:

local_deep_research/web_search_engines/search_engine_factory.py

@@ -56,14 +56,18 @@ def create_search_engine(
             api_key = engine_config.get("api_key")
 
         if not api_key:
-            logger.info(f"Required API key for {engine_name} not found in settings.")
+            logger.info(
+                f"Required API key for {engine_name} not found in settings."
+            )
             return None
 
         # Set the engine-specific environment variable if needed
         # This is to support engines that directly check environment variables
         if engine_name == "brave" and not os.getenv("BRAVE_API_KEY"):
             os.environ["BRAVE_API_KEY"] = api_key
-            logger.info("Set BRAVE_API_KEY environment variable from database setting")
+            logger.info(
+                "Set BRAVE_API_KEY environment variable from database setting"
+            )
 
     # Check for LLM requirements
     if engine_config.get("requires_llm", False) and not llm:
@@ -149,7 +153,9 @@ def _create_full_search_wrapper(
         class_name = engine_config.get("full_search_class")
 
         if not module_path or not class_name:
-            logger.warning(f"Full search configuration missing for {engine_name}")
+            logger.warning(
+                f"Full search configuration missing for {engine_name}"
+            )
             return base_engine
 
         # Import the full search class
@@ -163,10 +169,15 @@ def _create_full_search_wrapper(
         ]  # Skip 'self'
 
         # Extract relevant parameters for the full search wrapper
-        wrapper_params = {k: v for k, v in params.items() if k in wrapper_init_params}
+        wrapper_params = {
+            k: v for k, v in params.items() if k in wrapper_init_params
+        }
 
         # Special case for SerpAPI which needs the API key directly
-        if engine_name == "serpapi" and "serpapi_api_key" in wrapper_init_params:
+        if (
+            engine_name == "serpapi"
+            and "serpapi_api_key" in wrapper_init_params
+        ):
             serpapi_api_key = os.getenv("SERP_API_KEY")
             if serpapi_api_key:
                 wrapper_params["serpapi_api_key"] = serpapi_api_key
@@ -196,7 +207,9 @@ def _create_full_search_wrapper(
             if not brave_api_key:
                 from ..utilities.db_utils import get_db_setting
 
-                brave_api_key = get_db_setting("search.engine.web.brave.api_key")
+                brave_api_key = get_db_setting(
+                    "search.engine.web.brave.api_key"
+                )
 
             if brave_api_key:
                 wrapper_params["api_key"] = brave_api_key
@@ -236,7 +249,9 @@ def _create_full_search_wrapper(
         return full_search
 
     except Exception:
-        logger.exception(f"Failed to create full search wrapper for {engine_name}")
+        logger.exception(
+            f"Failed to create full search wrapper for {engine_name}"
+        )
         return base_engine
 
 
@@ -307,7 +322,9 @@ def get_search(
         )
     else:
         engine_type = type(engine).__name__
-        logger.info(f"Successfully created search engine of type: {engine_type}")
+        logger.info(
+            f"Successfully created search engine of type: {engine_type}"
+        )
         # Check if the engine has run method
         if hasattr(engine, "run"):
             logger.info(f"Engine has 'run' method: {getattr(engine, 'run')}")

local_deep_research/web_search_engines/search_engines_config.py

@@ -12,7 +12,9 @@ from ..utilities.db_utils import get_db_setting
 from .default_search_engines import get_default_elasticsearch_config
 
 
-def _extract_per_engine_config(raw_config: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
+def _extract_per_engine_config(
+    raw_config: Dict[str, Any],
+) -> Dict[str, Dict[str, Any]]:
     """
     Converts the "flat" configuration loaded from the settings database into
     individual settings dictionaries for each engine.
@@ -55,7 +57,9 @@ def search_config() -> Dict[str, Any]:
     search_engines = _extract_per_engine_config(config_data)
     search_engines["auto"] = get_db_setting("search.engine.auto", {})
 
-    logger.info(f"Loaded {len(search_engines)} search engines from configuration file")
+    logger.info(
+        f"Loaded {len(search_engines)} search engines from configuration file"
+    )
     logger.info(f"\n  {', '.join(sorted(search_engines.keys()))} \n")
 
     # Add alias for 'auto' if it exists
@@ -141,7 +145,9 @@ def local_search_engines() -> List[str]:
     local_collections_data.pop("local_all", None)
     # Remove disabled collections.
     local_collections_data = {
-        k: v for k, v in local_collections_data.items() if v.get("enabled", True)
+        k: v
+        for k, v in local_collections_data.items()
+        if v.get("enabled", True)
     }
 
     enabled_collections = list(local_collections_data.keys())

{local_deep_research-0.4.4.dist-info → local_deep_research-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: local-deep-research
-Version: 0.4.4
+Version: 0.5.0
 Summary: AI-powered research assistant with deep, iterative analysis using LLMs and web searches
 Author-Email: LearningCircuit <185559241+LearningCircuit@users.noreply.github.com>, HashedViking <6432677+HashedViking@users.noreply.github.com>
 License: MIT License
@@ -71,10 +71,12 @@ Requires-Dist: optuna>=4.3.0
 Requires-Dist: elasticsearch==8.14.0
 Requires-Dist: methodtools>=0.4.7
 Requires-Dist: loguru>=0.7.3
+Requires-Dist: cachetools>=5.5.2
 Requires-Dist: matplotlib>=3.10.3
 Requires-Dist: pandas>=2.2.3
 Requires-Dist: plotly>=6.0.1
 Requires-Dist: kaleido==0.2.1
+Requires-Dist: aiohttp>=3.9.0
 Description-Content-Type: text/markdown
 
 # Local Deep Research
@@ -400,6 +402,10 @@ For more information and examples of what Local Deep Research can produce:
 - [Reddit](https://www.reddit.com/r/LocalDeepResearch/): Announcements, updates, and community showcase
 - [GitHub Issues](https://github.com/LearningCircuit/local-deep-research/issues): Bug reports and feature requests
 
+## 🚀 Contributing
+
+We welcome contributions! Whether you're fixing bugs, adding features, or improving documentation, we'd love to have you as part of our community. Please see our [Contributing Guide](CONTRIBUTING.md) for guidelines on how to get started.
+
 ## 📄 License & Acknowledgments
 
 This project is licensed under the MIT License.