local-deep-research 0.4.4__py3-none-any.whl → 0.5.2__py3-none-any.whl

local_deep_research/web/utils/formatters.py

@@ -29,7 +29,9 @@ def convert_debug_to_markdown(raw_text, query):
         if "DETAILED FINDINGS:" in raw_text:
             logger.info("Found DETAILED FINDINGS section")
             detailed_index = raw_text.index("DETAILED FINDINGS:")
-            content = raw_text[detailed_index + len("DETAILED FINDINGS:") :].strip()
+            content = raw_text[
+                detailed_index + len("DETAILED FINDINGS:") :
+            ].strip()
         else:
             logger.info("No DETAILED FINDINGS section found, using full text")
             content = raw_text
@@ -40,7 +42,8 @@ def convert_debug_to_markdown(raw_text, query):
             [
                 line
                 for line in content.split("\n")
-                if not line.strip().startswith("===") and not line.strip() == "=" * 80
+                if not line.strip().startswith("===")
+                and not line.strip() == "=" * 80
             ]
         )
         lines_after = len(content.split("\n"))
@@ -54,7 +57,10 @@ def convert_debug_to_markdown(raw_text, query):
             for marker in ["DETAILED FINDINGS:", "COMPLETE RESEARCH:"]:
                 if marker in content[search_index:]:
                     marker_pos = content.index(marker, search_index)
-                    if next_major_section == -1 or marker_pos < next_major_section:
+                    if (
+                        next_major_section == -1
+                        or marker_pos < next_major_section
+                    ):
                         next_major_section = marker_pos
 
             if next_major_section != -1:

local_deep_research/web_search_engines/default_search_engines.py

@@ -3,10 +3,11 @@ Default search engine configurations.
 This file can be used to initialize the search engine configurations.
 """
 
+
 def get_default_elasticsearch_config():
     """
     Returns the default Elasticsearch search engine configuration.
-    
+
     Returns:
         dict: Default configuration for Elasticsearch search engine
     """
@@ -26,13 +27,14 @@ def get_default_elasticsearch_config():
         "reliability": "High, depending on your Elasticsearch setup",
     }
 
+
 def get_default_search_engine_configs():
     """
     Returns a dictionary of default search engine configurations.
-    
+
     Returns:
         dict: Dictionary of default search engine configurations
     """
     return {
         "elasticsearch": get_default_elasticsearch_config(),
-    } 
+    }

local_deep_research/web_search_engines/engines/full_search.py

@@ -74,7 +74,9 @@ class FullSearchResults:
         if not html or not html.strip():
             return ""
         paragraphs = justext.justext(html, justext.get_stoplist(self.language))
-        cleaned = "\n".join([p.text for p in paragraphs if not p.is_boilerplate])
+        cleaned = "\n".join(
+            [p.text for p in paragraphs if not p.is_boilerplate]
+        )
         return cleaned
 
     def run(self, query: str):
@@ -91,7 +93,11 @@ class FullSearchResults:
             filtered_results = search_results
 
         # Extract URLs from filtered results
-        urls = [result.get("link") for result in filtered_results if result.get("link")]
+        urls = [
+            result.get("link")
+            for result in filtered_results
+            if result.get("link")
+        ]
 
         if not urls:
             logger.error("\n === NO VALID LINKS ===\n")

local_deep_research/web_search_engines/engines/meta_search_engine.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
 from loguru import logger
 
 from ...utilities.db_utils import get_db_setting
-from ...web.services.socket_service import emit_socket_event
+from ...web.services.socket_service import SocketIOService
 from ..search_engine_base import BaseSearchEngine
 from ..search_engine_factory import create_search_engine
 from ..search_engines_config import search_config
@@ -39,7 +39,9 @@ class MetaSearchEngine(BaseSearchEngine):
         """
         # Initialize the BaseSearchEngine with the LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
         )
 
         self.use_api_key_services = use_api_key_services
@@ -82,7 +84,9 @@ class MetaSearchEngine(BaseSearchEngine):
                 )
             else:
                 # Format: search.engine.web.{engine_name}.use_in_auto_search
-                auto_search_setting = f"search.engine.web.{name}.use_in_auto_search"
+                auto_search_setting = (
+                    f"search.engine.web.{name}.use_in_auto_search"
+                )
 
             # Get setting from database, default to False if not found
             use_in_auto_search = get_db_setting(auto_search_setting, False)
@@ -95,7 +99,10 @@ class MetaSearchEngine(BaseSearchEngine):
                 continue
 
             # Skip engines that require API keys if we don't want to use them
-            if config_.get("requires_api_key", False) and not self.use_api_key_services:
+            if (
+                config_.get("requires_api_key", False)
+                and not self.use_api_key_services
+            ):
                 continue
 
             # Skip engines that require API keys if the key is not available
@@ -155,11 +162,15 @@ class MetaSearchEngine(BaseSearchEngine):
 
             # For searches containing "arxiv", prioritize the arxiv engine
             if "arxiv" in query_lower and "arxiv" in self.available_engines:
-                return ["arxiv"] + [e for e in self.available_engines if e != "arxiv"]
+                return ["arxiv"] + [
+                    e for e in self.available_engines if e != "arxiv"
+                ]
 
             # For searches containing "pubmed", prioritize the pubmed engine
             if "pubmed" in query_lower and "pubmed" in self.available_engines:
-                return ["pubmed"] + [e for e in self.available_engines if e != "pubmed"]
+                return ["pubmed"] + [
+                    e for e in self.available_engines if e != "pubmed"
+                ]
 
             # Check if SearXNG is available and prioritize it for general queries
             if "searxng" in self.available_engines:
@@ -169,7 +180,9 @@ class MetaSearchEngine(BaseSearchEngine):
                 ]
                 reliability_sorted = sorted(
                     engines_without_searxng,
-                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
+                    key=lambda x: search_config()
+                    .get(x, {})
+                    .get("reliability", 0),
                     reverse=True,
                 )
                 return ["searxng"] + reliability_sorted
@@ -182,7 +195,9 @@ class MetaSearchEngine(BaseSearchEngine):
                 # Return engines sorted by reliability
                 return sorted(
                     self.available_engines,
-                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
+                    key=lambda x: search_config()
+                    .get(x, {})
+                    .get("reliability", 0),
                     reverse=True,
                 )
 
@@ -213,7 +228,9 @@ class MetaSearchEngine(BaseSearchEngine):
                 )
                 return sorted(
                     self.available_engines,
-                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
+                    key=lambda x: search_config()
+                    .get(x, {})
+                    .get("reliability", 0),
                     reverse=True,
                 )
 
@@ -255,7 +272,10 @@ Example output: searxng,wikipedia,brave"""
                     valid_engines.append(cleaned_name)
 
             # If SearXNG is available but not selected by the LLM, add it as a fallback
-            if "searxng" in self.available_engines and "searxng" not in valid_engines:
+            if (
+                "searxng" in self.available_engines
+                and "searxng" not in valid_engines
+            ):
                 # Add it as the last option if the LLM selected others
                 if valid_engines:
                     valid_engines.append("searxng")
@@ -267,7 +287,9 @@ Example output: searxng,wikipedia,brave"""
             if not valid_engines:
                 valid_engines = sorted(
                     self.available_engines,
-                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
+                    key=lambda x: search_config()
+                    .get(x, {})
+                    .get("reliability", 0),
                     reverse=True,
                 )
 
@@ -278,13 +300,17 @@ Example output: searxng,wikipedia,brave"""
             if "searxng" in self.available_engines:
                 return ["searxng"] + sorted(
                     [e for e in self.available_engines if e != "searxng"],
-                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
+                    key=lambda x: search_config()
+                    .get(x, {})
+                    .get("reliability", 0),
                     reverse=True,
                 )
             else:
                 return sorted(
                     self.available_engines,
-                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
+                    key=lambda x: search_config()
+                    .get(x, {})
+                    .get("reliability", 0),
                     reverse=True,
                 )
 
@@ -342,9 +368,12 @@ Example output: searxng,wikipedia,brave"""
 
                     # Emit a socket event to inform about the selected engine
                     try:
-                        emit_socket_event(
+                        SocketIOService().emit_socket_event(
                             "search_engine_selected",
-                            {"engine": engine_name, "result_count": len(previews)},
+                            {
+                                "engine": engine_name,
+                                "result_count": len(previews),
+                            },
                         )
                     except Exception:
                         logger.exception("Socket emit error (non-critical)")
@@ -355,7 +384,9 @@ Example output: searxng,wikipedia,brave"""
                 all_errors.append(f"{engine_name} returned no previews")
 
             except Exception as e:
-                error_msg = f"Error getting previews from {engine_name}: {str(e)}"
+                error_msg = (
+                    f"Error getting previews from {engine_name}: {str(e)}"
+                )
                 logger.exception(error_msg)
                 all_errors.append(error_msg)
 
@@ -390,7 +421,9 @@ Example output: searxng,wikipedia,brave"""
         # Use the selected engine to get full content
         if hasattr(self, "_selected_engine"):
             try:
-                logger.info(f"Using {self._selected_engine_name} to get full content")
+                logger.info(
+                    f"Using {self._selected_engine_name} to get full content"
+                )
                 return self._selected_engine._get_full_content(relevant_items)
             except Exception:
                 logger.exception(
@@ -404,7 +437,9 @@ Example output: searxng,wikipedia,brave"""
             )
             return relevant_items
 
-    def _get_engine_instance(self, engine_name: str) -> Optional[BaseSearchEngine]:
+    def _get_engine_instance(
+        self, engine_name: str
+    ) -> Optional[BaseSearchEngine]:
         """Get or create an instance of the specified search engine"""
         # Return cached instance if available
         if engine_name in self.engine_cache:
@@ -418,11 +453,15 @@ Example output: searxng,wikipedia,brave"""
 
             # Add max_filtered_results if specified
             if self.max_filtered_results is not None:
-                common_params["max_filtered_results"] = self.max_filtered_results
+                common_params["max_filtered_results"] = (
+                    self.max_filtered_results
+                )
 
             engine = create_search_engine(engine_name, **common_params)
         except Exception:
-            logger.exception(f"Error creating engine instance for {engine_name}")
+            logger.exception(
+                f"Error creating engine instance for {engine_name}"
+            )
             return None
 
         if engine:

local_deep_research/web_search_engines/engines/search_engine_arxiv.py

@@ -148,6 +148,7 @@ class ArXivSearchEngine(BaseSearchEngine):
                         else None
                     ),
                     "journal_ref": paper.journal_ref,
+                    "source": "arXiv",
                 }
 
                 previews.append(preview)
@@ -234,8 +235,12 @@ class ArXivSearchEngine(BaseSearchEngine):
                 ):
                     try:
                         # Download the paper
-                        pdf_count += 1  # Increment counter before attempting download
-                        paper_path = paper.download_pdf(dirpath=self.download_dir)
+                        pdf_count += (
+                            1  # Increment counter before attempting download
+                        )
+                        paper_path = paper.download_pdf(
+                            dirpath=self.download_dir
+                        )
                         result["pdf_path"] = str(paper_path)
 
                         # Extract text from PDF
@@ -266,7 +271,9 @@ class ArXivSearchEngine(BaseSearchEngine):
                                     with pdfplumber.open(paper_path) as pdf:
                                         pdf_text = ""
                                         for page in pdf.pages:
-                                            pdf_text += page.extract_text() + "\n\n"
+                                            pdf_text += (
+                                                page.extract_text() + "\n\n"
+                                            )
 
                                         if (
                                             pdf_text.strip()
@@ -285,9 +292,13 @@ class ArXivSearchEngine(BaseSearchEngine):
                                     )
                         except Exception:
                             logger.exception("Error extracting text from PDF")
-                            logger.error("Using paper summary as content instead")
+                            logger.error(
+                                "Using paper summary as content instead"
+                            )
                     except Exception:
-                        logger.exception(f"Error downloading paper {paper.title}")
+                        logger.exception(
+                            f"Error downloading paper {paper.title}"
+                        )
                         result["pdf_path"] = None
                         pdf_count -= 1  # Decrement counter if download fails
                 elif (
@@ -400,7 +411,9 @@ class ArXivSearchEngine(BaseSearchEngine):
                 if self.include_full_text and self.download_dir:
                     try:
                         # Download the paper
-                        paper_path = paper.download_pdf(dirpath=self.download_dir)
+                        paper_path = paper.download_pdf(
+                            dirpath=self.download_dir
+                        )
                         result["pdf_path"] = str(paper_path)
                     except Exception:
                         logger.exception("Error downloading paper")

local_deep_research/web_search_engines/engines/search_engine_brave.py

@@ -46,7 +46,9 @@ class BraveSearchEngine(BaseSearchEngine):
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
         )
         self.include_full_content = include_full_content
 
@@ -144,7 +146,9 @@ class BraveSearchEngine(BaseSearchEngine):
 
                     raw_results = json.loads(raw_results)
                 except json.JSONDecodeError:
-                    logger.error("Error: Unable to parse BraveSearch response as JSON.")
+                    logger.error(
+                        "Error: Unable to parse BraveSearch response as JSON."
+                    )
                     return []
 
             # Format results as previews

local_deep_research/web_search_engines/engines/search_engine_ddg.py

@@ -36,7 +36,9 @@ class DuckDuckGoSearchEngine(BaseSearchEngine):
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
         )
         self.region = region
         self.safe_search = safe_search