local-deep-research 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl

local_deep_research/config.py

@@ -91,15 +91,15 @@ def get_search(search_tool=None):
     params = {
         "search_tool": tool,
         "llm_instance": get_llm(),
-        "max_results": settings.get("max_results"),
-        "region": settings.get("region"),
-        "time_period": settings.get("time_period"),
-        "safe_search": settings.get("safe_search"),
-        "search_snippets_only": settings.get("snippets_only"),
-        "search_language": settings.get("search_language"),
-        "max_filtered_results": settings.get("max_filtered_results")
+        "max_results": settings.search.max_results,
+        "region": settings.search.region,
+        "time_period": settings.search.time_period,
+        "safe_search": settings.search.safe_search,
+        "search_snippets_only": settings.search.snippets_only,
+        "search_language": settings.search.search_language,
+        "max_filtered_results": settings.search.max_filtered_results
     }
-    
+    logger.info(f"Search config params: {params}")
     # Create and return search engine
     return factory_get_search(**params)
 

local_deep_research/defaults/search_engines.toml

@@ -186,9 +186,48 @@ strengths = ["searches all local collections", "personal documents", "offline ac
 weaknesses = ["may return too many results", "requires indexing"]
 requires_llm = true
 
+[semantic_scholar]
+module_path = "local_deep_research.web_search_engines.engines.search_engine_semantic_scholar"
+class_name = "SemanticScholarSearchEngine"
+requires_api_key = false
+api_key_env = "S2_API_KEY"
+reliability = 0.95
+strengths = [
+    "comprehensive scientific literature", 
+    "extensive citation network", 
+    "AI-generated summaries (TLDRs)", 
+    "academic paper metadata", 
+    "cross-disciplinary coverage",
+    "200M+ papers across all fields",
+    "usable without API key"
+]
+weaknesses = [
+    "rate limited (1000 requests/day) without API key",
+    "limited to academic content"
+]
+supports_full_search = true
+requires_llm = false
+
+[semantic_scholar.default_params]
+max_results = 20
+get_abstracts = true
+get_tldr = true
+get_references = false
+get_citations = false
+get_embeddings = false
+citation_limit = 10
+reference_limit = 10
+optimize_queries = true
+max_retries = 5
+retry_backoff_factor = 1.0
+
 # Default search engine to use if none specified
 DEFAULT_SEARCH_ENGINE = "wikipedia"
 
+
+
+
+
 # Additional search engines can be added below
 # Uncomment and modify these templates as needed
 
@@ -224,21 +263,3 @@ DEFAULT_SEARCH_ENGINE = "wikipedia"
 # 
 # [guardian.default_params]
 # order_by = "relevance"
-
-# [medrxiv]
-# module_path = "local_deep_research.web_search_engines.engines.search_engine_medrxiv"
-# class_name = "MedRxivSearchEngine"
-# requires_api_key = false
-# reliability = 0.85
-# strengths = [
-#     "medical preprints", "health research", "covid-19 research", 
-#     "clinical studies", "medical sciences", "preliminary results"
-# ]
-# weaknesses = ["not peer-reviewed", "preliminary findings", "limited to medical research"]
-# requires_llm = true
-# 
-# [medrxiv.default_params]
-# sort_by = "relevance_score"
-# sort_order = "desc"
-# include_full_text = false
-# optimize_queries = true

local_deep_research/search_system.py

@@ -216,11 +216,12 @@ class AdvancedSearchSystem:
                             }
                         )
 
-                        if settings.general.knowledge_accumulation != KnowledgeAccumulationApproach.NO_KNOWLEDGE:
+                        if settings.general.knowledge_accumulation != str(KnowledgeAccumulationApproach.NO_KNOWLEDGE.value):
                             current_knowledge = current_knowledge + "\n\n\n New: \n" + results_with_links
                         
-                        print(current_knowledge)
-                        if settings.general.knowledge_accumulation == KnowledgeAccumulationApproach.QUESTION:
+                        logger.info(settings.general.knowledge_accumulation)
+                        if settings.general.knowledge_accumulation == str(KnowledgeAccumulationApproach.QUESTION.value):
+                            logger.info("Compressing knowledge")
                             self._update_progress(f"Compress Knowledge for: {question}", 
                                         int(question_progress_base + 0),
                                         {"phase": "analysis"})
@@ -240,10 +241,14 @@ class AdvancedSearchSystem:
             self._update_progress(f"Compressing knowledge after iteration {iteration}", 
                                  int((iteration / total_iterations) * 100 - 5),
                                  {"phase": "knowledge_compression"})
-
-            if settings.general.knowledge_accumulation == KnowledgeAccumulationApproach.ITERATION:
+            logger.info(str(iteration))
+            logger.info(settings.general.knowledge_accumulation)
+            logger.info(str(KnowledgeAccumulationApproach.ITERATION.value))
+            if settings.general.knowledge_accumulation == KnowledgeAccumulationApproach.ITERATION.value:
                 try:
+                    logger.info("ITERATION - Compressing Knowledge")
                     current_knowledge = self._compress_knowledge(current_knowledge , query, section_links)
+                    logger.info("FINISHED ITERATION - Compressing Knowledge")
                 except Exception as e:
                     error_msg = f"Error compressing knowledge: {str(e)}"
                     print(f"COMPRESSION ERROR: {error_msg}")
@@ -273,11 +278,12 @@ class AdvancedSearchSystem:
             "findings": findings,
             "iterations": iteration,
             "questions": self.questions_by_iteration,
-            "formatted_findings": formatted_findings if 'formatted_findings' in locals() else "Error: Findings not available.",
+            "formatted_findings": formatted_findings,
             "current_knowledge": current_knowledge
         }
 
     def _save_findings(self, findings: List[Dict], current_knowledge: str, query: str):
+        logger.info("Saving findings ...")
         self._update_progress("Saving research findings...", None)
         
         formatted_findings = format_findings_to_text(
@@ -287,15 +293,15 @@ class AdvancedSearchSystem:
             :50
         ]
         safe_query = safe_query.replace(" ", "_").lower()
-
-        output_dir = "research_outputs"
+        import local_deep_research.config as conf
+        output_dir = f"{conf.get_config_dir()}/research_outputs"
         if not os.path.exists(output_dir):
             os.makedirs(output_dir)
-
+        
         filename = os.path.join(output_dir, f"formatted_output_{safe_query}.txt")
 
         with open(filename, "w", encoding="utf-8") as text_file:
             text_file.write(formatted_findings)
-            
+        logger.info("Saved findings")
         self._update_progress("Research findings saved", None, {"filename": filename})
         return formatted_findings

local_deep_research/utilties/enums.py

@@ -2,8 +2,8 @@
 from enum import Enum, auto
 
 class KnowledgeAccumulationApproach(Enum):
-    QUESTION = auto()
-    ITERATION = auto()
-    NO_KNOWLEDGE = auto()
-    MAX_NR_OF_CHARACTERS = auto()
+    QUESTION = "QUESTION"
+    ITERATION = "ITERATION"
+    NO_KNOWLEDGE = "NO_KNOWLEDGE"
+    MAX_NR_OF_CHARACTERS = "MAX_NR_OF_CHARACTERS"
 

local_deep_research/web/app.py

@@ -1001,26 +1001,14 @@ def run_research_process(research_id, query, mode):
         if mode == 'quick':
             # Quick Summary
             if results.get('findings'):
-                #initial_analysis = [finding['content'] for finding in results['findings']]
-                summary = ""
-                
-                # Safer access to formatted_findings with logging
-                print(f"Results keys: {list(results.keys())}")
-                
-                # Check if formatted_findings exists in results
-                if 'formatted_findings' not in results:
-                    print("WARNING: 'formatted_findings' not found in results, using fallback")
-                    # Create fallback formatted findings from available data
-                    raw_formatted_findings = "# Research Findings\n\n"
-                    for i, finding in enumerate(results.get('findings', [])):
-                        raw_formatted_findings += f"## Finding {i+1}\n\n{finding.get('content', '')}\n\n"
-                else:
-                    raw_formatted_findings = results['formatted_findings']
-                    print(f"Found formatted_findings of length: {len(str(raw_formatted_findings))}")
+
+                raw_formatted_findings = results['formatted_findings']
+                logger.info(f"Found formatted_findings of length: {len(str(raw_formatted_findings))}")
                 
                 try:
+                    clean_markdown = raw_formatted_findings
                     # ADDED CODE: Convert debug output to clean markdown
-                    clean_markdown = convert_debug_to_markdown(raw_formatted_findings, query)
+                    #clean_markdown = convert_debug_to_markdown(raw_formatted_findings, query)
                     print(f"Successfully converted to clean markdown of length: {len(clean_markdown)}")
                     
                     # First send a progress update for generating the summary
@@ -1692,10 +1680,7 @@ def convert_debug_to_markdown(raw_text, query):
         lines_after = len(content.split("\n"))
         print(f"Removed {lines_before - lines_after} divider lines")
         
-        # If COMPLETE RESEARCH OUTPUT exists, remove that section
-        if "COMPLETE RESEARCH OUTPUT" in content:
-            print("Found and removing COMPLETE RESEARCH OUTPUT section")
-            content = content.split("COMPLETE RESEARCH OUTPUT")[0].strip()
+
         
         # Remove SEARCH QUESTIONS BY ITERATION section
         if "SEARCH QUESTIONS BY ITERATION:" in content:

local_deep_research/web_search_engines/engines/search_engine_arxiv.py

@@ -32,11 +32,9 @@ class ArXivSearchEngine(BaseSearchEngine):
             llm: Language model for relevance filtering
             max_filtered_results: Maximum number of results to keep after filtering
         """
-        # Initialize the BaseSearchEngine with the LLM and max_filtered_results
-        super().__init__(llm=llm, max_filtered_results=max_filtered_results)
-        
-        #max_results = min(max_results, 20) # required for arxiv
-        self.max_results = 20 # TODO this needs to be corrected.
+        # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
+        self.max_results=max(self.max_results,25)
         self.sort_by = sort_by
         self.sort_order = sort_order
         self.include_full_text = include_full_text

local_deep_research/web_search_engines/engines/search_engine_brave.py

@@ -19,7 +19,7 @@ class BraveSearchEngine(BaseSearchEngine):
                 api_key: Optional[str] = None,
                 language_code_mapping: Optional[Dict[str, str]] = None,
                 llm: Optional[BaseLLM] = None,
-                include_full_content: bool = False,
+                include_full_content: bool = True,
                 max_filtered_results: Optional[int] = None,
                 **kwargs):
         """
@@ -38,10 +38,8 @@ class BraveSearchEngine(BaseSearchEngine):
             max_filtered_results: Maximum number of results to keep after filtering
             **kwargs: Additional parameters (ignored but accepted for compatibility)
         """
-        # Initialize the BaseSearchEngine with the LLM and max_filtered_results
-        super().__init__(llm=llm, max_filtered_results=max_filtered_results)
-        
-        self.max_results = max_results
+        # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
         self.include_full_content = include_full_content
         
         # Set up language code mapping

local_deep_research/web_search_engines/engines/search_engine_ddg.py

@@ -14,7 +14,8 @@ class DuckDuckGoSearchEngine(BaseSearchEngine):
                 safe_search: bool = True,
                 llm: Optional[BaseLLM] = None,
                 language: str = "English",
-                include_full_content: bool = False):
+                include_full_content: bool = False,
+                max_filtered_results=5):
         """
         Initialize the DuckDuckGo search engine.
         
@@ -26,8 +27,8 @@ class DuckDuckGoSearchEngine(BaseSearchEngine):
             language: Language for content processing
             include_full_content: Whether to include full webpage content in results
         """
-        super().__init__(llm=llm)
-        self.max_results = max_results
+        # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
         self.region = region
         self.safe_search = safe_search
         self.language = language

local_deep_research/web_search_engines/engines/search_engine_github.py

@@ -39,10 +39,8 @@ class GitHubSearchEngine(BaseSearchEngine):
             llm: Language model for relevance filtering
             max_filtered_results: Maximum number of results to keep after filtering
         """
-        # Initialize the BaseSearchEngine with the LLM and max_filtered_results
-        super().__init__(llm=llm, max_filtered_results=max_filtered_results)
-        
-        self.max_results = max_results
+        # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
         self.api_key = api_key or os.getenv("GITHUB_API_KEY")
         self.search_type = search_type
         self.include_readme = include_readme

local_deep_research/web_search_engines/engines/search_engine_google_pse.py

@@ -47,10 +47,8 @@ class GooglePSESearchEngine(BaseSearchEngine):
             retry_delay: Base delay in seconds between retry attempts
             **kwargs: Additional parameters (ignored but accepted for compatibility)
         """
-        # Initialize the BaseSearchEngine with the LLM and max_filtered_results
-        super().__init__(llm=llm, max_filtered_results=max_filtered_results)
-        
-        self.max_results = max_results
+        # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
         self.include_full_content = include_full_content
         
         # Retry configuration