local-deep-research 0.1.15__tar.gz → 0.1.16__tar.gz

{local_deep_research-0.1.15/src/local_deep_research.egg-info → local_deep_research-0.1.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: local-deep-research
-Version: 0.1.15
+Version: 0.1.16
 Summary: AI-powered research assistant with deep, iterative analysis using LLMs and web searches
 Author-email: LearningCircuit <185559241+LearningCircuit@users.noreply.github.com>, HashedViking <6432677+HashedViking@users.noreply.github.com>
 License: MIT License

{local_deep_research-0.1.15 → local_deep_research-0.1.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "local-deep-research"
-version = "0.1.15"
+version = "0.1.16"
 description = "AI-powered research assistant with deep, iterative analysis using LLMs and web searches"
 readme = "README.md"
 requires-python = ">=3.8"

{local_deep_research-0.1.15 → local_deep_research-0.1.16}/src/local_deep_research/defaults/search_engines.toml

@@ -37,7 +37,7 @@ module_path = "local_deep_research.web_search_engines.engines.search_engine_pubm
 class_name = "PubMedSearchEngine"
 requires_api_key = false
 api_key_env = "NCBI_API_KEY"
-reliability = 0.95
+reliability = 0.98
 strengths = [
     "biomedical literature", "medical research", "clinical studies", 
     "life sciences", "health information", "scientific papers"
@@ -191,7 +191,7 @@ module_path = "local_deep_research.web_search_engines.engines.search_engine_sema
 class_name = "SemanticScholarSearchEngine"
 requires_api_key = false
 api_key_env = "S2_API_KEY"
-reliability = 0.95
+reliability = 0.87
 strengths = [
     "comprehensive scientific literature", 
     "extensive citation network", 

{local_deep_research-0.1.15 → local_deep_research-0.1.16}/src/local_deep_research/search_system.py

@@ -28,7 +28,7 @@ class AdvancedSearchSystem:
         
         # Check if search is available, log warning if not
         if self.search is None:
-            print("WARNING: Search system initialized with no search engine! Research will not be effective.")
+            logger.info("WARNING: Search system initialized with no search engine! Research will not be effective.")
             self._update_progress("WARNING: No search engine available", None, {"error": "No search engine configured properly"})
 
         
@@ -101,7 +101,7 @@ class AdvancedSearchSystem:
         self._update_progress("Knowledge compression complete", None)
         response = remove_think_tags(response.content)
         response = str(response) #+ "\n\n" + str(formatted_links)
-        print(response)
+
         return response
 
     def analyze_topic(self, query: str) -> Dict:
@@ -165,7 +165,7 @@ class AdvancedSearchSystem:
                         search_results = self.search.run(question)
                 except Exception as e:
                     error_msg = f"Error during search: {str(e)}"
-                    print(f"SEARCH ERROR: {error_msg}")
+                    logger.info(f"SEARCH ERROR: {error_msg}")
                     self._update_progress(error_msg, 
                                         int(question_progress_base + 2),
                                         {"phase": "search_error", "error": str(e)})
@@ -190,7 +190,7 @@ class AdvancedSearchSystem:
                 self._update_progress(f"Analyzing results for: {question}", 
                                      int(question_progress_base + 5),
                                      {"phase": "analysis"})
-                print("NR OF SOURCES: ", len(self.all_links_of_system))
+
 
                 try:
                     result = self.citation_handler.analyze_followup(
@@ -203,7 +203,7 @@ class AdvancedSearchSystem:
                     if links:
                         formatted_links=format_links(links=links)
                     
-                    logger.debug(f"Generated questions: {formatted_links}")                           
+                    logger.info(f"Generated questions: {formatted_links}")                           
                     if result is not None:
                         results_with_links = str(result["content"])
                         findings.append(
@@ -219,7 +219,6 @@ class AdvancedSearchSystem:
                         if settings.general.knowledge_accumulation != str(KnowledgeAccumulationApproach.NO_KNOWLEDGE.value):
                             current_knowledge = current_knowledge + "\n\n\n New: \n" + results_with_links
                         
-                        logger.info(settings.general.knowledge_accumulation)
                         if settings.general.knowledge_accumulation == str(KnowledgeAccumulationApproach.QUESTION.value):
                             logger.info("Compressing knowledge")
                             self._update_progress(f"Compress Knowledge for: {question}", 
@@ -232,7 +231,7 @@ class AdvancedSearchSystem:
                                             {"phase": "analysis_complete"})
                 except Exception as e:
                     error_msg = f"Error analyzing results: {str(e)}"
-                    print(f"ANALYSIS ERROR: {error_msg}")
+                    logger.info(f"ANALYSIS ERROR: {error_msg}")
                     self._update_progress(error_msg, 
                                         int(question_progress_base + 10),
                                         {"phase": "analysis_error", "error": str(e)})
@@ -251,7 +250,7 @@ class AdvancedSearchSystem:
                     logger.info("FINISHED ITERATION - Compressing Knowledge")
                 except Exception as e:
                     error_msg = f"Error compressing knowledge: {str(e)}"
-                    print(f"COMPRESSION ERROR: {error_msg}")
+                    logger.info(f"COMPRESSION ERROR: {error_msg}")
                     self._update_progress(error_msg, 
                                         int((iteration / total_iterations) * 100 - 3),
                                         {"phase": "compression_error", "error": str(e)})
@@ -266,7 +265,7 @@ class AdvancedSearchSystem:
                 formatted_findings = self._save_findings(findings, current_knowledge, query)
             except Exception as e:
                 error_msg = f"Error saving findings: {str(e)}"
-                print(f"SAVE ERROR: {error_msg}")
+                logger.info(f"SAVE ERROR: {error_msg}")
                 self._update_progress(error_msg, 
                                     int((iteration / total_iterations) * 100),
                                     {"phase": "save_error", "error": str(e)})

local_deep_research-0.1.16/src/local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py

@@ -0,0 +1,569 @@
+import requests
+import logging
+import json
+from typing import Dict, List, Any, Optional, Tuple, Union
+from langchain_core.language_models import BaseLLM
+import time
+import re
+from datetime import datetime
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
+from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
+from local_deep_research import config
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class SemanticScholarSearchEngine(BaseSearchEngine):
+    """
+    Semantic Scholar search engine implementation with two-phase approach.
+    Provides efficient access to scientific literature across all fields.
+    """
+    
+    def __init__(self, 
+                max_results: int = 10, 
+                api_key: Optional[str] = None,
+                year_range: Optional[Tuple[int, int]] = None,
+                get_abstracts: bool = True,
+                get_references: bool = False,
+                get_citations: bool = False,
+                get_embeddings: bool = False,
+                get_tldr: bool = True,
+                citation_limit: int = 10,
+                reference_limit: int = 10,
+                llm: Optional[BaseLLM] = None,
+                max_filtered_results: Optional[int] = None,
+                optimize_queries: bool = True,
+                max_retries: int = 5,
+                retry_backoff_factor: float = 1.0,
+                fields_of_study: Optional[List[str]] = None,
+                publication_types: Optional[List[str]] = None):
+        """
+        Initialize the Semantic Scholar search engine.
+        
+        Args:
+            max_results: Maximum number of search results
+            api_key: Semantic Scholar API key for higher rate limits (optional)
+            year_range: Optional tuple of (start_year, end_year) to filter results
+            get_abstracts: Whether to fetch abstracts for all results
+            get_references: Whether to fetch references for papers
+            get_citations: Whether to fetch citations for papers
+            get_embeddings: Whether to fetch SPECTER embeddings for papers
+            get_tldr: Whether to fetch TLDR summaries for papers
+            citation_limit: Maximum number of citations to fetch per paper
+            reference_limit: Maximum number of references to fetch per paper
+            llm: Language model for relevance filtering
+            max_filtered_results: Maximum number of results to keep after filtering
+            optimize_queries: Whether to optimize natural language queries
+            max_retries: Maximum number of retries for API requests
+            retry_backoff_factor: Backoff factor for retries
+            fields_of_study: List of fields of study to filter results
+            publication_types: List of publication types to filter results
+        """
+        # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
+        
+        self.api_key = api_key
+        self.year_range = year_range
+        self.get_abstracts = get_abstracts
+        self.get_references = get_references
+        self.get_citations = get_citations
+        self.get_embeddings = get_embeddings
+        self.get_tldr = get_tldr
+        self.citation_limit = citation_limit
+        self.reference_limit = reference_limit
+        self.optimize_queries = optimize_queries
+        self.max_retries = max_retries
+        self.retry_backoff_factor = retry_backoff_factor
+        self.fields_of_study = fields_of_study
+        self.publication_types = publication_types
+        
+        # Base API URLs
+        self.base_url = "https://api.semanticscholar.org/graph/v1"
+        self.paper_search_url = f"{self.base_url}/paper/search"
+        self.paper_details_url = f"{self.base_url}/paper"
+        
+        # Create a session with retry capabilities
+        self.session = self._create_session()
+        
+        # Rate limiting
+        self.rate_limit_wait = 1.0  # Default 1 second between requests
+        self.last_request_time = 0
+    
+    def _create_session(self) -> requests.Session:
+        """Create and configure a requests session with retry capabilities"""
+        session = requests.Session()
+        
+        # Configure automatic retries with exponential backoff
+        retry_strategy = Retry(
+            total=self.max_retries,
+            backoff_factor=self.retry_backoff_factor,
+            status_forcelist=[429, 500, 502, 503, 504],
+            allowed_methods={"HEAD", "GET", "POST", "OPTIONS"}
+        )
+        
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        session.mount("https://", adapter)
+        
+        # Set up headers
+        headers = {"Accept": "application/json"}
+        if self.api_key:
+            headers["x-api-key"] = self.api_key
+            
+        session.headers.update(headers)
+        
+        return session
+    
+    def _respect_rate_limit(self):
+        """Apply rate limiting between requests"""
+        current_time = time.time()
+        elapsed = current_time - self.last_request_time
+        
+        if elapsed < self.rate_limit_wait:
+            wait_time = self.rate_limit_wait - elapsed
+            logger.debug(f"Rate limiting: waiting {wait_time:.2f}s")
+            time.sleep(wait_time)
+            
+        self.last_request_time = time.time()
+    
+    def _make_request(self, url: str, params: Optional[Dict] = None, data: Optional[Dict] = None, 
+                     method: str = "GET") -> Dict:
+        """
+        Make a request to the Semantic Scholar API.
+        
+        Args:
+            url: API endpoint URL
+            params: Query parameters
+            data: JSON data for POST requests
+            method: HTTP method (GET or POST)
+            
+        Returns:
+            API response as dictionary
+        """
+        self._respect_rate_limit()
+        
+        try:
+            if method.upper() == "GET":
+                response = self.session.get(url, params=params, timeout=30)
+            elif method.upper() == "POST":
+                response = self.session.post(url, params=params, json=data, timeout=30)
+            else:
+                raise ValueError(f"Unsupported HTTP method: {method}")
+            
+            # Handle rate limiting manually if retry strategy fails
+            if response.status_code == 429:
+                logger.warning("Rate limit exceeded, waiting and retrying...")
+                time.sleep(2.0)  # Wait longer on rate limit
+                self.rate_limit_wait *= 1.5  # Increase wait time for future requests
+                return self._make_request(url, params, data, method)  # Retry
+            
+            response.raise_for_status()
+            return response.json()
+        except requests.RequestException as e:
+            logger.error(f"API request failed: {e}")
+            return {}
+    
+    def _optimize_query(self, query: str) -> str:
+        """
+        Optimize a natural language query for Semantic Scholar search.
+        If LLM is available, uses it to extract key terms and concepts.
+        
+        Args:
+            query: Natural language query
+            
+        Returns:
+            Optimized query string
+        """
+        if not self.llm or not self.optimize_queries:
+            return query
+            
+        try:
+            prompt = f"""Transform this natural language question into an optimized academic search query.
+
+Original query: "{query}"
+
+INSTRUCTIONS:
+1. Extract key academic concepts, technical terms, and proper nouns
+2. Remove generic words, filler words, and non-technical terms
+3. Add quotation marks around specific phrases that should be kept together
+4. Return ONLY the optimized search query with no explanation
+5. Keep it under 100 characters if possible
+
+EXAMPLE TRANSFORMATIONS:
+"What are the latest findings about mRNA vaccines and COVID-19?" → "mRNA vaccines COVID-19 recent findings"
+"How does machine learning impact climate change prediction?" → "machine learning "climate change" prediction"
+"Tell me about quantum computing approaches for encryption" → "quantum computing encryption"
+
+Return ONLY the optimized search query with no explanation.
+"""
+            
+            response = self.llm.invoke(prompt)
+            optimized_query = response.content.strip()
+            
+            # Clean up the query - remove any explanations
+            lines = optimized_query.split('\n')
+            optimized_query = lines[0].strip()
+            
+            # Safety check - if query looks too much like an explanation, use original
+            if len(optimized_query.split()) > 15 or ":" in optimized_query:
+                logger.warning("Query optimization result looks too verbose, using original")
+                return query
+                
+            logger.info(f"Original query: '{query}'")
+            logger.info(f"Optimized for search: '{optimized_query}'")
+            
+            return optimized_query
+        except Exception as e:
+            logger.error(f"Error optimizing query: {e}")
+            return query  # Fall back to original query on error
+    
+    def _direct_search(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Make a direct search request to the Semantic Scholar API.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of paper dictionaries
+        """
+        try:
+            # Configure fields to retrieve
+            fields = [
+                "paperId", 
+                "externalIds", 
+                "url", 
+                "title", 
+                "abstract", 
+                "venue", 
+                "year", 
+                "authors"
+            ]
+            
+            if self.get_tldr:
+                fields.append("tldr")
+                
+            params = {
+                "query": query,
+                "limit": min(self.max_results, 100),  # API limit is 100 per request
+                "fields": ",".join(fields)
+            }
+            
+            # Add year filter if specified
+            if self.year_range:
+                start_year, end_year = self.year_range
+                params["year"] = f"{start_year}-{end_year}"
+                
+            # Add fields of study filter if specified
+            if self.fields_of_study:
+                params["fieldsOfStudy"] = ",".join(self.fields_of_study)
+                
+            # Add publication types filter if specified
+            if self.publication_types:
+                params["publicationTypes"] = ",".join(self.publication_types)
+            
+            response = self._make_request(self.paper_search_url, params)
+            
+            if "data" in response:
+                papers = response["data"]
+                logger.info(f"Found {len(papers)} papers with direct search for query: '{query}'")
+                return papers
+            else:
+                logger.warning(f"No data in response for direct search query: '{query}'")
+                return []
+                
+        except Exception as e:
+            logger.error(f"Error in direct search: {e}")
+            return []
+    
+    def _adaptive_search(self, query: str) -> Tuple[List[Dict[str, Any]], str]:
+        """
+        Perform an adaptive search that adjusts based on result volume.
+        Uses LLM to generate better fallback queries when available.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            Tuple of (list of paper results, search strategy used)
+        """
+        # Start with a standard search
+        papers = self._direct_search(query)
+        strategy = "standard"
+        
+        # If no results, try different variations
+        if not papers:
+            # Try removing quotes to broaden search
+            if '"' in query:
+                unquoted_query = query.replace('"', '')
+                logger.info(f"No results with quoted terms, trying without quotes: {unquoted_query}")
+                papers = self._direct_search(unquoted_query)
+                
+                if papers:
+                    strategy = "unquoted"
+                    return papers, strategy
+            
+            # If LLM is available, use it to generate better fallback queries
+            if self.llm:
+                try:
+                    # Generate alternate search queries focusing on core concepts
+                    prompt = f"""You are helping refine a search query that returned no results.
+
+Original query: "{query}"
+
+The query might be too specific or use natural language phrasing that doesn't match academic paper keywords.
+
+Please provide THREE alternative search queries that:
+1. Focus on the core academic concepts
+2. Use precise terminology commonly found in academic papers
+3. Break down complex queries into more searchable components
+4. Format each as a concise keyword-focused search term (not a natural language question)
+
+Format each query on a new line with no numbering or explanation. Keep each query under 8 words and very focused.
+"""
+                    # Get the LLM's response
+                    response = self.llm.invoke(prompt)
+                    
+                    # Extract the alternative queries
+                    alt_queries = []
+                    if hasattr(response, 'content'):  # Handle various LLM response formats
+                        content = response.content
+                        alt_queries = [q.strip() for q in content.strip().split('\n') if q.strip()]
+                    elif isinstance(response, str):
+                        alt_queries = [q.strip() for q in response.strip().split('\n') if q.strip()]
+                    
+                    # Try each alternative query
+                    for alt_query in alt_queries[:3]:  # Limit to first 3 alternatives
+                        logger.info(f"Trying LLM-suggested query: {alt_query}")
+                        alt_papers = self._direct_search(alt_query)
+                        
+                        if alt_papers:
+                            logger.info(f"Found {len(alt_papers)} papers using LLM-suggested query: {alt_query}")
+                            strategy = "llm_alternative"
+                            return alt_papers, strategy
+                except Exception as e:
+                    logger.error(f"Error using LLM for query refinement: {e}")
+                    # Fall through to simpler strategies
+            
+            # Fallback: Try with the longest words (likely specific terms)
+            words = re.findall(r'\w+', query)
+            longer_words = [word for word in words if len(word) > 6]
+            if longer_words:
+                # Use up to 3 of the longest words
+                longer_words = sorted(longer_words, key=len, reverse=True)[:3]
+                key_terms_query = ' '.join(longer_words)
+                logger.info(f"Trying with key terms: {key_terms_query}")
+                papers = self._direct_search(key_terms_query)
+                
+                if papers:
+                    strategy = "key_terms"
+                    return papers, strategy
+                
+            # Final fallback: Try with just the longest word
+            if words:
+                longest_word = max(words, key=len)
+                if len(longest_word) > 5:  # Only use if it's reasonably long
+                    logger.info(f"Trying with single key term: {longest_word}")
+                    papers = self._direct_search(longest_word)
+                    
+                    if papers:
+                        strategy = "single_term"
+                        return papers, strategy
+        
+        return papers, strategy
+    
+    def _get_paper_details(self, paper_id: str) -> Dict[str, Any]:
+        """
+        Get detailed information about a specific paper.
+        
+        Args:
+            paper_id: Semantic Scholar Paper ID
+            
+        Returns:
+            Dictionary with paper details
+        """
+        try:
+            # Construct fields parameter
+            fields = [
+                "paperId", 
+                "externalIds", 
+                "corpusId", 
+                "url", 
+                "title", 
+                "abstract", 
+                "venue", 
+                "year", 
+                "authors", 
+                "fieldsOfStudy"
+            ]
+            
+            if self.get_tldr:
+                fields.append("tldr")
+                
+            if self.get_embeddings:
+                fields.append("embedding")
+                
+            # Add citation and reference fields if requested
+            if self.get_citations:
+                fields.append(f"citations.limit({self.citation_limit})")
+                
+            if self.get_references:
+                fields.append(f"references.limit({self.reference_limit})")
+            
+            # Make the request
+            url = f"{self.paper_details_url}/{paper_id}"
+            params = {"fields": ",".join(fields)}
+            
+            return self._make_request(url, params)
+            
+        except Exception as e:
+            logger.error(f"Error getting paper details for {paper_id}: {e}")
+            return {}
+
+    def _get_previews(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Get preview information for Semantic Scholar papers.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of preview dictionaries
+        """
+        logger.info(f"Getting Semantic Scholar previews for query: {query}")
+        
+        # Optimize the query if LLM is available
+        optimized_query = self._optimize_query(query)
+        
+        # Use the adaptive search approach
+        papers, strategy = self._adaptive_search(optimized_query)
+        
+        if not papers:
+            logger.warning(f"No Semantic Scholar results found")
+            return []
+        
+        # Format as previews
+        previews = []
+        for paper in papers:
+            try:
+                # Format authors - ensure we have a valid list with string values
+                authors = []
+                if "authors" in paper and paper["authors"]:
+                    authors = [author.get("name", "") for author in paper["authors"] if author and author.get("name")]
+                
+                # Ensure we have valid strings for all fields
+                paper_id = paper.get("paperId", "")
+                title = paper.get("title", "")
+                url = paper.get("url", "")
+                
+                # Handle abstract safely, ensuring we always have a string
+                abstract = paper.get("abstract")
+                snippet = ""
+                if abstract:
+                    snippet = abstract[:250] + "..." if len(abstract) > 250 else abstract
+                
+                venue = paper.get("venue", "")
+                year = paper.get("year")
+                external_ids = paper.get("externalIds", {})
+                
+                # Handle TLDR safely
+                tldr_text = ""
+                if paper.get("tldr") and isinstance(paper.get("tldr"), dict):
+                    tldr_text = paper.get("tldr", {}).get("text", "")
+                
+                # Create preview with basic information, ensuring no None values
+                preview = {
+                    "id": paper_id if paper_id else "",
+                    "title": title if title else "",
+                    "link": url if url else "",
+                    "snippet": snippet,
+                    "authors": authors,
+                    "venue": venue if venue else "",
+                    "year": year,
+                    "external_ids": external_ids if external_ids else {},
+                    "source": "Semantic Scholar",
+                    "_paper_id": paper_id if paper_id else "",
+                    "_search_strategy": strategy,
+                    "tldr": tldr_text
+                }
+                
+                # Store the full paper object for later reference
+                preview["_full_paper"] = paper
+                
+                previews.append(preview)
+            except Exception as e:
+                logger.error(f"Error processing paper preview: {e}")
+                # Continue with the next paper
+        
+        # Sort by year (newer first) if available
+        previews = sorted(
+            previews,
+            key=lambda p: p.get("year", 0) if p.get("year") is not None else 0,
+            reverse=True
+        )
+        
+        logger.info(f"Found {len(previews)} Semantic Scholar previews using strategy: {strategy}")
+        return previews
+    
+    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Get full content for the relevant Semantic Scholar papers.
+        Gets additional details like citations, references, and full metadata.
+        
+        Args:
+            relevant_items: List of relevant preview dictionaries
+            
+        Returns:
+            List of result dictionaries with full content
+        """
+        # Check if we should add full content
+        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+            logger.info("Snippet-only mode, skipping full content retrieval")
+            return relevant_items
+        
+        logger.info(f"Getting content for {len(relevant_items)} Semantic Scholar papers")
+        
+        results = []
+        for item in relevant_items:
+            result = item.copy()
+            paper_id = item.get("_paper_id", "")
+            
+            # Skip if no paper ID
+            if not paper_id:
+                results.append(result)
+                continue
+                
+            # Get paper details if citations or references are requested
+            if self.get_citations or self.get_references or self.get_embeddings:
+                paper_details = self._get_paper_details(paper_id)
+                
+                if paper_details:
+                    # Add citation information
+                    if self.get_citations and "citations" in paper_details:
+                        result["citations"] = paper_details["citations"]
+                        
+                    # Add reference information
+                    if self.get_references and "references" in paper_details:
+                        result["references"] = paper_details["references"]
+                        
+                    # Add embedding if available
+                    if self.get_embeddings and "embedding" in paper_details:
+                        result["embedding"] = paper_details["embedding"]
+                        
+                    # Add fields of study
+                    if "fieldsOfStudy" in paper_details:
+                        result["fields_of_study"] = paper_details["fieldsOfStudy"]
+            
+            # Remove temporary fields
+            if "_paper_id" in result:
+                del result["_paper_id"]
+            if "_search_strategy" in result:
+                del result["_search_strategy"]
+            if "_full_paper" in result:
+                del result["_full_paper"]
+            
+            results.append(result)
+        
+        return results