local-deep-research 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py

@@ -0,0 +1,1128 @@
+import requests
+import logging
+import json
+from typing import Dict, List, Any, Optional, Tuple, Union
+from langchain_core.language_models import BaseLLM
+import time
+import re
+from datetime import datetime
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
+from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
+from local_deep_research import config
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class SemanticScholarSearchEngine(BaseSearchEngine):
+    """
+    Semantic Scholar search engine implementation with two-phase approach.
+    Provides efficient access to scientific literature across all fields.
+    """
+    
+    def __init__(self, 
+                max_results: int = 10, 
+                api_key: Optional[str] = None,
+                year_range: Optional[Tuple[int, int]] = None,
+                get_abstracts: bool = True,
+                get_references: bool = False,
+                get_citations: bool = False,
+                get_embeddings: bool = False,
+                get_tldr: bool = True,
+                citation_limit: int = 10,
+                reference_limit: int = 10,
+                llm: Optional[BaseLLM] = None,
+                max_filtered_results: Optional[int] = None,
+                optimize_queries: bool = True,
+                max_retries: int = 5,
+                retry_backoff_factor: float = 1.0,
+                fields_of_study: Optional[List[str]] = None,
+                publication_types: Optional[List[str]] = None):
+        """
+        Initialize the Semantic Scholar search engine.
+        
+        Args:
+            max_results: Maximum number of search results
+            api_key: Semantic Scholar API key for higher rate limits (optional)
+            year_range: Optional tuple of (start_year, end_year) to filter results
+            get_abstracts: Whether to fetch abstracts for all results
+            get_references: Whether to fetch references for papers
+            get_citations: Whether to fetch citations for papers
+            get_embeddings: Whether to fetch SPECTER embeddings for papers
+            get_tldr: Whether to fetch TLDR summaries for papers
+            citation_limit: Maximum number of citations to fetch per paper
+            reference_limit: Maximum number of references to fetch per paper
+            llm: Language model for relevance filtering
+            max_filtered_results: Maximum number of results to keep after filtering
+            optimize_queries: Whether to optimize natural language queries
+            max_retries: Maximum number of retries for API requests
+            retry_backoff_factor: Backoff factor for retries
+            fields_of_study: List of fields of study to filter results
+            publication_types: List of publication types to filter results
+        """
+        # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
+        self.api_key = api_key
+        self.year_range = year_range
+        self.get_abstracts = get_abstracts
+        self.get_references = get_references
+        self.get_citations = get_citations
+        self.get_embeddings = get_embeddings
+        self.get_tldr = get_tldr
+        self.citation_limit = citation_limit
+        self.reference_limit = reference_limit
+        self.optimize_queries = optimize_queries
+        self.max_retries = max_retries
+        self.retry_backoff_factor = retry_backoff_factor
+        self.fields_of_study = fields_of_study
+        self.publication_types = publication_types
+        
+        # Base API URLs
+        self.base_url = "https://api.semanticscholar.org/graph/v1"
+        self.paper_search_url = f"{self.base_url}/paper/search"
+        self.paper_bulk_search_url = f"{self.base_url}/paper/search/bulk"
+        self.paper_batch_url = f"{self.base_url}/paper/batch"
+        self.paper_details_url = f"{self.base_url}/paper"
+        self.author_search_url = f"{self.base_url}/author/search"
+        self.author_details_url = f"{self.base_url}/author"
+        self.recommendations_url = "https://api.semanticscholar.org/recommendations/v1/papers"
+        self.datasets_url = "https://api.semanticscholar.org/datasets/v1"
+        
+        # Create a session with retry capabilities
+        self.session = self._create_session()
+        
+        # Rate limiting
+        self.rate_limit_wait = 1.0  # Default 1 second between requests
+        self.last_request_time = 0
+    
+    def _create_session(self) -> requests.Session:
+        """Create and configure a requests session with retry capabilities"""
+        session = requests.Session()
+        
+        # Configure automatic retries with exponential backoff
+        retry_strategy = Retry(
+            total=self.max_retries,
+            backoff_factor=self.retry_backoff_factor,
+            status_forcelist=[429, 500, 502, 503, 504],
+            allowed_methods={"HEAD", "GET", "POST", "OPTIONS"}
+        )
+        
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        session.mount("https://", adapter)
+        
+        # Set up headers
+        headers = {"Accept": "application/json"}
+        if self.api_key:
+            headers["x-api-key"] = self.api_key
+            
+        session.headers.update(headers)
+        
+        return session
+    
+    def _respect_rate_limit(self):
+        """Apply rate limiting between requests"""
+        current_time = time.time()
+        elapsed = current_time - self.last_request_time
+        
+        if elapsed < self.rate_limit_wait:
+            wait_time = self.rate_limit_wait - elapsed
+            logger.debug(f"Rate limiting: waiting {wait_time:.2f}s")
+            time.sleep(wait_time)
+            
+        self.last_request_time = time.time()
+    
+    def _get_headers(self) -> Dict[str, str]:
+        """Get the headers for API requests"""
+        headers = {"Accept": "application/json"}
+        
+        if self.api_key:
+            headers["x-api-key"] = self.api_key
+            
+        return headers
+    
+    def _make_request(self, url: str, params: Optional[Dict] = None, data: Optional[Dict] = None, 
+                     method: str = "GET") -> Dict:
+        """
+        Make a request to the Semantic Scholar API.
+        
+        Args:
+            url: API endpoint URL
+            params: Query parameters
+            data: JSON data for POST requests
+            method: HTTP method (GET or POST)
+            
+        Returns:
+            API response as dictionary
+        """
+        self._respect_rate_limit()
+        
+        try:
+            if method.upper() == "GET":
+                response = self.session.get(url, params=params, timeout=30)
+            elif method.upper() == "POST":
+                response = self.session.post(url, params=params, json=data, timeout=30)
+            else:
+                raise ValueError(f"Unsupported HTTP method: {method}")
+            
+            # Handle rate limiting manually if retry strategy fails
+            if response.status_code == 429:
+                logger.warning("Rate limit exceeded, waiting and retrying...")
+                time.sleep(2.0)  # Wait longer on rate limit
+                self.rate_limit_wait *= 1.5  # Increase wait time for future requests
+                return self._make_request(url, params, data, method)  # Retry
+            
+            response.raise_for_status()
+            return response.json()
+        except requests.RequestException as e:
+            logger.error(f"API request failed: {e}")
+            return {}
+    
+    def _optimize_query(self, query: str) -> str:
+        """
+        Optimize a natural language query for Semantic Scholar search.
+        If LLM is available, uses it to extract key terms and concepts.
+        
+        Args:
+            query: Natural language query
+            
+        Returns:
+            Optimized query string
+        """
+        if not self.llm or not self.optimize_queries:
+            return query
+            
+        try:
+            prompt = f"""Transform this natural language question into an optimized academic search query.
+
+Original query: "{query}"
+
+INSTRUCTIONS:
+1. Extract key academic concepts, technical terms, and proper nouns
+2. Remove generic words, filler words, and non-technical terms
+3. Add quotation marks around specific phrases that should be kept together
+4. Return ONLY the optimized search query with no explanation
+5. Keep it under 100 characters if possible
+
+EXAMPLE TRANSFORMATIONS:
+"What are the latest findings about mRNA vaccines and COVID-19?" → "mRNA vaccines COVID-19 recent findings"
+"How does machine learning impact climate change prediction?" → "machine learning "climate change" prediction"
+"Tell me about quantum computing approaches for encryption" → "quantum computing encryption"
+
+Return ONLY the optimized search query with no explanation.
+"""
+            
+            response = self.llm.invoke(prompt)
+            optimized_query = response.content.strip()
+            
+            # Clean up the query - remove any explanations
+            lines = optimized_query.split('\n')
+            optimized_query = lines[0].strip()
+            
+            # Safety check - if query looks too much like an explanation, use original
+            if len(optimized_query.split()) > 15 or ":" in optimized_query:
+                logger.warning("Query optimization result looks too verbose, using original")
+                return query
+                
+            logger.info(f"Original query: '{query}'")
+            logger.info(f"Optimized for Semantic Scholar: '{optimized_query}'")
+            
+            return optimized_query
+        except Exception as e:
+            logger.error(f"Error optimizing query: {e}")
+            return query  # Fall back to original query on error
+    
+    def _search_papers(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Search for papers matching the query.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of paper dictionaries
+        """
+        try:
+            fields = [
+                "paperId", 
+                "externalIds", 
+                "url", 
+                "title", 
+                "abstract", 
+                "venue", 
+                "year", 
+                "authors"
+            ]
+            
+            if self.get_tldr:
+                fields.append("tldr")
+                
+            params = {
+                "query": query,
+                "limit": min(self.max_results, 100),  # Regular search API can return up to 100 results
+                "fields": ",".join(fields)
+            }
+            
+            # Add year filter if specified
+            if self.year_range:
+                start_year, end_year = self.year_range
+                params["year"] = f"{start_year}-{end_year}"
+                
+            # Add fields of study filter if specified
+            if self.fields_of_study:
+                params["fieldsOfStudy"] = ",".join(self.fields_of_study)
+                
+            # Add publication types filter if specified
+            if self.publication_types:
+                params["publicationTypes"] = ",".join(self.publication_types)
+            
+            response = self._make_request(self.paper_search_url, params)
+            
+            if "data" in response:
+                papers = response["data"]
+                logger.info(f"Found {len(papers)} papers matching query: '{query}'")
+                return papers
+            else:
+                logger.warning(f"No data in response for query: '{query}'")
+                return []
+                
+        except Exception as e:
+            logger.error(f"Error searching papers: {e}")
+            return []
+            
+    def _search_papers_bulk(self, query: str, limit: int = 1000) -> List[Dict[str, Any]]:
+        """
+        Search for papers using the bulk search API, which can return up to 1000 papers.
+        
+        Args:
+            query: The search query
+            limit: Maximum number of results (up to 1000)
+            
+        Returns:
+            List of paper dictionaries
+        """
+        try:
+            fields = [
+                "paperId", 
+                "externalIds", 
+                "url", 
+                "title", 
+                "abstract", 
+                "venue", 
+                "year", 
+                "authors",
+                "fieldsOfStudy"
+            ]
+            
+            if self.get_tldr:
+                fields.append("tldr")
+                
+            params = {
+                "query": query,
+                "limit": min(limit, 1000),  # Bulk search API can return up to 1000 results
+                "fields": ",".join(fields)
+            }
+            
+            # Add year filter if specified
+            if self.year_range:
+                start_year, end_year = self.year_range
+                params["year"] = f"{start_year}-{end_year}"
+                
+            # Add fields of study filter if specified
+            if self.fields_of_study:
+                params["fieldsOfStudy"] = ",".join(self.fields_of_study)
+                
+            # Add publication types filter if specified
+            if self.publication_types:
+                params["publicationTypes"] = ",".join(self.publication_types)
+            
+            response = self._make_request(self.paper_bulk_search_url, params)
+            
+            if "data" in response:
+                papers = response["data"]
+                logger.info(f"Found {len(papers)} papers using bulk search for query: '{query}'")
+                total_count = response.get("total", 0)
+                logger.info(f"Total available results: {total_count}")
+                
+                # Handle continuation token for pagination if needed
+                if "token" in response and len(papers) < min(total_count, limit):
+                    token = response["token"]
+                    logger.info(f"Continuation token available: {token}")
+                    # The caller would need to handle continuation tokens for pagination
+                
+                return papers
+            else:
+                logger.warning(f"No data in response for bulk query: '{query}'")
+                return []
+                
+        except Exception as e:
+            logger.error(f"Error in bulk paper search: {e}")
+            return []
+    
+    def _get_paper_details(self, paper_id: str) -> Dict[str, Any]:
+        """
+        Get detailed information about a specific paper.
+        
+        Args:
+            paper_id: Semantic Scholar Paper ID
+            
+        Returns:
+            Dictionary with paper details
+        """
+        try:
+            # Construct fields parameter
+            fields = [
+                "paperId", 
+                "externalIds", 
+                "corpusId", 
+                "url", 
+                "title", 
+                "abstract", 
+                "venue", 
+                "year", 
+                "authors", 
+                "fieldsOfStudy"
+            ]
+            
+            if self.get_tldr:
+                fields.append("tldr")
+                
+            if self.get_embeddings:
+                fields.append("embedding")
+                
+            # Add citation and reference fields if requested
+            if self.get_citations:
+                fields.append(f"citations.limit({self.citation_limit})")
+                
+            if self.get_references:
+                fields.append(f"references.limit({self.reference_limit})")
+            
+            # Make the request
+            url = f"{self.paper_details_url}/{paper_id}"
+            params = {"fields": ",".join(fields)}
+            
+            return self._make_request(url, params)
+            
+        except Exception as e:
+            logger.error(f"Error getting paper details for {paper_id}: {e}")
+            return {}
+    
+
+    def _adaptive_search(self, query: str) -> Tuple[List[Dict[str, Any]], str]:
+        """
+        Perform an adaptive search that adjusts based on result volume.
+        Uses LLM to generate better fallback queries when available.
+        
+        Args:
+            query: The search query (already optimized)
+            
+        Returns:
+            Tuple of (list of paper results, search strategy used)
+        """
+        # Start with a standard search
+        papers = self._search_papers(query)
+        strategy = "standard"
+        
+        # If no results, try different variations
+        if not papers:
+            # Try removing quotes to broaden search
+            if '"' in query:
+                unquoted_query = query.replace('"', '')
+                logger.info(f"No results with quoted terms, trying without quotes: {unquoted_query}")
+                papers = self._search_papers(unquoted_query)
+                
+                if papers:
+                    strategy = "unquoted"
+                    return papers, strategy
+            
+            # If LLM is available, use it to generate better fallback queries
+            if self.llm:
+                try:
+                    # Generate alternate search queries focusing on core concepts
+                    prompt = f"""You are helping refine a search query for academic papers related to cancer research that returned no results.
+
+    Original query: "{query}"
+
+    The query might be too specific, contain future dates, or use natural language phrasing that doesn't match academic paper keywords.
+
+    Please provide THREE alternative search queries that:
+    1. Focus on the core academic concepts about cancer treatment, research, or therapies
+    2. Remove future dates or references to "latest" or "current" (replace with terms like "recent" or "novel")
+    3. Use precise medical/scientific terminology commonly found in academic papers
+    4. Break down complex queries into more searchable components
+    5. Format each as a concise keyword-focused search term (not a natural language question)
+
+    Format each query on a new line with no numbering or explanation. Keep each query under 8 words and very focused.
+    """
+                    # Get the LLM's response
+                    response = self.llm.invoke(prompt)
+                    
+                    # Extract the alternative queries
+                    alt_queries = []
+                    if hasattr(response, 'content'):  # Handle various LLM response formats
+                        content = response.content
+                        alt_queries = [q.strip() for q in content.strip().split('\n') if q.strip()]
+                    elif isinstance(response, str):
+                        alt_queries = [q.strip() for q in response.strip().split('\n') if q.strip()]
+                    
+                    # Try each alternative query
+                    for alt_query in alt_queries[:3]:  # Limit to first 3 alternatives
+                        logger.info(f"Trying LLM-suggested query: {alt_query}")
+                        alt_papers = self._search_papers(alt_query)
+                        
+                        if alt_papers:
+                            logger.info(f"Found {len(alt_papers)} papers using LLM-suggested query: {alt_query}")
+                            strategy = "llm_alternative"
+                            return alt_papers, strategy
+                except Exception as e:
+                    logger.error(f"Error using LLM for query refinement: {e}")
+                    # Fall through to simpler strategies
+            
+            # Fallback 1: Try extracting important cancer-related terms
+            cancer_terms = ["cancer", "tumor", "oncology", "carcinoma", "sarcoma", "leukemia", 
+                        "lymphoma", "metastasis", "therapy", "immunotherapy", "targeted", 
+                        "treatment", "drug", "clinical", "trial", "biomarker"]
+            
+            words = re.findall(r'\b\w+\b', query.lower())
+            important_terms = [word for word in words if word in cancer_terms or len(word) > 7]
+            
+            if important_terms:
+                important_query = ' '.join(important_terms[:5])  # Limit to 5 terms
+                logger.info(f"Trying with important cancer terms: {important_query}")
+                papers = self._search_papers(important_query)
+                
+                if papers:
+                    strategy = "cancer_terms"
+                    return papers, strategy
+                    
+            # Fallback 2: Try with just specific cancer types or treatment modalities
+            cancer_types = ["breast", "lung", "colorectal", "prostate", "melanoma", "lymphoma", 
+                        "leukemia", "myeloma", "sarcoma", "glioblastoma"]
+            treatment_types = ["immunotherapy", "chemotherapy", "radiotherapy", "targeted", 
+                            "surgery", "vaccine", "antibody", "CAR-T", "inhibitor"]
+            
+            cancer_matches = [word for word in words if word in cancer_types]
+            treatment_matches = [word for word in words if word in treatment_types]
+            
+            if cancer_matches and treatment_matches:
+                specific_query = f"{cancer_matches[0]} {treatment_matches[0]}"
+                logger.info(f"Trying with specific cancer-treatment pair: {specific_query}")
+                papers = self._search_papers(specific_query)
+                
+                if papers:
+                    strategy = "specific_pair"
+                    return papers, strategy
+            
+            # Fallback 3: Extract the longest word (likely a specific term)
+            longest_word = max(re.findall(r'\w+', query), key=len, default='')
+            if len(longest_word) > 6:
+                logger.info(f"Trying with primary keyword: {longest_word}")
+                papers = self._search_papers(longest_word)
+                
+                if papers:
+                    strategy = "primary_keyword"
+                    return papers, strategy
+        
+        return papers, strategy
+
+
+    def _get_previews(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Get preview information for Semantic Scholar papers.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of preview dictionaries
+        """
+        logger.info(f"Getting Semantic Scholar previews for query: {query}")
+        
+        # Optimize the query if LLM is available
+        optimized_query = self._optimize_query(query)
+        
+        # Perform adaptive search
+        papers, strategy = self._adaptive_search(optimized_query)
+        
+        if not papers:
+            logger.warning(f"No Semantic Scholar results found using strategy: {strategy}")
+            return []
+        
+        # Format as previews
+        previews = []
+        for paper in papers:
+            try:
+                # Format authors - ensure we have a valid list with string values
+                authors = []
+                if "authors" in paper and paper["authors"]:
+                    authors = [author.get("name", "") for author in paper["authors"] if author and author.get("name")]
+                
+                # Ensure we have valid strings for all fields
+                paper_id = paper.get("paperId", "")
+                title = paper.get("title", "")
+                url = paper.get("url", "")
+                
+                # Handle abstract safely, ensuring we always have a string
+                abstract = paper.get("abstract")
+                snippet = ""
+                if abstract:
+                    snippet = abstract[:250] + "..." if len(abstract) > 250 else abstract
+                
+                venue = paper.get("venue", "")
+                year = paper.get("year")
+                external_ids = paper.get("externalIds", {})
+                
+                # Handle TLDR safely
+                tldr_text = ""
+                if paper.get("tldr") and isinstance(paper.get("tldr"), dict):
+                    tldr_text = paper.get("tldr", {}).get("text", "")
+                
+                # Create preview with basic information, ensuring no None values
+                preview = {
+                    "id": paper_id if paper_id else "",
+                    "title": title if title else "",
+                    "link": url if url else "",
+                    "snippet": snippet,  # Already handled above
+                    "authors": authors,  # List of strings, safe to use directly
+                    "venue": venue if venue else "",
+                    "year": year,  # Can be None, handled in downstream processing
+                    "external_ids": external_ids if external_ids else {},
+                    "source": "Semantic Scholar",
+                    "_paper_id": paper_id if paper_id else "",
+                    "_search_strategy": strategy,
+                    "tldr": tldr_text
+                }
+                
+                # Store the full paper object for later reference
+                preview["_full_paper"] = paper
+                
+                previews.append(preview)
+            except Exception as e:
+                logger.error(f"Error processing paper preview: {e}")
+                # Continue with the next paper
+        
+        logger.info(f"Found {len(previews)} Semantic Scholar previews using strategy: {strategy}")
+        return previews
+    
+    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Get full content for the relevant Semantic Scholar papers.
+        Gets additional details like citations, references, and full metadata.
+        
+        Args:
+            relevant_items: List of relevant preview dictionaries
+            
+        Returns:
+            List of result dictionaries with full content
+        """
+        # Check if we should add full content
+        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+            logger.info("Snippet-only mode, skipping full content retrieval")
+            return relevant_items
+        
+        logger.info(f"Getting content for {len(relevant_items)} Semantic Scholar papers")
+        
+        results = []
+        for item in relevant_items:
+            result = item.copy()
+            paper_id = item.get("_paper_id", "")
+            
+            # Skip if no paper ID
+            if not paper_id:
+                results.append(result)
+                continue
+                
+            # Get paper details if citations or references are requested
+            if self.get_citations or self.get_references or self.get_embeddings:
+                paper_details = self._get_paper_details(paper_id)
+                
+                if paper_details:
+                    # Add citation information
+                    if self.get_citations and "citations" in paper_details:
+                        result["citations"] = paper_details["citations"]
+                        
+                    # Add reference information
+                    if self.get_references and "references" in paper_details:
+                        result["references"] = paper_details["references"]
+                        
+                    # Add embedding if available
+                    if self.get_embeddings and "embedding" in paper_details:
+                        result["embedding"] = paper_details["embedding"]
+                        
+                    # Add fields of study
+                    if "fieldsOfStudy" in paper_details:
+                        result["fields_of_study"] = paper_details["fieldsOfStudy"]
+            
+            # Remove temporary fields
+            if "_paper_id" in result:
+                del result["_paper_id"]
+            if "_search_strategy" in result:
+                del result["_search_strategy"]
+            if "_full_paper" in result:
+                del result["_full_paper"]
+            
+            results.append(result)
+        
+        return results
+    
+    def search_by_author(self, author_name: str, max_results: Optional[int] = None) -> List[Dict[str, Any]]:
+        """
+        Search for papers by a specific author.
+        
+        Args:
+            author_name: Name of the author
+            max_results: Maximum number of results (defaults to self.max_results)
+            
+        Returns:
+            List of papers by the author
+        """
+        original_max_results = self.max_results
+        
+        try:
+            if max_results:
+                self.max_results = max_results
+            
+            # First search for the author
+            params = {
+                "query": author_name,
+                "limit": 5  # Limit to top 5 author matches
+            }
+            
+            response = self._make_request(self.author_search_url, params)
+            
+            if "data" not in response or not response["data"]:
+                logger.warning(f"No authors found matching: {author_name}")
+                return []
+                
+            # Use the first (best) author match
+            author = response["data"][0]
+            author_id = author.get("authorId")
+            
+            if not author_id:
+                logger.warning(f"No valid author ID found for: {author_name}")
+                return []
+                
+            # Get the author's papers
+            fields = [
+                "papers.paperId", 
+                "papers.title", 
+                "papers.abstract", 
+                "papers.venue", 
+                "papers.year", 
+                "papers.authors"
+            ]
+            
+            if self.get_tldr:
+                fields.append("papers.tldr")
+                
+            url = f"{self.author_details_url}/{author_id}"
+            author_params = {
+                "fields": ",".join(fields)
+            }
+            
+            author_data = self._make_request(url, author_params)
+            
+            if "papers" not in author_data or not author_data["papers"]:
+                logger.warning(f"No papers found for author: {author_name}")
+                return []
+                
+            # Format as paper results
+            papers = author_data["papers"][:self.max_results]
+            
+            # Convert to standard results format
+            results = []
+            for paper in papers:
+                # Format authors
+                authors = []
+                if "authors" in paper and paper["authors"]:
+                    authors = [author.get("name", "") for author in paper["authors"]]
+                
+                result = {
+                    "id": paper.get("paperId", ""),
+                    "title": paper.get("title", ""),
+                    "link": f"https://www.semanticscholar.org/paper/{paper.get('paperId', '')}",
+                    "snippet": paper.get("abstract", "")[:250] + "..." if paper.get("abstract", "") and len(paper.get("abstract", "")) > 250 else paper.get("abstract", ""),
+                    "authors": authors,
+                    "venue": paper.get("venue", ""),
+                    "year": paper.get("year"),
+                    "source": "Semantic Scholar",
+                    
+                    # Include TLDR if available
+                    "tldr": paper.get("tldr", {}).get("text", "") if paper.get("tldr") else ""
+                }
+                
+                results.append(result)
+            
+            # Add citations and references if needed
+            if self.get_citations or self.get_references:
+                results = self._get_full_content(results)
+                
+            return results
+            
+        finally:
+            # Restore original value
+            self.max_results = original_max_results
+    
+    def search_by_venue(self, venue_name: str, max_results: Optional[int] = None) -> List[Dict[str, Any]]:
+        """
+        Search for papers in a specific venue.
+        
+        Args:
+            venue_name: Name of the venue (conference or journal)
+            max_results: Maximum number of results (defaults to self.max_results)
+            
+        Returns:
+            List of papers from the venue
+        """
+        original_max_results = self.max_results
+        
+        try:
+            if max_results:
+                self.max_results = max_results
+                
+            # Semantic Scholar doesn't have a dedicated venue search API
+            # So we search for papers with the venue in the query
+            query = f'venue:"{venue_name}"'
+            return self.run(query)
+            
+        finally:
+            # Restore original value
+            self.max_results = original_max_results
+    
+    def search_by_year(self, query: str, year: int, max_results: Optional[int] = None) -> List[Dict[str, Any]]:
+        """
+        Search for papers from a specific year matching the query.
+        
+        Args:
+            query: The search query
+            year: Publication year
+            max_results: Maximum number of results (defaults to self.max_results)
+            
+        Returns:
+            List of papers from the specified year matching the query
+        """
+        original_max_results = self.max_results
+        original_year_range = self.year_range
+        
+        try:
+            if max_results:
+                self.max_results = max_results
+            
+            # Set year range for this search
+            self.year_range = (year, year)
+            
+            return self.run(query)
+            
+        finally:
+            # Restore original values
+            self.max_results = original_max_results
+            self.year_range = original_year_range
+    
+    def search_by_field(self, query: str, field_of_study: str, max_results: Optional[int] = None) -> List[Dict[str, Any]]:
+        """
+        Search for papers in a specific field of study.
+        
+        Args:
+            query: The search query
+            field_of_study: Field of study (e.g., "Computer Science", "Medicine")
+            max_results: Maximum number of results (defaults to self.max_results)
+            
+        Returns:
+            List of papers in the specified field matching the query
+        """
+        original_max_results = self.max_results
+        
+        try:
+            if max_results:
+                self.max_results = max_results
+                
+            # Add field of study to query
+            field_query = f'{query} fieldofstudy:"{field_of_study}"'
+            return self.run(field_query)
+            
+        finally:
+            # Restore original value
+            self.max_results = original_max_results
+    
+    def get_paper_by_id(self, paper_id: str) -> Dict[str, Any]:
+        """
+        Get a specific paper by its Semantic Scholar ID.
+        
+        Args:
+            paper_id: Semantic Scholar paper ID
+            
+        Returns:
+            Dictionary with paper information
+        """
+        paper_details = self._get_paper_details(paper_id)
+        
+        if not paper_details:
+            return {}
+            
+        # Format authors
+        authors = []
+        if "authors" in paper_details and paper_details["authors"]:
+            authors = [author.get("name", "") for author in paper_details["authors"]]
+        
+        # Create formatted result
+        result = {
+            "id": paper_details.get("paperId", ""),
+            "title": paper_details.get("title", ""),
+            "link": paper_details.get("url", ""),
+            "abstract": paper_details.get("abstract", ""),
+            "authors": authors,
+            "venue": paper_details.get("venue", ""),
+            "year": paper_details.get("year"),
+            "fields_of_study": paper_details.get("fieldsOfStudy", []),
+            "external_ids": paper_details.get("externalIds", {}),
+            "source": "Semantic Scholar",
+            
+            # Include TLDR if available
+            "tldr": paper_details.get("tldr", {}).get("text", "") if paper_details.get("tldr") else ""
+        }
+        
+        # Add citations and references if requested
+        if self.get_citations and "citations" in paper_details:
+            result["citations"] = paper_details["citations"]
+            
+        if self.get_references and "references" in paper_details:
+            result["references"] = paper_details["references"]
+            
+        # Add embedding if requested
+        if self.get_embeddings and "embedding" in paper_details:
+            result["embedding"] = paper_details["embedding"]
+            
+        return result
+    
+    def get_paper_by_doi(self, doi: str) -> Dict[str, Any]:
+        """
+        Get a paper by its DOI.
+        
+        Args:
+            doi: Digital Object Identifier
+            
+        Returns:
+            Dictionary with paper information
+        """
+        try:
+            # The Semantic Scholar API supports DOI lookup
+            url = f"{self.paper_details_url}/DOI:{doi}"
+            fields = [
+                "paperId", 
+                "externalIds", 
+                "url", 
+                "title", 
+                "abstract", 
+                "venue", 
+                "year", 
+                "authors", 
+                "fieldsOfStudy"
+            ]
+            
+            if self.get_tldr:
+                fields.append("tldr")
+                
+            if self.get_embeddings:
+                fields.append("embedding")
+                
+            # Add citation and reference fields if requested
+            if self.get_citations:
+                fields.append(f"citations.limit({self.citation_limit})")
+                
+            if self.get_references:
+                fields.append(f"references.limit({self.reference_limit})")
+                
+            params = {"fields": ",".join(fields)}
+            paper_details = self._make_request(url, params)
+            
+            if not paper_details:
+                return {}
+                
+            # Format the paper info the same way as get_paper_by_id
+            # Format authors
+            authors = []
+            if "authors" in paper_details and paper_details["authors"]:
+                authors = [author.get("name", "") for author in paper_details["authors"]]
+            
+            # Create formatted result
+            result = {
+                "id": paper_details.get("paperId", ""),
+                "title": paper_details.get("title", ""),
+                "link": paper_details.get("url", ""),
+                "abstract": paper_details.get("abstract", ""),
+                "authors": authors,
+                "venue": paper_details.get("venue", ""),
+                "year": paper_details.get("year"),
+                "fields_of_study": paper_details.get("fieldsOfStudy", []),
+                "external_ids": paper_details.get("externalIds", {}),
+                "source": "Semantic Scholar",
+                
+                # Include TLDR if available
+                "tldr": paper_details.get("tldr", {}).get("text", "") if paper_details.get("tldr") else ""
+            }
+            
+            # Add citations and references if requested
+            if self.get_citations and "citations" in paper_details:
+                result["citations"] = paper_details["citations"]
+                
+            if self.get_references and "references" in paper_details:
+                result["references"] = paper_details["references"]
+                
+            # Add embedding if requested
+            if self.get_embeddings and "embedding" in paper_details:
+                result["embedding"] = paper_details["embedding"]
+                
+            return result
+            
+        except Exception as e:
+            logger.error(f"Error getting paper by DOI {doi}: {e}")
+            return {}
+    
+    def get_papers_batch(self, paper_ids: List[str], fields: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        """
+        Get details for multiple papers in a single batch request.
+        
+        Args:
+            paper_ids: List of paper IDs (Semantic Scholar IDs, DOIs, arXiv IDs, etc.)
+            fields: Fields to include in the response
+            
+        Returns:
+            List of paper details
+        """
+        if not paper_ids:
+            return []
+            
+        if fields is None:
+            fields = [
+                "paperId", 
+                "externalIds", 
+                "url", 
+                "title", 
+                "abstract", 
+                "venue", 
+                "year", 
+                "authors",
+                "referenceCount",
+                "citationCount"
+            ]
+            
+            if self.get_tldr:
+                fields.append("tldr")
+        
+        try:
+            # Construct request params
+            params = {
+                "fields": ",".join(fields)
+            }
+            
+            # Make POST request with paper IDs in the body
+            response = self._make_request(
+                self.paper_batch_url,
+                params=params,
+                data={"ids": paper_ids},
+                method="POST"
+            )
+            
+            if isinstance(response, list):
+                return response
+            else:
+                logger.warning("Unexpected response format from batch API")
+                return []
+                
+        except Exception as e:
+            logger.error(f"Error in batch paper lookup: {e}")
+            return []
+    
+    def get_paper_recommendations(self, 
+                                 positive_paper_ids: List[str], 
+                                 negative_paper_ids: Optional[List[str]] = None,
+                                 max_results: Optional[int] = None) -> List[Dict[str, Any]]:
+        """
+        Get recommended papers based on positive and negative examples.
+        
+        Args:
+            positive_paper_ids: List of paper IDs to use as positive examples
+            negative_paper_ids: Optional list of paper IDs to use as negative examples
+            max_results: Maximum number of recommendations to return
+            
+        Returns:
+            List of recommended papers
+        """
+        if not positive_paper_ids:
+            return []
+            
+        limit = max_results or self.max_results
+        
+        try:
+            # Construct the request payload
+            payload = {
+                "positivePaperIds": positive_paper_ids
+            }
+            
+            if negative_paper_ids:
+                payload["negativePaperIds"] = negative_paper_ids
+            
+            # Define fields to include in the response
+            fields = [
+                "paperId", 
+                "externalIds", 
+                "url", 
+                "title", 
+                "abstract", 
+                "venue", 
+                "year", 
+                "authors"
+            ]
+            
+            if self.get_tldr:
+                fields.append("tldr")
+                
+            # Request parameters
+            params = {
+                "fields": ",".join(fields),
+                "limit": limit
+            }
+            
+            # Make POST request to recommendations endpoint
+            response = self._make_request(
+                self.recommendations_url,
+                params=params,
+                data=payload,
+                method="POST"
+            )
+            
+            if "recommendedPapers" not in response:
+                return []
+                
+            papers = response["recommendedPapers"]
+            
+            # Format as standard results
+            results = []
+            for paper in papers:
+                # Format authors
+                authors = []
+                if "authors" in paper and paper["authors"]:
+                    authors = [author.get("name", "") for author in paper["authors"]]
+                
+                result = {
+                    "id": paper.get("paperId", ""),
+                    "title": paper.get("title", ""),
+                    "link": paper.get("url", ""),
+                    "snippet": paper.get("abstract", "")[:250] + "..." if paper.get("abstract", "") and len(paper.get("abstract", "")) > 250 else paper.get("abstract", ""),
+                    "authors": authors,
+                    "venue": paper.get("venue", ""),
+                    "year": paper.get("year"),
+                    "source": "Semantic Scholar",
+                    
+                    # Include TLDR if available
+                    "tldr": paper.get("tldr", {}).get("text", "") if paper.get("tldr") else ""
+                }
+                
+                results.append(result)
+            
+            return results
+            
+        except Exception as e:
+            logger.error(f"Error getting paper recommendations: {e}")
+            return []