aiagents4pharma 1.41.0__py3-none-any.whl → 1.43.0__py3-none-any.whl

aiagents4pharma/talk2scholars/tools/paper_download/__init__.py

@@ -5,10 +5,8 @@ biorxiv and medrxiv.
 """
 
 # Import modules
-from . import download_arxiv_input
+from . import paper_downloader
 
 __all__ = [
-    "download_arxiv_input",
-    "download_biorxiv_input",
-    "download_medrxiv_input",
+    "paper_downloader",
 ]

aiagents4pharma/talk2scholars/tools/paper_download/paper_downloader.py

@@ -0,0 +1,457 @@
+#!/usr/bin/env python3
+"""
+Unified paper download tool for LangGraph.
+Supports downloading papers from arXiv, medRxiv, bioRxiv, and PubMed through a single interface.
+"""
+
+import logging
+import threading
+from typing import Annotated, Any, List, Literal, Optional
+
+import hydra
+from hydra.core.global_hydra import GlobalHydra
+from omegaconf import OmegaConf
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.types import Command
+from pydantic import BaseModel, Field
+from .utils.arxiv_downloader import ArxivDownloader
+from .utils.base_paper_downloader import BasePaperDownloader
+from .utils.biorxiv_downloader import BiorxivDownloader
+from .utils.medrxiv_downloader import MedrxivDownloader
+from .utils.pubmed_downloader import PubmedDownloader
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class UnifiedPaperDownloadInput(BaseModel):
+    """Input schema for the unified paper download tool."""
+
+    service: Optional[Literal["arxiv", "medrxiv", "biorxiv", "pubmed"]] = Field(
+        default=None,
+        description=(
+            "Paper service to download from: 'arxiv', 'medrxiv', 'biorxiv', or 'pubmed'. "
+            "If not specified, uses the configured default service."
+        ),
+    )
+    identifiers: List[str] = Field(
+        description=(
+            "List of paper identifiers. Format depends on service:\n"
+            "- arxiv: arXiv IDs (e.g., ['1234.5678', '2301.12345'])\n"
+            "- medrxiv: DOIs (e.g., ['10.1101/2020.09.09.20191205'])\n"
+            "- biorxiv: DOIs (e.g., ['10.1101/2020.09.09.20191205'])\n"
+            "- pubmed: PMIDs (e.g., ['12345678', '87654321'])"
+        )
+    )
+    tool_call_id: Annotated[str, InjectedToolCallId]
+
+
+class PaperDownloaderFactory:
+    """Factory class for creating paper downloader instances."""
+
+    # Class-level cache for configuration
+    _cached_config = None
+    _config_lock = None
+
+    @classmethod
+    def clear_cache(cls) -> None:
+        """Clear cached configuration."""
+        cls._cached_config = None
+
+    @staticmethod
+    def get_default_service() -> Literal["arxiv", "medrxiv", "biorxiv", "pubmed"]:
+        """
+        Get the default service from configuration.
+
+        Returns:
+            Default service name from config, fallback to 'pubmed'
+        """
+        config = PaperDownloaderFactory._get_unified_config()
+        default_service = getattr(config.tool, "default_service", "pubmed")
+        # Ensure the default service is valid and return with proper type
+        if default_service == "arxiv":
+            return "arxiv"
+        if default_service == "medrxiv":
+            return "medrxiv"
+        if default_service == "biorxiv":
+            return "biorxiv"
+        if default_service == "pubmed":
+            return "pubmed"
+        logger.warning(
+            "Invalid default service '%s' in config, falling back to 'pubmed'",
+            default_service,
+        )
+        return "pubmed"
+
+    @staticmethod
+    def create(
+        service: Literal["arxiv", "medrxiv", "biorxiv", "pubmed"],
+    ) -> BasePaperDownloader:
+        """
+        Create appropriate downloader instance for the specified service.
+
+        Args:
+            service: Service name ('arxiv', 'medrxiv', 'biorxiv', 'pubmed')
+
+        Returns:
+            Configured downloader instance
+
+        Raises:
+            ValueError: If service is not supported
+        """
+        config = PaperDownloaderFactory._get_unified_config()
+        service_config = PaperDownloaderFactory._build_service_config(config, service)
+
+        if service == "arxiv":
+            return ArxivDownloader(service_config)
+        if service == "medrxiv":
+            return MedrxivDownloader(service_config)
+        if service == "biorxiv":
+            return BiorxivDownloader(service_config)
+        # service == "pubmed"
+        return PubmedDownloader(service_config)
+
+    @staticmethod
+    def _get_unified_config() -> Any:
+        """
+        Load unified paper download configuration using Hydra with caching.
+        This avoids the GlobalHydra reinitialization issue by caching the config.
+
+        Returns:
+            Unified configuration object
+        """
+        # Return cached config if available
+        if PaperDownloaderFactory._cached_config is not None:
+            return PaperDownloaderFactory._cached_config
+
+        # Ensure lock exists and get a local reference
+        lock = PaperDownloaderFactory._config_lock
+        if lock is None:
+            lock = threading.Lock()
+            PaperDownloaderFactory._config_lock = lock
+
+        # Thread-safe config loading with guaranteed non-None lock
+        with lock:
+            # Double-check pattern - another thread might have loaded it
+            if PaperDownloaderFactory._cached_config is not None:
+                return PaperDownloaderFactory._cached_config
+
+            try:
+
+                # Clear if already initialized
+                if GlobalHydra().is_initialized():
+                    logger.info(
+                        "GlobalHydra already initialized, clearing for config load"
+                    )
+                    GlobalHydra.instance().clear()
+
+                # Load configuration
+                with hydra.initialize(version_base=None, config_path="../../configs"):
+                    cfg = hydra.compose(
+                        config_name="config", overrides=["tools/paper_download=default"]
+                    )
+
+                # Cache the configuration
+                PaperDownloaderFactory._cached_config = cfg.tools.paper_download
+                logger.info(
+                    "Successfully loaded and cached paper download configuration"
+                )
+
+                return PaperDownloaderFactory._cached_config
+
+            except Exception as e:
+                logger.error(
+                    "Failed to load unified paper download configuration: %s", e
+                )
+                raise RuntimeError(f"Configuration loading failed: {e}") from e
+
+    @staticmethod
+    def _build_service_config(unified_config: Any, service: str) -> Any:
+        """
+        Build service-specific configuration by merging common and service settings.
+        Handles Hydra's OmegaConf objects properly.
+
+        Args:
+            unified_config: The unified configuration object
+            service: Service name
+
+        Returns:
+            Service-specific configuration object
+        """
+        if (
+            not hasattr(unified_config, "services")
+            or service not in unified_config.services
+        ):
+            raise ValueError(f"Service '{service}' not found in configuration")
+
+        # Create a simple config object that combines common and service-specific settings
+        class ServiceConfig:
+            """Service-specific configuration holder."""
+
+            def get_config_dict(self):
+                """Return configuration as dictionary."""
+                return {k: v for k, v in self.__dict__.items() if not k.startswith("_")}
+
+            def has_attribute(self, name: str) -> bool:
+                """Check if configuration has a specific attribute."""
+                return hasattr(self, name)
+
+        config_obj = ServiceConfig()
+
+        # Handle common config (using helper method to reduce branches)
+        PaperDownloaderFactory._apply_config(
+            config_obj, unified_config.common, "common"
+        )
+
+        # Handle service-specific config (using helper method to reduce branches)
+        PaperDownloaderFactory._apply_config(
+            config_obj, unified_config.services[service], service
+        )
+
+        return config_obj
+
+    @staticmethod
+    def _apply_config(config_obj: Any, source_config: Any, config_type: str) -> None:
+        """
+        Apply configuration from source to target object using multiple fallback methods.
+        This preserves all the original logic but reduces branches in the main method.
+
+        Args:
+            config_obj: Target configuration object
+            source_config: Source configuration to extract from
+            config_type: Type description for logging
+        """
+        try:
+            PaperDownloaderFactory._try_config_extraction(config_obj, source_config)
+        except (AttributeError, TypeError, KeyError) as e:
+            logger.warning("Failed to process %s config: %s", config_type, e)
+
+    @staticmethod
+    def _try_config_extraction(config_obj: Any, source_config: Any) -> None:
+        """Try different methods to extract configuration data."""
+        # Method 1: Try OmegaConf conversion
+        if hasattr(source_config, "_content"):
+            PaperDownloaderFactory._extract_from_omegaconf(config_obj, source_config)
+            return
+
+        # Method 2: Try direct attribute access
+        if hasattr(source_config, "__dict__"):
+            PaperDownloaderFactory._extract_from_dict(
+                config_obj, source_config.__dict__
+            )
+            return
+
+        # Method 3: Try items() method
+        if hasattr(source_config, "items"):
+            PaperDownloaderFactory._extract_from_items(config_obj, source_config)
+            return
+
+        # Method 4: Try dir() approach as fallback
+        PaperDownloaderFactory._extract_from_dir(config_obj, source_config)
+
+    @staticmethod
+    def _extract_from_omegaconf(config_obj: Any, source_config: Any) -> None:
+        """Extract configuration from OmegaConf object."""
+        config_dict = OmegaConf.to_container(source_config, resolve=True)
+        if isinstance(config_dict, dict):
+            for key, value in config_dict.items():
+                if isinstance(key, str):  # Type guard for key
+                    setattr(config_obj, key, value)
+
+    @staticmethod
+    def _extract_from_dict(config_obj: Any, config_dict: dict) -> None:
+        """Extract configuration from dictionary."""
+        for key, value in config_dict.items():
+            if not key.startswith("_"):
+                setattr(config_obj, key, value)
+
+    @staticmethod
+    def _extract_from_items(config_obj: Any, source_config: Any) -> None:
+        """Extract configuration using items() method."""
+        for key, value in source_config.items():
+            if isinstance(key, str):  # Type guard for key
+                setattr(config_obj, key, value)
+
+    @staticmethod
+    def _extract_from_dir(config_obj: Any, source_config: Any) -> None:
+        """Extract configuration using dir() approach as fallback."""
+        for key in dir(source_config):
+            if not key.startswith("_"):
+                value = getattr(source_config, key)
+                if not callable(value):
+                    setattr(config_obj, key, value)
+
+
+@tool(
+    args_schema=UnifiedPaperDownloadInput,
+    parse_docstring=True,
+)
+def download_papers(
+    service: Optional[Literal["arxiv", "medrxiv", "biorxiv", "pubmed"]],
+    identifiers: List[str],
+    tool_call_id: Annotated[str, InjectedToolCallId],
+) -> Command[Any]:
+    """
+    Universal paper download tool supporting multiple academic paper services.
+
+    Downloads paper metadata and PDFs from arXiv, medRxiv, bioRxiv, or PubMed and stores them
+    in temporary files for further processing. The downloaded PDFs can be accessed
+    using the temp_file_path in the returned metadata.
+
+    Args:
+        service: Paper service to download from (optional, uses configured default if not specified)
+            - 'arxiv': For arXiv preprints (requires arXiv IDs)
+            - 'medrxiv': For medRxiv preprints (requires DOIs)
+            - 'biorxiv': For bioRxiv preprints (requires DOIs)
+            - 'pubmed': For PubMed papers (requires PMIDs)
+        identifiers: List of paper identifiers in the format expected by the service
+
+    Returns:
+        Command with article_data containing paper metadata and local file paths
+
+    Examples:
+        # Download from arXiv
+        download_papers("arxiv", ["1234.5678", "2301.12345"])
+
+        # Download from medRxiv
+        download_papers("medrxiv", ["10.1101/2020.09.09.20191205"])
+
+        # Download from bioRxiv
+        download_papers("biorxiv", ["10.1101/2020.09.09.20191205"])
+
+        # Download from PubMed
+        download_papers("pubmed", ["12345678", "87654321"])
+
+        # Use default service (configured in default.yaml)
+        download_papers(None, ["12345678", "87654321"])
+    """
+    return _download_papers_impl(service, identifiers, tool_call_id)
+
+
+# Convenience functions for backward compatibility (optional)
+# These functions explicitly specify the service, bypassing the default service config
+def download_arxiv_papers(
+    arxiv_ids: List[str], tool_call_id: Annotated[str, InjectedToolCallId]
+) -> Command[Any]:
+    """Convenience function for downloading arXiv papers (explicitly uses arXiv service)."""
+    return _download_papers_impl("arxiv", arxiv_ids, tool_call_id)
+
+
+def download_medrxiv_papers(
+    dois: List[str], tool_call_id: Annotated[str, InjectedToolCallId]
+) -> Command[Any]:
+    """Convenience function for downloading medRxiv papers (explicitly uses medRxiv service)."""
+    return _download_papers_impl("medrxiv", dois, tool_call_id)
+
+
+def download_biorxiv_papers(
+    dois: List[str], tool_call_id: Annotated[str, InjectedToolCallId]
+) -> Command[Any]:
+    """Convenience function for downloading bioRxiv papers (explicitly uses bioRxiv service)."""
+    return _download_papers_impl("biorxiv", dois, tool_call_id)
+
+
+def download_pubmed_papers(
+    pmids: List[str], tool_call_id: Annotated[str, InjectedToolCallId]
+) -> Command[Any]:
+    """Convenience function for downloading PubMed papers (explicitly uses PubMed service)."""
+    return _download_papers_impl("pubmed", pmids, tool_call_id)
+
+
+def _download_papers_impl(
+    service: Optional[Literal["arxiv", "medrxiv", "biorxiv", "pubmed"]],
+    identifiers: List[str],
+    tool_call_id: str,
+) -> Command[Any]:
+    """
+    Internal implementation function that contains the actual download logic.
+    This is called by both the decorated tool and the convenience functions.
+    """
+    # Resolve default service if not specified
+    if service is None:
+        service = PaperDownloaderFactory.get_default_service()
+        logger.info("No service specified, using configured default: %s", service)
+    logger.info(
+        "Starting unified paper download for service '%s' with %d identifiers: %s",
+        service,
+        len(identifiers),
+        identifiers,
+    )
+
+    try:
+        # Step 1: Create appropriate downloader using factory
+        downloader = PaperDownloaderFactory.create(service)
+        logger.info("Created %s downloader successfully", downloader.get_service_name())
+
+        # Step 2: Process all identifiers
+        article_data = downloader.process_identifiers(identifiers)
+
+        # Step 3: Build summary for user
+        content = downloader.build_summary(article_data)
+
+        # Step 4: Log results summary
+        total_papers = len(article_data)
+        successful_downloads = sum(
+            1
+            for paper in article_data.values()
+            if paper.get("access_type") == "open_access_downloaded"
+        )
+        logger.info(
+            "Download complete for %s: %d papers processed, %d PDFs downloaded",
+            service,
+            total_papers,
+            successful_downloads,
+        )
+
+        # Step 5: Return command with results
+        return Command(
+            update={
+                "article_data": article_data,
+                "messages": [
+                    ToolMessage(
+                        content=content,
+                        tool_call_id=tool_call_id,
+                        artifact=article_data,
+                    )
+                ],
+            }
+        )
+
+    except ValueError as e:
+        # Handle service/configuration errors
+        error_msg = f"Service error for '{service}': {str(e)}"
+        logger.error(error_msg)
+
+        return Command(
+            update={
+                "article_data": {},
+                "messages": [
+                    ToolMessage(
+                        content=f"Error: {error_msg}",
+                        tool_call_id=tool_call_id,
+                        artifact={},
+                    )
+                ],
+            }
+        )
+
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        # Handle unexpected errors
+        error_msg = f"Unexpected error during paper download: {str(e)}"
+        logger.error(error_msg, exc_info=True)
+
+        return Command(
+            update={
+                "article_data": {},
+                "messages": [
+                    ToolMessage(
+                        content=f"Error: {error_msg}",
+                        tool_call_id=tool_call_id,
+                        artifact={},
+                    )
+                ],
+            }
+        )

aiagents4pharma/talk2scholars/tools/paper_download/utils/__init__.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+"""
+This package provides modules for fetching and downloading academic papers from arXiv,
+biorxiv and medrxiv.
+"""
+
+# Import modules
+from . import arxiv_downloader
+from . import base_paper_downloader
+from . import biorxiv_downloader
+from . import medrxiv_downloader
+from . import pubmed_downloader
+
+__all__ = [
+    "arxiv_downloader",
+    "base_paper_downloader",
+    "biorxiv_downloader",
+    "medrxiv_downloader",
+    "pubmed_downloader",
+]

aiagents4pharma/talk2scholars/tools/paper_download/utils/arxiv_downloader.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""
+ArXiv paper downloader implementation.
+"""
+
+import logging
+import xml.etree.ElementTree as ET
+from typing import Any, Dict, Optional, Tuple
+
+import requests
+
+from .base_paper_downloader import BasePaperDownloader
+
+logger = logging.getLogger(__name__)
+
+
+class ArxivDownloader(BasePaperDownloader):
+    """ArXiv-specific implementation of paper downloader."""
+
+    def __init__(self, config: Any):
+        """Initialize ArXiv downloader with configuration."""
+        super().__init__(config)
+        self.api_url = config.api_url
+        self.pdf_base_url = config.pdf_base_url
+        # XML namespace configuration
+        self.xml_namespaces = getattr(
+            config, "xml_namespace", {"atom": "http://www.w3.org/2005/Atom"}
+        )
+
+    def fetch_metadata(self, identifier: str) -> ET.Element:
+        """
+        Fetch paper metadata from arXiv API.
+
+        Args:
+            identifier: arXiv ID (e.g., '1234.5678' or '2301.12345')
+
+        Returns:
+            XML root element from arXiv API response
+
+        Raises:
+            requests.RequestException: If API call fails
+            RuntimeError: If no entry found in response
+        """
+        query_url = f"{self.api_url}?search_query=id:{identifier}&start=0&max_results=1"
+        logger.info("Fetching metadata for arXiv ID %s from: %s", identifier, query_url)
+
+        response = requests.get(query_url, timeout=self.request_timeout)
+        response.raise_for_status()
+
+        root = ET.fromstring(response.text)
+        entry = root.find("atom:entry", self.xml_namespaces)
+
+        if entry is None:
+            raise RuntimeError("No entry found in arXiv API response")
+
+        return root
+
+    def construct_pdf_url(self, metadata: ET.Element, identifier: str) -> str:
+        """
+        Extract or construct PDF URL from arXiv metadata.
+
+        Args:
+            metadata: XML root from arXiv API
+            identifier: arXiv ID
+
+        Returns:
+            PDF URL string
+        """
+        entry = metadata.find("atom:entry", self.xml_namespaces)
+
+        if entry is None:
+            return ""
+
+        # Try to find PDF link in metadata first
+        pdf_url = next(
+            (
+                link.attrib.get("href")
+                for link in entry.findall("atom:link", self.xml_namespaces)
+                if link.attrib.get("title") == "pdf"
+            ),
+            None,
+        )
+
+        # Fallback to constructed PDF URL if not found in metadata
+        if not pdf_url:
+            pdf_url = f"{self.pdf_base_url}/{identifier}.pdf"
+            logger.info("Using constructed PDF URL for %s: %s", identifier, pdf_url)
+
+        return pdf_url
+
+    def extract_paper_metadata(
+        self,
+        metadata: ET.Element,
+        identifier: str,
+        pdf_result: Optional[Tuple[str, str]],
+    ) -> Dict[str, Any]:
+        """
+        Extract structured metadata from arXiv API response.
+
+        Args:
+            metadata: XML root from arXiv API
+            identifier: arXiv ID
+            pdf_result: Tuple of (temp_file_path, filename) if PDF downloaded
+
+        Returns:
+            Standardized paper metadata dictionary
+        """
+        entry = metadata.find("atom:entry", self.xml_namespaces)
+
+        if entry is None:
+            raise RuntimeError("No entry found in metadata")
+
+        # Extract basic metadata
+        basic_metadata = self._extract_basic_metadata(entry, self.xml_namespaces)
+
+        # Handle PDF download results
+        pdf_metadata = self._extract_pdf_metadata(pdf_result, identifier)
+
+        # Combine all metadata
+        return {
+            **basic_metadata,
+            **pdf_metadata,
+            "source": "arxiv",
+            "arxiv_id": identifier,
+        }
+
+    def _extract_basic_metadata(self, entry: ET.Element, ns: dict) -> Dict[str, Any]:
+        """Extract basic metadata (title, authors, abstract, date) from entry."""
+        title = self._extract_title(entry, ns)
+        authors = self._extract_authors(entry, ns)
+        abstract = self._extract_abstract(entry, ns)
+        pub_date = self._extract_publication_date(entry, ns)
+
+        return {
+            "Title": title,
+            "Authors": authors,
+            "Abstract": abstract,
+            "Publication Date": pub_date,
+        }
+
+    def _extract_title(self, entry: ET.Element, ns: dict) -> str:
+        """Extract title from entry."""
+        title_elem = entry.find("atom:title", ns)
+        return (title_elem.text or "").strip() if title_elem is not None else "N/A"
+
+    def _extract_authors(self, entry: ET.Element, ns: dict) -> list:
+        """Extract authors from entry."""
+        authors = []
+        for author_elem in entry.findall("atom:author", ns):
+            name_elem = author_elem.find("atom:name", ns)
+            if name_elem is not None and name_elem.text:
+                authors.append(name_elem.text.strip())
+        return authors
+
+    def _extract_abstract(self, entry: ET.Element, ns: dict) -> str:
+        """Extract abstract from entry."""
+        summary_elem = entry.find("atom:summary", ns)
+        return (summary_elem.text or "").strip() if summary_elem is not None else "N/A"
+
+    def _extract_publication_date(self, entry: ET.Element, ns: dict) -> str:
+        """Extract publication date from entry."""
+        published_elem = entry.find("atom:published", ns)
+        return (
+            (published_elem.text or "").strip() if published_elem is not None else "N/A"
+        )
+
+    def _extract_pdf_metadata(
+        self, pdf_result: Optional[Tuple[str, str]], identifier: str
+    ) -> Dict[str, Any]:
+        """Extract PDF-related metadata."""
+        if pdf_result:
+            temp_file_path, filename = pdf_result
+            return {
+                "URL": temp_file_path,
+                "pdf_url": temp_file_path,
+                "filename": filename,
+                "access_type": "open_access_downloaded",
+                "temp_file_path": temp_file_path,
+            }
+
+        return {
+            "URL": "",
+            "pdf_url": "",
+            "filename": self.get_default_filename(identifier),
+            "access_type": "download_failed",
+            "temp_file_path": "",
+        }
+
+    def get_service_name(self) -> str:
+        """Return service name."""
+        return "arXiv"
+
+    def get_identifier_name(self) -> str:
+        """Return identifier display name."""
+        return "arXiv ID"
+
+    def get_default_filename(self, identifier: str) -> str:
+        """Generate default filename for arXiv paper."""
+        return f"{identifier}.pdf"
+
+    def _get_paper_identifier_info(self, paper: Dict[str, Any]) -> str:
+        """Get arXiv-specific identifier info for paper summary."""
+        arxiv_id = paper.get("arxiv_id", "N/A")
+        pub_date = paper.get("Publication Date", "N/A")
+        return f" (arXiv:{arxiv_id}, {pub_date})"
+
+    def _add_service_identifier(self, entry: Dict[str, Any], identifier: str) -> None:
+        """Add arXiv ID field to entry."""
+        entry["arxiv_id"] = identifier