paper-search-cli 1.0.2__py3-none-any.whl

paper_search/__init__.py

@@ -0,0 +1,3 @@
+from .config import load_env_file
+
+load_env_file()

paper_search/academic_platforms/acm.py

@@ -0,0 +1,113 @@
+"""ACM Digital Library connector — optional, requires API key env.
+
+This module is a **skeleton only**.  No real ACM DL API requests are made
+unless the ``PAPER_SEARCH_MCP_ACM_API_KEY`` (or legacy ``ACM_API_KEY``)
+environment variable is configured.  All methods
+raise :class:`NotImplementedError` with a descriptive message when accessed
+without a valid key so that the rest of the platform continues to work without
+any paid credentials.
+
+Enable usage::
+
+    export PAPER_SEARCH_MCP_ACM_API_KEY=<your_acm_api_key>
+
+.. note::
+    ACM recently opened a limited metadata API.  Check
+    https://libraries.acm.org/digital-library/acm-open for Open Access content
+    that does NOT require a key.  Full-text/PDF download requires ACM membership
+    or institutional access.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import List
+
+from .base import PaperSource
+from ..paper import Paper
+from ..config import get_env
+
+logger = logging.getLogger(__name__)
+
+_NOT_CONFIGURED_MSG = (
+    "ACM Digital Library is not configured.  Set PAPER_SEARCH_MCP_ACM_API_KEY "
+    "(or legacy ACM_API_KEY) environment "
+    "variable to enable ACM DL search.  "
+    "See https://libraries.acm.org/digital-library/acm-open for access options."
+)
+
+
+class ACMSearcher(PaperSource):
+    """Skeleton connector for ACM Digital Library.
+
+    Instantiating this class without ``PAPER_SEARCH_MCP_ACM_API_KEY``
+    (or ``ACM_API_KEY``) set will log a warning
+    but will NOT raise an error.  All actual operations raise
+    :class:`NotImplementedError` with a clear message directing the user to
+    configure their API key.
+    """
+
+    # ACM DL base URL (placeholder — real endpoint TBD once API key is available)
+    BASE_URL = "https://dl.acm.org/action/doSearch"
+
+    def __init__(self) -> None:
+        self.api_key: str = get_env("ACM_API_KEY", "")
+        if not self.api_key:
+            logger.warning(
+                "ACMSearcher initialised without PAPER_SEARCH_MCP_ACM_API_KEY/ACM_API_KEY.  "
+                "All calls will raise NotImplementedError until the key is set."
+            )
+
+    # ------------------------------------------------------------------
+    # Public helpers
+    # ------------------------------------------------------------------
+
+    def is_configured(self) -> bool:
+        """Return True only when a non-empty ACM API key is available."""
+        return bool(self.api_key)
+
+    # ------------------------------------------------------------------
+    # PaperSource interface
+    # ------------------------------------------------------------------
+
+    def search(self, query: str, max_results: int = 10, **kwargs) -> List[Paper]:  # type: ignore[override]
+        """Search ACM Digital Library — requires PAPER_SEARCH_MCP_ACM_API_KEY or ACM_API_KEY.
+
+        Raises:
+            NotImplementedError: Always, when ACM API key env is not set.
+        """
+        if not self.is_configured():
+            raise NotImplementedError(_NOT_CONFIGURED_MSG)
+
+        # TODO: implement real ACM DL API call here once key is available
+        raise NotImplementedError(
+            "ACM DL search is not yet implemented.  "
+            "Contribute at https://github.com/your-repo/paper-search-cli."
+        )
+
+    def download_pdf(self, paper_id: str, save_path: str = "./downloads") -> str:
+        """Download a PDF from ACM DL — requires ACM API key env and institutional access.
+
+        Raises:
+            NotImplementedError: Always, until key + download logic are implemented.
+        """
+        if not self.is_configured():
+            raise NotImplementedError(_NOT_CONFIGURED_MSG)
+
+        raise NotImplementedError(
+            "ACM DL PDF download is not yet implemented.  "
+            "Note: full-text access also requires ACM membership or institutional access."
+        )
+
+    def read_paper(self, paper_id: str, save_path: str = "./downloads") -> str:
+        """Read paper content from ACM DL — requires ACM API key env.
+
+        Raises:
+            NotImplementedError: Always, until download + extraction are implemented.
+        """
+        if not self.is_configured():
+            raise NotImplementedError(_NOT_CONFIGURED_MSG)
+
+        raise NotImplementedError(
+            "ACM DL paper reading is not yet implemented."
+        )

paper_search/academic_platforms/arxiv.py

@@ -0,0 +1,157 @@
+# paper_search/sources/arxiv.py
+from typing import List
+from datetime import datetime
+import requests
+import feedparser
+import time
+from ..paper import Paper
+from ..utils import extract_doi
+from .base import PaperSource
+from PyPDF2 import PdfReader
+import os
+
+class ArxivSearcher(PaperSource):
+    """Searcher for arXiv papers"""
+    BASE_URL = "http://export.arxiv.org/api/query"
+
+    def __init__(self):
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'paper-search-cli/1.0.0 (mailto:openags@example.com)',
+            'Accept': 'application/atom+xml, application/xml;q=0.9, */*;q=0.8',
+        })
+
+    def search(self, query: str, max_results: int = 10) -> List[Paper]:
+        params = {
+            'search_query': f'all:{query}',
+            'max_results': max_results,
+            'sortBy': 'submittedDate',
+            'sortOrder': 'descending'
+        }
+        response = None
+        for attempt in range(3):
+            try:
+                response = self.session.get(self.BASE_URL, params=params, timeout=30)
+            except requests.RequestException:
+                time.sleep((attempt + 1) * 1.5)
+                continue
+            if response.status_code == 200:
+                break
+            if response.status_code in (429, 500, 502, 503, 504):
+                time.sleep((attempt + 1) * 1.5)
+                continue
+            break
+
+        if response is None or response.status_code != 200:
+            return []
+
+        feed = feedparser.parse(response.content)
+        papers = []
+        for entry in feed.entries:
+            try:
+                authors = [author.name for author in entry.authors]
+                published = datetime.strptime(entry.published, '%Y-%m-%dT%H:%M:%SZ')
+                updated = datetime.strptime(entry.updated, '%Y-%m-%dT%H:%M:%SZ')
+                pdf_url = next((link.href for link in entry.links if link.type == 'application/pdf'), '')
+                
+                # Try to extract DOI from entry.doi or links or summary
+                doi = entry.get('doi', '') or extract_doi(entry.summary) or extract_doi(entry.id)
+                for link in entry.links:
+                    if link.get('title') == 'doi':
+                        doi = doi or extract_doi(link.href)
+
+                papers.append(Paper(
+                    paper_id=entry.id.split('/')[-1],
+                    title=entry.title,
+                    authors=authors,
+                    abstract=entry.summary,
+                    url=entry.id,
+                    pdf_url=pdf_url,
+                    published_date=published,
+                    updated_date=updated,
+                    source='arxiv',
+                    categories=[tag.term for tag in entry.tags],
+                    keywords=[],
+                    doi=doi
+                ))
+            except Exception as e:
+                print(f"Error parsing arXiv entry: {e}")
+        return papers
+
+    def download_pdf(self, paper_id: str, save_path: str) -> str:
+        pdf_url = f"https://arxiv.org/pdf/{paper_id}.pdf"
+        response = requests.get(pdf_url)
+        os.makedirs(save_path, exist_ok=True)
+        output_file = f"{save_path}/{paper_id}.pdf"
+        with open(output_file, 'wb') as f:
+            f.write(response.content)
+        return output_file
+
+    def read_paper(self, paper_id: str, save_path: str = "./downloads") -> str:
+        """Read a paper and convert it to text format.
+        
+        Args:
+            paper_id: arXiv paper ID
+            save_path: Directory where the PDF is/will be saved
+            
+        Returns:
+            str: The extracted text content of the paper
+        """
+        # First ensure we have the PDF
+        pdf_path = f"{save_path}/{paper_id}.pdf"
+        if not os.path.exists(pdf_path):
+            pdf_path = self.download_pdf(paper_id, save_path)
+        
+        # Read the PDF
+        try:
+            reader = PdfReader(pdf_path)
+            text = ""
+            
+            # Extract text from each page
+            for page in reader.pages:
+                text += page.extract_text() + "\n"
+            
+            return text.strip()
+        except Exception as e:
+            print(f"Error reading PDF for paper {paper_id}: {e}")
+            return ""
+
+if __name__ == "__main__":
+    # 测试 ArxivSearcher 的功能
+    searcher = ArxivSearcher()
+    
+    # 测试搜索功能
+    print("Testing search functionality...")
+    query = "machine learning"
+    max_results = 5
+    try:
+        papers = searcher.search(query, max_results=max_results)
+        print(f"Found {len(papers)} papers for query '{query}':")
+        for i, paper in enumerate(papers, 1):
+            print(f"{i}. {paper.title} (ID: {paper.paper_id})")
+    except Exception as e:
+        print(f"Error during search: {e}")
+    
+    # 测试 PDF 下载功能
+    if papers:
+        print("\nTesting PDF download functionality...")
+        paper_id = papers[0].paper_id
+        save_path = "./downloads"  # 确保此目录存在
+        try:
+            os.makedirs(save_path, exist_ok=True)
+            pdf_path = searcher.download_pdf(paper_id, save_path)
+            print(f"PDF downloaded successfully: {pdf_path}")
+        except Exception as e:
+            print(f"Error during PDF download: {e}")
+
+    # 测试论文阅读功能
+    if papers:
+        print("\nTesting paper reading functionality...")
+        paper_id = papers[0].paper_id
+        try:
+            text_content = searcher.read_paper(paper_id)
+            print(f"\nFirst 500 characters of the paper content:")
+            print(text_content[:500] + "...")
+            print(f"\nTotal length of extracted text: {len(text_content)} characters")
+        except Exception as e:
+            print(f"Error during paper reading: {e}")

paper_search/academic_platforms/base.py

@@ -0,0 +1,54 @@
+"""Base class for all academic paper source searchers."""
+from abc import ABC, abstractmethod
+from typing import List
+from ..paper import Paper
+
+
+class PaperSource(ABC):
+    """Abstract base class for academic paper sources."""
+
+    @abstractmethod
+    def search(self, query: str, **kwargs) -> List[Paper]:
+        """Search papers matching the query.
+
+        Args:
+            query: Search query string.
+            **kwargs: Source-specific parameters (e.g., max_results, year).
+
+        Returns:
+            List of Paper objects.
+        """
+
+    def download_pdf(self, paper_id: str, save_path: str) -> str:
+        """Download the PDF for a given paper.
+
+        Args:
+            paper_id: Platform-specific paper identifier.
+            save_path: Directory to save the downloaded PDF.
+
+        Returns:
+            Path to the saved PDF file.
+
+        Raises:
+            NotImplementedError: If the source does not support PDF downloads.
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not support PDF downloads."
+        )
+
+    def read_paper(self, paper_id: str, save_path: str = "./downloads") -> str:
+        """Download and extract text from a paper PDF.
+
+        Args:
+            paper_id: Platform-specific paper identifier.
+            save_path: Directory where the PDF is/will be saved.
+
+        Returns:
+            Extracted text content of the paper.
+
+        Raises:
+            NotImplementedError: If the source does not support paper reading.
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not support reading paper content."
+        )

paper_search/academic_platforms/base_search.py

@@ -0,0 +1,253 @@
+# paper_search/academic_platforms/base_search.py
+"""Searcher for BASE (Bielefeld Academic Search Engine).
+
+BASE is one of the world's most voluminous search engines especially for
+academic open access web resources. It provides OAI-PMH access to metadata
+from thousands of repositories.
+
+OAI-PMH Endpoint: https://api.base-search.net/cgi-bin/BaseHttpSearchInterface.fcgi
+Documentation: https://www.base-search.net/about/en/about_sources_date.php
+"""
+
+from typing import List, Optional, Dict, Any
+import logging
+from .oaipmh import OAIPMHSearcher
+from ..paper import Paper
+
+logger = logging.getLogger(__name__)
+
+
+class BASESearcher(OAIPMHSearcher):
+    """Searcher for BASE (Bielefeld Academic Search Engine)."""
+
+    def __init__(self):
+        """Initialize BASE searcher with OAI-PMH endpoint."""
+        super().__init__(
+            base_url="https://api.base-search.net/cgi-bin/BaseHttpSearchInterface.fcgi",
+            metadata_prefix="oai_dc"
+        )
+        # Update User-Agent for BASE
+        self.session.headers.update({
+            'User-Agent': 'paper-search-cli/1.0.0 (BASE OAI-PMH client; https://github.com/openags/paper-search-cli)'
+        })
+
+    def search(self, query: str, max_results: int = 10, **kwargs) -> List[Paper]:
+        """Search BASE using OAI-PMH with query filtering.
+
+        Args:
+            query: Search query string
+            max_results: Maximum number of results to return
+            **kwargs: Additional parameters:
+                - set: OAI-PMH set specification (e.g., 'pubtype:article')
+                - from_date: Harvest from date (YYYY-MM-DD)
+                - until_date: Harvest until date (YYYY-MM-DD)
+                - language: Filter by language (e.g., 'en', 'de')
+                - subject: Filter by subject category
+                - has_fulltext: Filter for fulltext availability (True/False)
+                - open_access: Filter for open access content (True/False)
+
+        Returns:
+            List of Paper objects
+        """
+        # BASE-specific sets
+        if 'has_fulltext' in kwargs and kwargs['has_fulltext']:
+            kwargs['set'] = kwargs.get('set', '') + ' dcterms:accessRights:open'
+        if 'open_access' in kwargs and kwargs['open_access']:
+            kwargs['set'] = kwargs.get('set', '') + ' dcterms:accessRights:open'
+
+        # Call parent OAI-PMH search
+        papers = super().search(query, max_results, **kwargs)
+
+        # Apply additional BASE-specific filtering
+        filtered_papers = []
+        for paper in papers:
+            if self._filter_paper(paper, kwargs):
+                filtered_papers.append(paper)
+            if len(filtered_papers) >= max_results:
+                break
+
+        return filtered_papers[:max_results]
+
+    def _filter_paper(self, paper: Paper, filters: Dict[str, Any]) -> bool:
+        """Apply BASE-specific filters to paper.
+
+        Args:
+            paper: Paper object
+            filters: Filter parameters
+
+        Returns:
+            True if paper passes all filters
+        """
+        # Language filter
+        if 'language' in filters and filters['language']:
+            paper_lang = paper.extra.get('language', '').lower() if paper.extra else ''
+            if not paper_lang or paper_lang != filters['language'].lower():
+                return False
+
+        # Subject filter
+        if 'subject' in filters and filters['subject']:
+            subject_lower = filters['subject'].lower()
+            in_categories = any(subject_lower in cat.lower() for cat in paper.categories)
+            in_keywords = any(subject_lower in kw.lower() for kw in paper.keywords)
+            if not in_categories and not in_keywords:
+                return False
+
+        # Open access filter (already handled in OAI-PMH set)
+        # Fulltext filter
+        if 'has_fulltext' in filters and filters['has_fulltext']:
+            if not paper.pdf_url and not paper.url:
+                return False
+
+        return True
+
+    def _enrich_paper_from_oai(self, paper: Paper, dc_root):
+        """Enrich Paper object with BASE-specific metadata.
+
+        Overrides parent method to extract BASE-specific fields.
+
+        Args:
+            paper: Paper object to enrich
+            dc_root: Dublin Core XML element
+        """
+        super()._enrich_paper_from_oai(paper, dc_root)
+
+        # BASE-specific fields
+        if not paper.extra:
+            paper.extra = {}
+
+        # Extract BASE-specific identifiers
+        import xml.etree.ElementTree as ET
+        identifiers = dc_root.findall('.//{http://purl.org/dc/elements/1.1/}identifier') or \
+                     dc_root.findall('identifier')
+
+        for ident_elem in identifiers:
+            if ident_elem.text:
+                ident_text = ident_elem.text.lower()
+                if 'base-search.net' in ident_text:
+                    paper.extra['base_id'] = ident_text
+                elif 'urn:nbn:' in ident_text:
+                    paper.extra['urn'] = ident_text
+                elif 'hdl.handle.net' in ident_text:
+                    paper.extra['handle'] = ident_text
+
+        # Extract rights information
+        rights_elems = dc_root.findall('.//{http://purl.org/dc/elements/1.1/}rights') or \
+                      dc_root.findall('rights')
+        if rights_elems:
+            paper.extra['rights'] = [elem.text for elem in rights_elems if elem.text]
+
+        # Extract source repository
+        source_elems = dc_root.findall('.//{http://purl.org/dc/elements/1.1/}source') or \
+                      dc_root.findall('source')
+        if source_elems:
+            paper.extra['repository'] = source_elems[0].text if source_elems[0].text else ''
+
+        # Try to extract PDF URL from identifiers
+        if not paper.pdf_url:
+            for ident_elem in identifiers:
+                if ident_elem.text and ident_elem.text.lower().endswith('.pdf'):
+                    paper.pdf_url = ident_elem.text
+                    break
+
+        # Extract BASE relevance score if available
+        # (BASE doesn't provide relevance scores in OAI-PMH, but we might add it from other sources)
+
+    def download_pdf(self, paper_id: str, save_path: str) -> str:
+        """Download PDF for a BASE record.
+
+        BASE often provides direct PDF links in metadata.
+
+        Args:
+            paper_id: BASE identifier or OAI-PMH identifier
+            save_path: Directory to save PDF
+
+        Returns:
+            Path to saved PDF file
+
+        Raises:
+            NotImplementedError: If PDF cannot be downloaded
+        """
+        # Try parent method first (searches for PDF URL)
+        try:
+            return super().download_pdf(paper_id, save_path)
+        except Exception as e:
+            logger.warning(f"Parent download failed: {e}")
+
+        # Try alternative approach: search for paper and use first PDF link
+        papers = self.search(paper_id, max_results=1)
+        if not papers:
+            raise ValueError(f"BASE record not found: {paper_id}")
+
+        paper = papers[0]
+        if paper.pdf_url:
+            import os
+            import requests
+            response = self.session.get(paper.pdf_url, timeout=30)
+            response.raise_for_status()
+            os.makedirs(save_path, exist_ok=True)
+
+            # Create safe filename
+            safe_id = paper_id.replace('/', '_').replace(':', '_')
+            filename = f"base_{safe_id}.pdf"
+            output_file = os.path.join(save_path, filename)
+
+            with open(output_file, 'wb') as f:
+                f.write(response.content)
+
+            logger.info(f"Downloaded PDF to {output_file}")
+            return output_file
+
+        raise NotImplementedError(
+            f"No PDF available for BASE record: {paper_id}"
+        )
+
+    def read_paper(self, paper_id: str, save_path: str = "./downloads") -> str:
+        """Read paper text from PDF.
+
+        Args:
+            paper_id: Paper identifier
+            save_path: Directory where PDF is/will be saved
+
+        Returns:
+            Extracted text content
+
+        Raises:
+            NotImplementedError: If PDF cannot be read
+        """
+        try:
+            return super().read_paper(paper_id, save_path)
+        except Exception as e:
+            logger.error(f"Error reading BASE paper {paper_id}: {e}")
+            raise NotImplementedError(
+                f"Cannot read paper from BASE: {e}"
+            )
+
+
+if __name__ == "__main__":
+    """Test the BASESearcher."""
+    import logging
+    logging.basicConfig(level=logging.INFO)
+
+    searcher = BASESearcher()
+
+    # Test search
+    print("Testing BASE search...")
+
+    # Test queries
+    test_queries = [
+        "machine learning",
+        "artificial intelligence",
+        "data science"
+    ]
+
+    for query in test_queries[:1]:  # Test first query only
+        print(f"\nSearching BASE for: '{query}'")
+        papers = searcher.search(query, max_results=3)
+        print(f"Found {len(papers)} papers")
+        for i, paper in enumerate(papers):
+            print(f"{i+1}. {paper.title}")
+            print(f"   Authors: {', '.join(paper.authors[:3])}")
+            print(f"   Source: {paper.source}")
+            print(f"   PDF: {'Yes' if paper.pdf_url else 'No'}")
+            print(f"   URL: {paper.url}")
+            print()