academic-search-mcp 0.1.3__py3-none-any.whl

paper_search_mcp/academic_platforms/ssrn.py

@@ -0,0 +1,385 @@
+"""SSRN integration for preprints and early-stage research.
+
+SSRN is a repository specializing in preprints from social sciences, law, business, and humanities.
+Note: SSRN doesn't have a public API, so we use web scraping with proper rate limiting.
+"""
+from typing import List, Optional, Dict
+from datetime import datetime
+from curl_cffi import requests
+from bs4 import BeautifulSoup
+import re
+import os
+import logging
+import time
+
+from ..paper import Paper
+from ..pdf_utils import extract_text_from_pdf
+
+logger = logging.getLogger(__name__)
+
+
+class SSRNSearcher:
+    """Searcher for SSRN preprints and early research.
+
+    SSRN covers:
+    - Economics and finance
+    - Law and legal studies
+    - Business (management, marketing, accounting)
+    - Social sciences
+    - Humanities
+    - Computer science
+    """
+
+    BASE_URL = "https://papers.ssrn.com"
+    ABSTRACT_URL = f"{BASE_URL}/sol3/papers.cfm"
+    SEARCH_URL = f"{BASE_URL}/sol3/JELJOUR_Results.cfm"
+
+    def __init__(self):
+        """Initialize SSRN searcher."""
+        self.impersonate = "chrome"
+        self.last_request_time = 0
+
+    def _rate_limit(self, delay: float = 1.0):
+        """Apply rate limiting to avoid being blocked."""
+        elapsed = time.time() - self.last_request_time
+        if elapsed < delay:
+            time.sleep(delay - elapsed)
+        self.last_request_time = time.time()
+
+    def search(
+        self,
+        query: str,
+        max_results: int = 10,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+        **kwargs
+    ) -> List[Paper]:
+        """Search SSRN for papers.
+
+        Args:
+            query: Search query string
+            max_results: Maximum number of papers (max: 100)
+            date_from: Start date YYYY-MM-DD (e.g., '2024-01-01')
+            date_to: End date YYYY-MM-DD (e.g., '2024-12-31')
+            **kwargs: Additional parameters (topic, author_id)
+
+        Returns:
+            List of Paper objects
+        """
+        papers = []
+
+        try:
+            self._rate_limit()
+
+            # Use the search URL with query
+            url = f"{self.BASE_URL}/sol3/displayAbstractSearch.cfm"
+            params = {
+                "txtKey_Words": query,
+                "search": "Search"
+            }
+
+            response = requests.get(url, params=params, timeout=30, impersonate=self.impersonate)
+
+            if response.status_code != 200:
+                logger.error(f"SSRN search failed with status {response.status_code}")
+                return papers
+
+            soup = BeautifulSoup(response.content, 'lxml')
+
+            # Parse results
+            results = self._parse_search_results(soup)
+
+            # Apply date filters
+            if date_from or date_to:
+                year_from = int(date_from[:4]) if date_from else 0
+                year_to = int(date_to[:4]) if date_to else 9999
+
+                filtered = []
+                for paper in results:
+                    paper_year = paper.published_date.year if paper.published_date and paper.published_date != datetime.min else 0
+                    if year_from <= paper_year <= year_to:
+                        filtered.append(paper)
+                results = filtered
+
+            papers = results[:max_results]
+            logger.info(f"SSRN search: found {len(papers)} papers for '{query}'")
+
+        except Exception as e:
+            logger.error(f"SSRN search error: {e}")
+
+        return papers
+
+    def get_paper_by_id(self, paper_id: str) -> Optional[Paper]:
+        """Get a specific paper by its SSRN ID.
+
+        Args:
+            paper_id: SSRN paper ID (abstract_id)
+
+        Returns:
+            Paper object or None
+        """
+        try:
+            self._rate_limit()
+
+            url = self.ABSTRACT_URL
+            params = {"abstract_id": paper_id}
+            response = requests.get(url, params=params, timeout=30, impersonate=self.impersonate)
+
+            if response.status_code == 404:
+                return None
+            response.raise_for_status()
+
+            return self._parse_paper_page(response.content, paper_id)
+
+        except Exception as e:
+            logger.error(f"Error fetching SSRN paper {paper_id}: {e}")
+            return None
+
+    def search_by_doi(self, doi: str) -> Optional[Paper]:
+        """Search for paper by DOI.
+
+        Args:
+            doi: Digital Object Identifier
+
+        Returns:
+            Paper object or None
+        """
+        clean_doi = doi.replace("https://doi.org/", "").replace("doi:", "").strip()
+
+        # SSRN DOIs typically contain "ssrn" - extract the ID
+        if "ssrn" in clean_doi.lower():
+            match = re.search(r'(\d{6,})', clean_doi)
+            if match:
+                return self.get_paper_by_id(match.group(1))
+
+        # Try searching
+        results = self.search(clean_doi, max_results=1)
+        return results[0] if results else None
+
+    def download_pdf(self, paper_id: str, save_path: str = "./downloads") -> str:
+        """Download PDF from SSRN.
+
+        Args:
+            paper_id: SSRN paper ID
+            save_path: Directory to save
+
+        Returns:
+            Path to PDF or error message
+        """
+        try:
+            os.makedirs(save_path, exist_ok=True)
+
+            self._rate_limit()
+
+            # SSRN download URL format
+            download_url = f"{self.BASE_URL}/sol3/Delivery.cfm/SSRN_ID{paper_id}.pdf?abstractid={paper_id}&mirid=1"
+
+            response = requests.get(download_url, timeout=60, allow_redirects=True, impersonate=self.impersonate)
+
+            if response.status_code != 200:
+                return f"PDF download not available for {paper_id}"
+
+            content_type = response.headers.get("Content-Type", "")
+
+            if "pdf" in content_type.lower():
+                filename = f"ssrn_{paper_id}.pdf"
+                file_path = os.path.join(save_path, filename)
+
+                with open(file_path, 'wb') as f:
+                    f.write(response.content)
+
+                return file_path
+
+            return f"PDF not available for {paper_id} (may require login)"
+
+        except Exception as e:
+            logger.error(f"Error downloading SSRN PDF: {e}")
+            return f"Failed to download PDF: {e}"
+
+    def read_paper(self, paper_id: str, save_path: str = "./downloads") -> str:
+        """Read and extract text from a SSRN paper PDF.
+
+        Args:
+            paper_id: SSRN paper ID
+            save_path: Directory for PDF storage
+
+        Returns:
+            Extracted text or error message
+        """
+        pdf_path = self.download_pdf(paper_id, save_path)
+        if not os.path.exists(pdf_path):
+            # If PDF not available, return abstract
+            paper = self.get_paper_by_id(paper_id)
+            if paper and paper.abstract:
+                return f"[Abstract only - PDF requires login]\n\n{paper.abstract}"
+            return pdf_path  # Return error message
+
+        text = extract_text_from_pdf(pdf_path)
+        return text if text else "Failed to extract text from PDF"
+
+    def _parse_search_results(self, soup: BeautifulSoup) -> List[Paper]:
+        """Parse search results page into Paper objects."""
+        papers = []
+
+        # Find paper entries - SSRN uses various formats
+        # Try finding by abstract links
+        for link in soup.find_all("a", href=re.compile(r"abstract_id=\d+")):
+            try:
+                href = link.get("href", "")
+                match = re.search(r"abstract_id=(\d+)", href)
+                if not match:
+                    continue
+
+                paper_id = match.group(1)
+                title = link.get_text(strip=True)
+
+                if not title or len(title) < 5:
+                    continue
+
+                # Try to find parent container for more info
+                parent = link.find_parent("div") or link.find_parent("tr")
+
+                authors = []
+                abstract = ""
+                published_date = datetime.min
+
+                if parent:
+                    # Look for author info
+                    author_elem = parent.find("span", {"class": re.compile(r"author", re.I)})
+                    if author_elem:
+                        author_text = author_elem.get_text(strip=True)
+                        authors = [a.strip() for a in re.split(r'[,;]', author_text) if a.strip()]
+
+                    # Look for date
+                    date_match = re.search(r'(\w+\s+\d{1,2},?\s+\d{4}|\d{4})', parent.get_text())
+                    if date_match:
+                        try:
+                            published_date = datetime.strptime(date_match.group(1), "%B %d, %Y")
+                        except:
+                            try:
+                                published_date = datetime.strptime(date_match.group(1), "%Y")
+                            except:
+                                pass
+
+                url = f"{self.ABSTRACT_URL}?abstract_id={paper_id}"
+
+                papers.append(Paper(
+                    paper_id=paper_id,
+                    title=title,
+                    authors=authors,
+                    abstract=abstract,
+                    doi=f"10.2139/ssrn.{paper_id}",
+                    published_date=published_date,
+                    pdf_url="",
+                    url=url,
+                    source="ssrn",
+                    categories=[],
+                    keywords=[],
+                    citations=0,
+                    references=[],
+                    extra={"ssrn_id": paper_id}
+                ))
+
+            except Exception as e:
+                logger.warning(f"Error parsing SSRN result: {e}")
+                continue
+
+        return papers
+
+    def _parse_paper_page(self, content: bytes, paper_id: str) -> Optional[Paper]:
+        """Parse a paper detail page."""
+        try:
+            soup = BeautifulSoup(content, 'lxml')
+
+            # Title - try multiple selectors
+            title = ""
+            for selector in [
+                ("h1", {}),
+                ("meta", {"property": "og:title"}),
+                ("meta", {"name": "citation_title"}),
+                ("div", {"class": "title"})
+            ]:
+                elem = soup.find(selector[0], selector[1])
+                if elem:
+                    title = elem.get("content") if elem.name == "meta" else elem.get_text(strip=True)
+                    if title:
+                        break
+
+            if not title:
+                return None
+
+            # Authors
+            authors = []
+            # Try meta tags first
+            for meta in soup.find_all("meta", {"name": "citation_author"}):
+                name = meta.get("content", "")
+                if name:
+                    authors.append(name)
+
+            # Fallback to page content
+            if not authors:
+                author_section = soup.find("div", {"class": re.compile(r"author", re.I)})
+                if author_section:
+                    for link in author_section.find_all("a"):
+                        name = link.get_text(strip=True)
+                        if name and len(name) > 2:
+                            authors.append(name)
+
+            # Abstract
+            abstract = ""
+            abstract_elem = soup.find("div", {"class": "abstract-text"})
+            if not abstract_elem:
+                abstract_elem = soup.find("meta", {"name": "description"})
+            if abstract_elem:
+                abstract = abstract_elem.get("content") if abstract_elem.name == "meta" else abstract_elem.get_text(strip=True)
+
+            # Date
+            published_date = datetime.min
+            date_meta = soup.find("meta", {"name": "citation_publication_date"})
+            if date_meta:
+                date_str = date_meta.get("content", "")
+                try:
+                    published_date = datetime.strptime(date_str, "%Y/%m/%d")
+                except:
+                    try:
+                        published_date = datetime.strptime(date_str, "%Y-%m-%d")
+                    except:
+                        pass
+
+            # Keywords
+            keywords = []
+            keywords_meta = soup.find("meta", {"name": "citation_keywords"})
+            if keywords_meta:
+                kw_text = keywords_meta.get("content", "")
+                keywords = [k.strip() for k in kw_text.split(",") if k.strip()]
+
+            # URL and DOI
+            url = f"{self.ABSTRACT_URL}?abstract_id={paper_id}"
+            doi = f"10.2139/ssrn.{paper_id}"
+
+            # PDF URL
+            pdf_url = ""
+            pdf_meta = soup.find("meta", {"name": "citation_pdf_url"})
+            if pdf_meta:
+                pdf_url = pdf_meta.get("content", "")
+
+            return Paper(
+                paper_id=paper_id,
+                title=title,
+                authors=authors,
+                abstract=abstract[:5000] if abstract else "",
+                doi=doi,
+                published_date=published_date,
+                pdf_url=pdf_url,
+                url=url,
+                source="ssrn",
+                categories=[],
+                keywords=keywords[:10],
+                citations=0,
+                references=[],
+                extra={"ssrn_id": paper_id}
+            )
+
+        except Exception as e:
+            logger.error(f"Error parsing SSRN page: {e}")
+            return None

paper_search_mcp/paper.py

@@ -0,0 +1,69 @@
+# paper_search_mcp/paper.py
+from dataclasses import dataclass
+from datetime import datetime
+from typing import List, Dict, Optional
+
+@dataclass
+class Paper:
+    """Standardized paper format with core fields for academic sources"""
+    # 核心字段（必填，但允许空值或默认值）
+    paper_id: str              # Unique identifier (e.g., arXiv ID, PMID, DOI)
+    title: str                 # Paper title
+    authors: List[str]         # List of author names
+    abstract: str              # Abstract text
+    doi: str                   # Digital Object Identifier
+    published_date: datetime   # Publication date
+    pdf_url: str               # Direct PDF link
+    url: str                   # URL to paper page
+    source: str                # Source platform (e.g., 'arxiv', 'pubmed')
+
+    # 可选字段
+    updated_date: Optional[datetime] = None    # Last updated date
+    categories: List[str] = None               # Subject categories
+    keywords: List[str] = None                 # Keywords
+    citations: int = 0                         # Citation count
+    references: Optional[List[str]] = None     # List of reference IDs/DOIs
+    extra: Optional[Dict] = None               # Source-specific extra metadata
+
+    def __post_init__(self):
+        """Post-initialization to handle default values"""
+        if self.authors is None:
+            self.authors = []
+        if self.categories is None:
+            self.categories = []
+        if self.keywords is None:
+            self.keywords = []
+        if self.references is None:
+            self.references = []
+        if self.extra is None:
+            self.extra = {}
+
+    def to_dict(self, abstract_limit: int = 200) -> Dict:
+        """Convert paper to dictionary format for serialization.
+
+        Args:
+            abstract_limit: Max chars for abstract. 0 = omit, -1 = full (default: 200)
+        """
+        # Process abstract based on limit
+        if abstract_limit == 0:
+            abstract = None
+        elif abstract_limit > 0 and self.abstract and len(self.abstract) > abstract_limit:
+            abstract = self.abstract[:abstract_limit] + '...'
+        else:
+            abstract = self.abstract
+
+        result = {
+            'id': self.paper_id,
+            'source': self.source or None,
+            'title': self.title,
+            'authors': self.authors if self.authors else None,
+            'abstract': abstract,
+            'date': self.published_date.strftime('%Y-%m-%d') if self.published_date else None,
+            'doi': self.doi or None,
+            'pdf': self.pdf_url or None,  # Include when available (e.g., open access)
+            'categories': self.categories if self.categories else None,
+            'citations': self.citations if self.citations else None,
+        }
+
+        # Remove None/empty values
+        return {k: v for k, v in result.items() if v is not None and v != ''}

paper_search_mcp/pdf_utils.py

@@ -0,0 +1,67 @@
+"""PDF text extraction utilities with pdftotext (poppler) support."""
+
+import subprocess
+import shutil
+from typing import Optional
+
+
+def extract_text_from_pdf(pdf_path: str) -> str:
+    """Extract text from a PDF file.
+
+    Uses pdftotext (poppler) for best quality extraction of academic papers,
+    with fallback to PyPDF2 if pdftotext is not available.
+
+    Args:
+        pdf_path: Path to the PDF file
+
+    Returns:
+        Extracted text content
+    """
+    # Try pdftotext first (better quality for academic papers)
+    text = _extract_with_pdftotext(pdf_path)
+    if text is not None:
+        return text
+
+    # Fallback to PyPDF2
+    return _extract_with_pypdf(pdf_path)
+
+
+def _extract_with_pdftotext(pdf_path: str) -> Optional[str]:
+    """Extract text using pdftotext (poppler).
+
+    Returns None if pdftotext is not available.
+    """
+    if not shutil.which('pdftotext'):
+        return None
+
+    try:
+        # -layout preserves the original physical layout
+        # -enc UTF-8 ensures proper encoding
+        result = subprocess.run(
+            ['pdftotext', '-layout', '-enc', 'UTF-8', pdf_path, '-'],
+            capture_output=True,
+            text=True,
+            timeout=60
+        )
+        if result.returncode == 0:
+            return result.stdout.strip()
+        return None
+    except (subprocess.TimeoutExpired, subprocess.SubprocessError):
+        return None
+
+
+def _extract_with_pypdf(pdf_path: str) -> str:
+    """Extract text using PyPDF2 as fallback."""
+    try:
+        from pypdf import PdfReader
+    except ImportError:
+        from PyPDF2 import PdfReader
+
+    try:
+        reader = PdfReader(pdf_path)
+        text = ""
+        for page in reader.pages:
+            text += page.extract_text() + "\n"
+        return text.strip()
+    except Exception as e:
+        return f"Error extracting text: {e}"