academic-search-mcp 0.1.3__py3-none-any.whl

paper_search_mcp/academic_platforms/cyberleninka.py

@@ -0,0 +1,396 @@
+"""CyberLeninka integration for Russian academic papers.
+
+CyberLeninka is a Russian open access repository with scientific articles
+from Russian journals. Supports filtering by VAK, RSCI, and SCOPUS indexed journals.
+"""
+from typing import List, Optional, Dict
+from datetime import datetime
+from curl_cffi import requests
+from bs4 import BeautifulSoup
+import re
+import os
+import logging
+import time
+
+from ..paper import Paper
+from ..pdf_utils import extract_text_from_pdf
+
+logger = logging.getLogger(__name__)
+
+
+class CyberLeninkaSearcher:
+    """Searcher for CyberLeninka Russian academic repository.
+
+    Supports filtering by:
+    - VAK (ВАК) - Russian Higher Attestation Commission journals
+    - RSCI (РИНЦ) - Russian Science Citation Index
+    - SCOPUS - Scopus indexed journals
+    - Subject categories (e.g., economics, law, medicine)
+    """
+
+    BASE_URL = "https://cyberleninka.ru"
+    API_URL = f"{BASE_URL}/api/search"
+
+    # Catalog IDs for journal indexing filters
+    CATALOGS = {
+        "vak": 8,      # ВАК
+        "rsci": 22,    # РИНЦ
+        "scopus": 2,   # SCOPUS
+    }
+
+    # Common subject category IDs (terms)
+    CATEGORIES = {
+        "economics": 35,
+        "law": 36,
+        "medicine": 43,
+        "psychology": 46,
+        "sociology": 47,
+        "pedagogy": 44,
+        "philosophy": 48,
+        "history": 41,
+        "politics": 45,
+        "philology": 49,
+    }
+
+    def __init__(self):
+        """Initialize CyberLeninka searcher."""
+        self.impersonate = "chrome"
+        self.last_request_time = 0
+
+    def _rate_limit(self, delay: float = 0.5):
+        """Apply rate limiting."""
+        elapsed = time.time() - self.last_request_time
+        if elapsed < delay:
+            time.sleep(delay - elapsed)
+        self.last_request_time = time.time()
+
+    def search(
+        self,
+        query: str,
+        max_results: int = 10,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+        catalog: Optional[str] = None,
+        category: Optional[str] = None,
+        **kwargs
+    ) -> List[Paper]:
+        """Search CyberLeninka for papers.
+
+        Args:
+            query: Search query string
+            max_results: Maximum number of papers (default: 10, max: 100)
+            date_from: Start date YYYY-MM-DD (only year is used)
+            date_to: End date YYYY-MM-DD (only year is used)
+            catalog: Filter by indexing: 'vak', 'rsci', or 'scopus'
+            category: Filter by subject: 'economics', 'law', 'medicine', etc.
+            **kwargs: Additional parameters
+
+        Returns:
+            List of Paper objects
+        """
+        papers = []
+
+        try:
+            self._rate_limit()
+
+            # Build request payload
+            payload = {
+                "q": query,
+                "mode": "articles",
+                "size": min(max_results, 100),
+                "from": kwargs.get("offset", 0)
+            }
+
+            # Date filters (year only)
+            if date_from:
+                payload["year_from"] = int(date_from[:4])
+            if date_to:
+                payload["year_to"] = int(date_to[:4])
+
+            # Catalog filter (VAK, RSCI, SCOPUS)
+            if catalog and catalog.lower() in self.CATALOGS:
+                payload["catalogs"] = [self.CATALOGS[catalog.lower()]]
+
+            # Category/subject filter
+            if category and category.lower() in self.CATEGORIES:
+                payload["terms"] = [self.CATEGORIES[category.lower()]]
+
+            response = requests.post(
+                self.API_URL,
+                json=payload,
+                timeout=30,
+                impersonate=self.impersonate
+            )
+
+            if response.status_code != 200:
+                logger.error(f"CyberLeninka search failed with status {response.status_code}")
+                return papers
+
+            data = response.json()
+            articles = data.get("articles", [])
+
+            logger.info(f"CyberLeninka search: found {len(articles)} articles for '{query}'")
+
+            for article in articles:
+                try:
+                    paper = self._parse_search_result(article)
+                    if paper:
+                        papers.append(paper)
+                except Exception as e:
+                    logger.warning(f"Error parsing CyberLeninka result: {e}")
+                    continue
+
+        except Exception as e:
+            logger.error(f"CyberLeninka search error: {e}")
+
+        return papers
+
+    def _parse_search_result(self, article: Dict) -> Optional[Paper]:
+        """Parse a search result into a Paper object."""
+        try:
+            # Extract article ID from link
+            link = article.get("link", "")
+            paper_id = link.split("/")[-1] if link else ""
+
+            title = article.get("name", "")
+            if not title:
+                return None
+
+            # Authors
+            authors = []
+            authors_data = article.get("authors", [])
+            if isinstance(authors_data, list):
+                for author in authors_data:
+                    if isinstance(author, dict):
+                        name = author.get("name", "")
+                    else:
+                        name = str(author)
+                    if name:
+                        authors.append(name)
+
+            # Abstract/annotation
+            abstract = article.get("annotation", "") or ""
+
+            # Year
+            year = article.get("year")
+            published_date = datetime(int(year), 1, 1) if year else datetime.min
+
+            # Journal info
+            journal = article.get("journal", {})
+            journal_name = journal.get("name", "") if isinstance(journal, dict) else ""
+
+            # URL
+            url = f"{self.BASE_URL}{link}" if link else ""
+            pdf_url = f"{url}/pdf" if url else ""
+
+            # Keywords
+            keywords = []
+            kw_data = article.get("keywords", [])
+            if isinstance(kw_data, list):
+                keywords = [k for k in kw_data if k]
+
+            return Paper(
+                paper_id=paper_id,
+                title=title,
+                authors=authors,
+                abstract=abstract[:5000] if abstract else "",
+                doi="",  # DOI requires fetching article page
+                published_date=published_date,
+                pdf_url=pdf_url,
+                url=url,
+                source="cyberleninka",
+                categories=[],
+                keywords=keywords[:10],
+                citations=0,
+                references=[],
+                extra={
+                    "journal": journal_name,
+                    "year": year,
+                    "cyberleninka_id": paper_id
+                }
+            )
+
+        except Exception as e:
+            logger.error(f"Error parsing CyberLeninka result: {e}")
+            return None
+
+    def get_paper_by_id(self, paper_id: str) -> Optional[Paper]:
+        """Get a specific paper by its CyberLeninka ID (URL slug).
+
+        Args:
+            paper_id: CyberLeninka article slug (e.g., 'tsifrovoy-suverenitet-i-strany-vostoka')
+
+        Returns:
+            Paper object or None
+        """
+        try:
+            self._rate_limit()
+
+            url = f"{self.BASE_URL}/article/n/{paper_id}"
+            response = requests.get(url, timeout=30, impersonate=self.impersonate)
+
+            if response.status_code == 404:
+                return None
+            response.raise_for_status()
+
+            return self._parse_article_page(response.text, paper_id, url)
+
+        except Exception as e:
+            logger.error(f"Error fetching CyberLeninka paper {paper_id}: {e}")
+            return None
+
+    def _parse_article_page(self, html: str, paper_id: str, url: str) -> Optional[Paper]:
+        """Parse an article page into a Paper object."""
+        try:
+            soup = BeautifulSoup(html, 'lxml')
+
+            # Title from meta tag
+            title = ""
+            title_meta = soup.find("meta", {"name": "citation_title"})
+            if title_meta:
+                title = title_meta.get("content", "")
+            if not title:
+                title_meta = soup.find("meta", {"property": "og:title"})
+                if title_meta:
+                    title = title_meta.get("content", "")
+
+            if not title:
+                return None
+
+            # Authors from meta tags
+            authors = []
+            for meta in soup.find_all("meta", {"name": "citation_author"}):
+                name = meta.get("content", "")
+                if name:
+                    authors.append(name)
+
+            # Abstract from meta description
+            abstract = ""
+            desc_meta = soup.find("meta", {"name": "description"})
+            if desc_meta:
+                abstract = desc_meta.get("content", "")
+
+            # DOI
+            doi = ""
+            doi_div = soup.find("div", {"class": "label-doi"})
+            if doi_div:
+                doi_match = re.search(r'DOI:\s*(10\.\S+)', doi_div.get("title", ""))
+                if doi_match:
+                    doi = doi_match.group(1)
+
+            # Publication date
+            published_date = datetime.min
+            date_meta = soup.find("meta", {"name": "citation_publication_date"})
+            if date_meta:
+                date_str = date_meta.get("content", "")
+                try:
+                    published_date = datetime.strptime(date_str, "%Y")
+                except:
+                    pass
+
+            # Journal
+            journal = ""
+            journal_meta = soup.find("meta", {"name": "citation_journal_title"})
+            if journal_meta:
+                journal = journal_meta.get("content", "")
+
+            # Keywords
+            keywords = []
+            kw_meta = soup.find("meta", {"name": "citation_keywords"})
+            if kw_meta:
+                kw_text = kw_meta.get("content", "")
+                keywords = [k.strip() for k in kw_text.split(",") if k.strip()]
+
+            # PDF URL
+            pdf_url = ""
+            pdf_meta = soup.find("meta", {"name": "citation_pdf_url"})
+            if pdf_meta:
+                pdf_url = pdf_meta.get("content", "")
+
+            # ISSN
+            issn = ""
+            issn_meta = soup.find("meta", {"name": "citation_issn"})
+            if issn_meta:
+                issn = issn_meta.get("content", "")
+
+            return Paper(
+                paper_id=paper_id,
+                title=title,
+                authors=authors,
+                abstract=abstract[:5000] if abstract else "",
+                doi=doi,
+                published_date=published_date,
+                pdf_url=pdf_url or f"{url}/pdf",
+                url=url,
+                source="cyberleninka",
+                categories=[],
+                keywords=keywords[:10],
+                citations=0,
+                references=[],
+                extra={
+                    "journal": journal,
+                    "issn": issn,
+                    "cyberleninka_id": paper_id
+                }
+            )
+
+        except Exception as e:
+            logger.error(f"Error parsing CyberLeninka page: {e}")
+            return None
+
+    def download_pdf(self, paper_id: str, save_path: str = "./downloads") -> str:
+        """Download PDF from CyberLeninka.
+
+        Args:
+            paper_id: CyberLeninka article slug
+            save_path: Directory to save
+
+        Returns:
+            Path to PDF or error message
+        """
+        try:
+            os.makedirs(save_path, exist_ok=True)
+
+            self._rate_limit()
+
+            # PDF URL format
+            pdf_url = f"{self.BASE_URL}/article/n/{paper_id}/pdf"
+
+            response = requests.get(pdf_url, timeout=60, impersonate=self.impersonate)
+
+            if response.status_code != 200:
+                return f"PDF download failed for {paper_id}"
+
+            content_type = response.headers.get("Content-Type", "")
+
+            if "pdf" in content_type.lower():
+                filename = f"cyberleninka_{paper_id}.pdf"
+                file_path = os.path.join(save_path, filename)
+
+                with open(file_path, 'wb') as f:
+                    f.write(response.content)
+
+                return file_path
+
+            return f"PDF not available for {paper_id}"
+
+        except Exception as e:
+            logger.error(f"Error downloading CyberLeninka PDF: {e}")
+            return f"Failed to download PDF: {e}"
+
+    def read_paper(self, paper_id: str, save_path: str = "./downloads") -> str:
+        """Read and extract text from a CyberLeninka paper PDF.
+
+        Args:
+            paper_id: CyberLeninka article slug
+            save_path: Directory for PDF storage
+
+        Returns:
+            Extracted text or error message
+        """
+        pdf_path = self.download_pdf(paper_id, save_path)
+        if not os.path.exists(pdf_path):
+            return pdf_path  # Return error message
+
+        text = extract_text_from_pdf(pdf_path)
+        return text if text else "Failed to extract text from PDF"

paper_search_mcp/academic_platforms/google_scholar.py

@@ -0,0 +1,249 @@
+from typing import List, Optional
+from datetime import datetime
+import requests
+from bs4 import BeautifulSoup
+import time
+import random
+from ..paper import Paper
+import logging
+
+logger = logging.getLogger(__name__)
+
+class PaperSource:
+    """Abstract base class for paper sources"""
+    def search(self, query: str, **kwargs) -> List[Paper]:
+        raise NotImplementedError
+
+    def download_pdf(self, paper_id: str, save_path: str) -> str:
+        raise NotImplementedError
+
+    def read_paper(self, paper_id: str, save_path: str) -> str:
+        raise NotImplementedError
+    
+
+class GoogleScholarSearcher(PaperSource):
+    """Custom implementation of Google Scholar paper search"""
+    
+    SCHOLAR_URL = "https://scholar.google.com/scholar"
+    BROWSERS = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
+    ]
+
+    def __init__(self):
+        self._setup_session()
+
+    def _setup_session(self):
+        """Initialize session with random user agent"""
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': random.choice(self.BROWSERS),
+            'Accept': 'text/html,application/xhtml+xml',
+            'Accept-Language': 'en-US,en;q=0.9'
+        })
+
+    def _extract_year(self, text: str) -> Optional[int]:
+        """Extract year from publication info"""
+        for word in text.split():
+            if word.isdigit() and 1900 <= int(word) <= datetime.now().year:
+                return int(word)
+        return None
+
+    def _extract_cluster_id(self, item) -> Optional[str]:
+        """Extract Google Scholar cluster ID from result item"""
+        import re
+        # Look for cluster/cites ID in the links (gs_fl div contains "Cited by", "All versions", etc.)
+        links_div = item.find('div', class_='gs_fl')
+        if links_div:
+            for a in links_div.find_all('a', href=True):
+                href = a['href']
+                # Match cluster=ID or cites=ID
+                match = re.search(r'(?:cluster|cites)=(\d+)', href)
+                if match:
+                    return match.group(1)
+
+        # Also check data-cid attribute on the result container
+        if item.get('data-cid'):
+            return item['data-cid']
+
+        return None
+
+    def _extract_citations(self, item) -> int:
+        """Extract citation count from result item"""
+        import re
+        links_div = item.find('div', class_='gs_fl')
+        if links_div:
+            for a in links_div.find_all('a'):
+                text = a.get_text()
+                if 'Cited by' in text:
+                    match = re.search(r'Cited by (\d+)', text)
+                    if match:
+                        return int(match.group(1))
+        return 0
+
+    def _parse_paper(self, item) -> Optional[Paper]:
+        """Parse single paper entry from HTML"""
+        try:
+            # Extract main paper elements
+            title_elem = item.find('h3', class_='gs_rt')
+            info_elem = item.find('div', class_='gs_a')
+            abstract_elem = item.find('div', class_='gs_rs')
+
+            if not title_elem or not info_elem:
+                return None
+
+            # Process title and URL
+            title = title_elem.get_text(strip=True).replace('[PDF]', '').replace('[HTML]', '')
+            link = title_elem.find('a', href=True)
+            url = link['href'] if link else ''
+
+            # Extract cluster ID (Google Scholar's unique paper identifier)
+            cluster_id = self._extract_cluster_id(item)
+
+            # Fallback to URL hash if no cluster ID found
+            paper_id = cluster_id if cluster_id else f"gs_{abs(hash(url))}"
+
+            # Note: DOI not available in GS search results (would require extra requests)
+
+            # Process author info
+            info_text = info_elem.get_text()
+            authors = [a.strip() for a in info_text.split('-')[0].split(',')]
+            year = self._extract_year(info_text)
+
+            # Extract citation count
+            citations = self._extract_citations(item)
+
+            # Create paper object
+            return Paper(
+                paper_id=paper_id,
+                title=title,
+                authors=authors,
+                abstract=abstract_elem.get_text() if abstract_elem else "",
+                url=url,
+                pdf_url="",
+                published_date=datetime(year, 1, 1) if year else None,
+                updated_date=None,
+                source="google_scholar",
+                categories=[],
+                keywords=[],
+                doi="",  # Not available in GS search results
+                citations=citations
+            )
+        except Exception as e:
+            logger.warning(f"Failed to parse paper: {e}")
+            return None
+
+    def search(self, query: str, max_results: int = 10, date_from: str = None, date_to: str = None) -> List[Paper]:
+        """
+        Search Google Scholar with custom parameters
+
+        Args:
+            query: Search query string
+            max_results: Maximum number of papers to return
+            date_from: Start date in YYYY-MM-DD format (only year is used)
+            date_to: End date in YYYY-MM-DD format (only year is used)
+        """
+        papers = []
+        start = 0
+        results_per_page = min(10, max_results)
+
+        while len(papers) < max_results:
+            try:
+                # Construct search parameters
+                params = {
+                    'q': query,
+                    'start': start,
+                    'hl': 'en',
+                    'as_sdt': '0,5'  # Include articles and citations
+                }
+
+                # Add year filters if provided (extract year from YYYY-MM-DD format)
+                if date_from:
+                    try:
+                        year_from = int(date_from.split('-')[0])
+                        params['as_ylo'] = year_from
+                    except (ValueError, IndexError):
+                        logger.warning(f"Invalid date_from format: {date_from}")
+
+                if date_to:
+                    try:
+                        year_to = int(date_to.split('-')[0])
+                        params['as_yhi'] = year_to
+                    except (ValueError, IndexError):
+                        logger.warning(f"Invalid date_to format: {date_to}")
+
+                # Make request with random delay
+                time.sleep(random.uniform(1.0, 3.0))
+                response = self.session.get(self.SCHOLAR_URL, params=params)
+                
+                if response.status_code != 200:
+                    logger.error(f"Search failed with status {response.status_code}")
+                    break
+
+                # Parse results
+                soup = BeautifulSoup(response.text, 'html.parser')
+                results = soup.find_all('div', class_='gs_ri')
+
+                if not results:
+                    break
+
+                # Process each result
+                for item in results:
+                    if len(papers) >= max_results:
+                        break
+                        
+                    paper = self._parse_paper(item)
+                    if paper:
+                        papers.append(paper)
+
+                start += results_per_page
+
+            except Exception as e:
+                logger.error(f"Search error: {e}")
+                break
+
+        return papers[:max_results]
+
+    def download_pdf(self, paper_id: str, save_path: str) -> str:
+        """
+        Google Scholar doesn't support direct PDF downloads
+        
+        Raises:
+            NotImplementedError: Always raises this error
+        """
+        raise NotImplementedError(
+            "Google Scholar doesn't provide direct PDF downloads. "
+            "Please use the paper URL to access the publisher's website."
+        )
+
+    def read_paper(self, paper_id: str, save_path: str = "./downloads") -> str:
+        """
+        Google Scholar doesn't support direct paper reading
+        
+        Returns:
+            str: Message indicating the feature is not supported
+        """
+        return (
+            "Google Scholar doesn't support direct paper reading. "
+            "Please use the paper URL to access the full text on the publisher's website."
+        )
+
+if __name__ == "__main__":
+    # Test Google Scholar searcher
+    searcher = GoogleScholarSearcher()
+    
+    print("Testing search functionality...")
+    query = "machine learning"
+    max_results = 5
+    
+    try:
+        papers = searcher.search(query, max_results=max_results)
+        print(f"\nFound {len(papers)} papers for query '{query}':")
+        for i, paper in enumerate(papers, 1):
+            print(f"\n{i}. {paper.title}")
+            print(f"   Authors: {', '.join(paper.authors)}")
+            print(f"   Citations: {paper.citations}")
+            print(f"   URL: {paper.url}")
+    except Exception as e:
+        print(f"Error during search: {e}")