scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/.legacy/metadata/doi/sources/_ArXivSource.py

@@ -1,306 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-14 21:19:10 (ywatanabe)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/doi/sources/_ArXivSource.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-from typing import Dict, List, Optional, Union
-
-import feedparser
-import requests
-from bs4 import BeautifulSoup
-from tenacity import (
-    retry,
-    retry_if_exception_type,
-    retry_if_result,
-    stop_after_attempt,
-    wait_exponential,
-)
-
-from scitex import logging
-
-from ..utils import to_complete_metadata_structure
-from ._BaseDOISource import BaseDOISource
-
-logger = logging.getLogger(__name__)
-import json
-
-
-class ArXivSource(BaseDOISource):
-    """ArXiv source for open access papers."""
-
-    def __init__(self, email: str = "research@example.com"):
-        super().__init__()
-        self.email = email
-        self.base_url = "http://export.arxiv.org/api/query"
-
-    def _get_user_agent(self) -> str:
-        """Get ArXiv-specific user agent."""
-        return f"SciTeX Scholar (mailto:{self.email})"
-
-    @property
-    def name(self) -> str:
-        return "arXiv"
-
-    @property
-    def rate_limit_delay(self) -> float:
-        return 3.0
-
-    def search(
-        self,
-        title: Optional[str] = None,
-        year: Optional[Union[int, str]] = None,
-        authors: Optional[List[str]] = None,
-        doi: Optional[str] = None,
-        max_results=1,
-        return_as: Optional[str] = "dict",
-        **kwargs,
-    ) -> Optional[Dict]:
-        """When doi is provided, all the information other than doi is ignored"""
-        if doi:
-            return self._search_by_doi(doi, return_as)
-        else:
-            return self._search_by_metadata(
-                title, year, authors, max_results, return_as
-            )
-
-    @retry(
-        stop=stop_after_attempt(3),
-        wait=wait_exponential(multiplier=1, min=2, max=10),
-        retry=retry_if_exception_type((requests.HTTPError, requests.ConnectionError)),
-    )
-    def _search_by_doi(self, doi: str, return_as: str) -> Optional[Dict]:
-        """Search solely on doi"""
-        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
-        arxiv_id = doi.split("arXiv.")[-1]
-
-        params = {
-            "search_query": f'id:"{arxiv_id}"',
-            "start": 0,
-            "max_results": 1,
-            "sortBy": "relevance",
-            "sortOrder": "descending",
-        }
-
-        try:
-            assert return_as in [
-                "dict",
-                "json",
-            ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(self.base_url, params=params, timeout=30)
-            response.raise_for_status()
-
-            feed = feedparser.parse(response.text)
-            for entry in feed.entries:
-                return self._extract_metadata_from_entry(entry, return_as)
-            return None
-        except Exception as exc:
-            logger.warn(f"ArXiv DOI search error: {exc}")
-            return None
-
-    @retry(
-        stop=stop_after_attempt(3),
-        wait=wait_exponential(multiplier=1, min=2, max=10),
-        retry=retry_if_exception_type((requests.HTTPError, requests.ConnectionError)),
-    )
-    def _search_by_metadata(
-        self,
-        title: str,
-        year: Optional[Union[int, str]] = None,
-        authors: Optional[List[str]] = None,
-        max_results: Optional[int] = 1,
-        return_as: Optional[str] = "dict",
-    ) -> Optional[Dict]:
-        """Search by metadata other than doi"""
-        if not title:
-            return None
-
-        params = {
-            "search_query": f'ti:"{title}"',
-            "start": 0,
-            "max_results": max_results,
-            "sortBy": "relevance",
-            "sortOrder": "descending",
-        }
-
-        try:
-            assert return_as in [
-                "dict",
-                "json",
-            ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(self.base_url, params=params, timeout=30)
-            response.raise_for_status()
-
-            feed = feedparser.parse(response.text)
-            for entry in feed.entries:
-                entry_title = entry.get("title", "").replace("\n", " ").strip()
-                if self._is_title_match(title, entry_title):
-                    return self._extract_metadata_from_entry(entry, return_as)
-            return None
-        except Exception as exc:
-            logger.warn(f"ArXiv metadata error: {exc}")
-            return None
-
-    @retry(
-        stop=stop_after_attempt(3),
-        wait=wait_exponential(multiplier=1, min=2, max=10),
-        retry=retry_if_exception_type((requests.HTTPError, requests.ConnectionError)),
-    )
-    def _scrape_doi(self, url_publisher):
-        try:
-            response = requests.get(url_publisher)
-            soup = BeautifulSoup(response.content, "html.parser")
-            doi_link = soup.find("a", href=lambda href: href and "doi.org" in href)
-            doi = doi_link.get("href").split("doi.org/")[-1]
-            return doi
-        except Exception as e:
-            logger.warn(f"DOI not scraped from {url_publisher}\n{str(e)}")
-
-    @retry(
-        stop=stop_after_attempt(3),
-        wait=wait_exponential(multiplier=1, min=2, max=10),
-        retry=retry_if_exception_type((requests.HTTPError, requests.ConnectionError)),
-    )
-    def _extract_metadata_from_entry(self, entry, return_as: str) -> Optional[Dict]:
-        """Extract metadata from ArXiv entry"""
-        arxiv_id = entry.id.split("/abs/")[-1].split("v")[0]
-        year = entry.get("published", "")[:4]
-        title = entry.get("title")
-        if title and title.endswith("."):
-            title = title[:-1]
-        abstract = entry.get("summary")
-        authors = [author.get("name") for author in entry.get("authors")]
-        url_publisher = entry.get("link")
-        doi = self._scrape_doi(url_publisher)
-
-        # Generate ArXiv DOI if not found
-        if not doi:
-            doi = f"10.48550/arxiv.{arxiv_id.lower()}"
-
-        metadata = {
-            "id": {
-                "doi": doi,
-                "doi_sources": [self.name],
-                "arxiv_id": arxiv_id,
-                "arxiv_id_sources": [self.name],
-            },
-            "basic": {
-                "title": title if title else None,
-                "title_sources": [self.name] if title else None,
-                "year": year if year else None,
-                "year_sources": [self.name] if year else None,
-                "abstract": abstract.replace("\n", " ") if abstract else None,
-                "abstract_sources": [self.name] if abstract else None,
-                "authors": authors if authors else None,
-                "authors_sources": [self.name] if authors else None,
-            },
-            "publication": {
-                "journal": self.name,
-                "journal_sources": [self.name],
-            },
-            "url": {
-                "doi": f"https://doi.org/{doi}",
-                "doi_sources": [self.name],
-                "publisher": url_publisher if url_publisher else None,
-                "publisher_sources": [self.name] if url_publisher else None,
-            },
-            "system": {
-                f"searched_by_{self.name}": True,
-            },
-        }
-
-        metadata = to_complete_metadata_structure(metadata)
-
-        if return_as == "dict":
-            return metadata
-        if return_as == "json":
-            return json.dumps(metadata, indent=2)
-
-    # def _extract_metadata_from_entry(
-    #     self, entry, return_as: str
-    # ) -> Optional[Dict]:
-    #     """Extract metadata from ArXiv entry"""
-    #     arxiv_id = entry.id.split("/abs/")[-1].split("v")[0]
-    #     year = entry.get("published", "")[:4]
-    #     title = entry.get("title")
-    #     if title and title.endswith("."):
-    #         title = title[:-1]
-
-    #     abstract = entry.get("summary")
-    #     authors = [author.get("name") for author in entry.get("authors")]
-    #     url_publisher = entry.get("link")
-    #     doi = self._scrape_doi(url_publisher)
-
-    #     metadata = {
-    #         "id": {
-    #             "doi": doi if doi else None,
-    #             "doi_sources": [self.name] if doi else None,
-    #             "arxiv_id": arxiv_id if arxiv_id else None,
-    #             "arxiv_id_sources": [self.name] if arxiv_id else None,
-    #         },
-    #         "basic": {
-    #             "title": title if title else None,
-    #             "title_sources": [self.name] if title else None,
-    #             "year": year if year else None,
-    #             "year_sources": [self.name] if year else None,
-    #             "abstract": abstract.replace("\n", " ") if abstract else None,
-    #             "abstract_sources": [self.name] if abstract else None,
-    #             "authors": authors if authors else None,
-    #             "authors_sources": [self.name] if authors else None,
-    #         },
-    #         "publication": {
-    #             "journal": self.name,
-    #             "journal_sources": [self.name],
-    #         },
-    #         "url": {
-    #             "doi": "https://doi.org/" + doi if doi else None,
-    #             "doi_sources": [self.name] if doi else None,
-    #             "publisher": url_publisher if url_publisher else None,
-    #             "publisher_sources": [self.name] if url_publisher else None,
-    #         },
-    #         "system": {
-    #             f"searched_by_{self.name}": True,
-    #         },
-    #     }
-
-    #     metadata = to_complete_metadata_structure(metadata)
-    #     if return_as == "dict":
-    #         return metadata
-    #     if return_as == "json":
-    #         return json.dumps(metadata, indent=2)
-
-
-if __name__ == "__main__":
-    from pprint import pprint
-
-    TITLE = "Attention is All You Need"
-    DOI = "https://doi.org/10.48550/arXiv.1706.03762"
-
-    # Example: ArXiv search
-    source = ArXivSource("test@example.com")
-
-    outputs = {}
-
-    # Search by title
-    outputs["metadata_by_title_dict"] = source.search(title=TITLE)
-    outputs["metadata_by_title_json"] = source.search(title=TITLE, return_as="json")
-
-    # Search by DOI
-    outputs["metadata_by_doi_dict"] = source.search(doi=DOI)
-    outputs["metadata_by_doi_json"] = source.search(doi=DOI, return_as="json")
-
-    for k, v in outputs.items():
-        print("----------------------------------------")
-        print(k)
-        print("----------------------------------------")
-        pprint(v)
-
-# python -m scitex.scholar.metadata.doi.sources._ArXivSource
-
-# EOF

scitex/scholar/.legacy/metadata/doi/sources/_BaseDOISource.py

@@ -1,347 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-14 20:09:47 (ywatanabe)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/doi/sources/_BaseDOISource.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-"""
-Abstract base class for DOI sources with enhanced rate limit handling.
-
-This module defines the interface that all DOI sources must implement,
-including automatic rate limit detection and retry mechanisms.
-"""
-
-import asyncio
-import re
-import time
-from abc import ABC, abstractmethod
-from typing import List, Optional
-
-import requests
-
-from scitex import logging
-
-from ..utils import PubMedConverter, TextNormalizer, URLDOIExtractor
-
-logger = logging.getLogger(__name__)
-
-
-class BaseDOISource(ABC):
-    """Abstract base class for DOI sources with enhanced rate limit handling."""
-
-    def __init__(self, email: str = "research@example.com"):
-        """Initialize base source."""
-        self.email = email
-        self.rate_limit_handler = None  # Will be injected by SingleDOIResolver
-        self.last_request_time = 0.0
-        self._request_count = 0
-
-        # Lazy-loaded utilities - will be initialized when first accessed
-        self._text_normalizer = None
-        self._url_doi_extractor = None
-        self._pubmed_converter = None
-        self._session = None
-
-    @abstractmethod
-    def search(
-        self,
-        title: str,
-        year: Optional[int] = None,
-        authors: Optional[List[str]] = None,
-    ) -> Optional[str]:
-        """Search for DOI by title."""
-        pass
-
-    @property
-    @abstractmethod
-    def name(self) -> str:
-        """Source name for logging."""
-        pass
-
-    @property
-    def rate_limit_delay(self) -> float:
-        return 1.0
-
-    def set_rate_limit_handler(self, handler):
-        """Set the rate limit handler for this source."""
-        self.rate_limit_handler = handler
-
-    @property
-    def text_normalizer(self):
-        """Get TextNormalizer utility with lazy loading."""
-        if self._text_normalizer is None:
-            self._text_normalizer = TextNormalizer(ascii_fallback=False)
-        return self._text_normalizer
-
-    @property
-    def url_doi_extractor(self):
-        """Get URLDOISource utility with lazy loading."""
-        if self._url_doi_extractor is None:
-            self._url_doi_extractor = URLDOIExtractor()
-        return self._url_doi_extractor
-
-    @property
-    def pubmed_converter(self):
-        """Get PubMedConverter utility with lazy loading."""
-        if self._pubmed_converter is None:
-            self._pubmed_converter = PubMedConverter()
-        return self._pubmed_converter
-
-    @property
-    def session(self):
-        if self._session is None:
-            self._session = requests.Session()
-            self._session.headers.update({"User-Agent": self._get_user_agent()})
-        return self._session
-
-    def _get_user_agent(self) -> str:
-        """Get user agent string. Override in subclasses if needed."""
-        return f"SciTeX/1.0 (mailto:{self.email})"
-
-    def _make_request_with_retry(
-        self,
-        url: str,
-        params: Optional[dict] = None,
-        headers: Optional[dict] = None,
-        timeout: float = 30.0,
-        max_retries: int = 3,
-    ) -> Optional[requests.Response]:
-        """Make HTTP request with automatic rate limit handling and retries.
-
-        Args:
-            url: Request URL
-            params: Query parameters
-            headers: Request headers
-            timeout: Request timeout
-            max_retries: Maximum number of retries
-
-        Returns:
-            Response object or None if all retries failed
-        """
-        session = getattr(self, "session", requests)
-
-        for attempt in range(max_retries + 1):
-            try:
-                # Apply rate limiting before request
-                self._apply_rate_limiting()
-
-                # Make the request
-                self._request_count += 1
-                self.last_request_time = time.time()
-
-                response = session.get(
-                    url, params=params, headers=headers, timeout=timeout
-                )
-
-                # Check for rate limits
-                if self.rate_limit_handler:
-                    rate_limit_info = self.rate_limit_handler.detect_rate_limit(
-                        source=self.name.lower(), response=response
-                    )
-
-                    if rate_limit_info:
-                        self.rate_limit_handler.record_rate_limit(rate_limit_info)
-
-                        # If this isn't the last attempt, wait and retry
-                        if attempt < max_retries:
-                            logger.info(
-                                f"Rate limited on attempt {attempt + 1}/{max_retries + 1} "
-                                f"for {self.name}, waiting {rate_limit_info.wait_time:.1f}s"
-                            )
-                            time.sleep(rate_limit_info.wait_time)
-                            continue
-                        else:
-                            logger.warning(
-                                f"Max retries exceeded for {self.name} due to rate limiting"
-                            )
-                            return None
-
-                # Check for successful response
-                if response.status_code == 200:
-                    if self.rate_limit_handler:
-                        self.rate_limit_handler.record_success(self.name.lower())
-                        # Record success for adaptive rate limiting
-                        self.rate_limit_handler.record_request_outcome(
-                            self.name.lower(), success=True
-                        )
-                    return response
-                elif response.status_code in [429, 503, 502, 504]:
-                    # Server errors that might be temporary
-                    if attempt < max_retries:
-                        wait_time = min(2**attempt, 30)  # Exponential backoff, max 30s
-                        logger.info(
-                            f"Server error {response.status_code} on attempt {attempt + 1}, "
-                            f"waiting {wait_time}s before retry"
-                        )
-                        time.sleep(wait_time)
-                        continue
-                    else:
-                        logger.warning(
-                            f"Server error {response.status_code} after max retries"
-                        )
-                        return response  # Return for caller to handle
-                else:
-                    # Other HTTP errors
-                    logger.debug(f"HTTP {response.status_code} from {self.name}: {url}")
-                    return response
-
-            except requests.exceptions.Timeout as e:
-                if self.rate_limit_handler:
-                    rate_limit_info = self.rate_limit_handler.detect_rate_limit(
-                        source=self.name.lower(), exception=e
-                    )
-                    if rate_limit_info and attempt < max_retries:
-                        self.rate_limit_handler.record_rate_limit(rate_limit_info)
-                        logger.info(
-                            f"Timeout on attempt {attempt + 1}, waiting {rate_limit_info.wait_time:.1f}s"
-                        )
-                        time.sleep(rate_limit_info.wait_time)
-                        continue
-
-                if attempt < max_retries:
-                    wait_time = min(2**attempt, 15)
-                    logger.info(
-                        f"Timeout on attempt {attempt + 1}, waiting {wait_time}s"
-                    )
-                    time.sleep(wait_time)
-                    continue
-                else:
-                    logger.warning(
-                        f"Timeout after {max_retries + 1} attempts for {self.name}"
-                    )
-                    if self.rate_limit_handler:
-                        self.rate_limit_handler.record_failure(self.name.lower(), e)
-                        # Record failure for adaptive rate limiting
-                        self.rate_limit_handler.record_request_outcome(
-                            self.name.lower(), success=False
-                        )
-                    return None
-
-            except requests.exceptions.RequestException as e:
-                if attempt < max_retries:
-                    wait_time = min(2**attempt, 15)
-                    logger.info(
-                        f"Request error on attempt {attempt + 1}: {e}, waiting {wait_time}s"
-                    )
-                    time.sleep(wait_time)
-                    continue
-                else:
-                    logger.warning(
-                        f"Request failed after {max_retries + 1} attempts: {e}"
-                    )
-                    if self.rate_limit_handler:
-                        self.rate_limit_handler.record_failure(self.name.lower(), e)
-                        # Record failure for adaptive rate limiting
-                        self.rate_limit_handler.record_request_outcome(
-                            self.name.lower(), success=False
-                        )
-                    return None
-
-            except Exception as e:
-                logger.error(f"Unexpected error in {self.name}: {e}")
-                if self.rate_limit_handler:
-                    self.rate_limit_handler.record_failure(self.name.lower(), e)
-                    # Record failure for adaptive rate limiting
-                    self.rate_limit_handler.record_request_outcome(
-                        self.name.lower(), success=False
-                    )
-                return None
-
-        return None
-
-    def _apply_rate_limiting(self):
-        """Apply rate limiting before making a request."""
-        if not self.rate_limit_handler:
-            logger.error(
-                f"No rate limit handler set for {self.name}. This should not happen!"
-            )
-            return
-
-        # Use advanced rate limiting with adaptive delays
-        wait_time = self.rate_limit_handler.get_wait_time_for_source(self.name.lower())
-        if wait_time > 0:
-            logger.debug(f"Rate limiting {self.name}: waiting {wait_time:.1f}s")
-            time.sleep(wait_time)
-
-    async def _apply_rate_limiting_async(self):
-        """Apply rate limiting before making a request (async version)."""
-        if not self.rate_limit_handler:
-            logger.error(
-                f"No rate limit handler set for {self.name}. This should not happen!"
-            )
-            return
-
-        # Use advanced rate limiting with countdown and adaptive delays
-        wait_time = self.rate_limit_handler.get_wait_time_for_source(self.name.lower())
-        if wait_time > 0:
-            await self.rate_limit_handler.wait_with_countdown_async(
-                wait_time, self.name
-            )
-
-    async def search_async(
-        self,
-        title: str,
-        year: Optional[int] = None,
-        authors: Optional[List[str]] = None,
-    ) -> Optional[str]:
-        """Async version of search method."""
-        # Apply rate limiting
-        await self._apply_rate_limiting_async()
-
-        # Run sync search in executor
-        loop = asyncio.get_event_loop()
-        try:
-            result = await loop.run_in_executor(None, self.search, title, year, authors)
-            return result
-        except Exception as e:
-            logger.error(f"Error in async search for {self.name}: {e}")
-            if self.rate_limit_handler:
-                self.rate_limit_handler.record_failure(self.name.lower(), e)
-            return None
-
-    def get_request_stats(self) -> dict:
-        """Get request statistics for this source."""
-        return {
-            "total_requests": self._request_count,
-            "last_request_time": self.last_request_time,
-            "rate_limit_delay": self.rate_limit_delay,
-        }
-
-    def extract_doi_from_url(self, url: str) -> Optional[str]:
-        """Extract DOI from URL if present."""
-        if not url:
-            return None
-
-        # Direct DOI URLs
-        if "doi.org/" in url:
-            match = re.search(r"doi\.org/(.+?)(?:\?|$|#)", url)
-            if match:
-                return match.group(1).strip()
-
-        # DOI pattern in URL
-        doi_pattern = r"10\.\d{4,}/[-._;()/:\w]+"
-        match = re.search(doi_pattern, url)
-        if match:
-            return match.group(0)
-
-        return None
-
-    def _is_title_match(
-        self, title1: str, title2: str, threshold: float = 0.95
-    ) -> bool:
-        """
-        Check if two titles match using the enhanced TextNormalizer utility.
-
-        DEPRECATED: Use self.text_normalizer.is_title_match() directly in new code.
-        This method is kept for backward compatibility.
-        """
-        return self.text_normalizer.is_title_match(title1, title2, threshold)
-
-
-# EOF