scitex 2.14.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/url_finder/.tmp/open_url/_DOIToURLResolver.py

@@ -0,0 +1,694 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-08-09 00:47:35 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/scholar/metadata/urls/open_url/_DOIToURLResolver.py
+# ----------------------------------------
+from __future__ import annotations
+
+import os
+
+__FILE__ = (
+    "./scholar/metadata/urls/open_url/_DOIToURLResolver.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+from datetime import datetime
+
+"""
+Convert DOIs to accessible publisher URLs using OpenURL resolvers.
+
+This module implements Critical Task #5: Resolve publisher URLs from DOIs
+using institutional OpenURL resolvers for authenticate_async access.
+"""
+
+import asyncio
+import json
+import re
+from pathlib import Path
+from typing import Dict, List, Optional
+from urllib.parse import urlencode
+
+import aiohttp
+from playwright.async_api import Page, async_playwright
+
+from scitex import logging
+from scitex.scholar.auth import AuthenticationManager
+from scitex.scholar.config import ScholarConfig
+
+from ._OpenURLResolver import OpenURLResolver
+
+logger = logging.getLogger(__name__)
+
+
+class DOIToURLResolver:
+    """Resolve DOIs to accessible publisher URLs via OpenURL."""
+
+    def __init__(self, config: Optional[ScholarConfig] = None):
+        """
+        Initialize DOI to URL resolver.
+
+        Args:
+            config: Scholar configuration (uses default if not provided)
+        """
+        self.config = config or ScholarConfig()
+
+        # Initialize auth manager for OpenURL resolver
+        auth_manager = AuthenticationManager(config=self.config)
+        self.openurl_resolver = OpenURLResolver(auth_manager=auth_manager)
+
+        # Initialize path manager for screenshots
+        self.path_manager = self.config.path_manager
+
+        # Cache for resolved URLs
+        self.cache_dir = Path.home() / ".scitex" / "scholar" / "url_cache"
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.cache_file = self.cache_dir / "doi_url_cache.json"
+        self.cache = self._load_cache()
+
+        # Track failures for adaptive behavior
+        self.failures = {}
+
+    async def _capture_workflow_screenshot_async(
+        self, doi: str, url: str, stage: str, page: Optional[Page] = None
+    ):
+        """
+        Capture systematic screenshots during DOI resolution workflow.
+
+        Args:
+            doi: DOI being resolved
+            url: Current URL
+            stage: Workflow stage (e.g., "doi_redirect", "openurl_result", "publisher_page", "access_check")
+            page: Existing page object (if None, creates new browser)
+        """
+        try:
+            # Create paper info from DOI for directory structure
+            paper_info = {
+                "title": f"DOI_Resolution_{doi.replace('/', '_')}",
+                "authors": [],
+                "year": None,
+                "doi": doi,
+                "journal": None,
+                "url": url,
+            }
+
+            storage_paths = self.path_manager.get_paper_storage_paths(
+                paper_info, "doi_resolution"
+            )
+            screenshots_dir = storage_paths["storage_path"] / "screenshots"
+            screenshots_dir.mkdir(parents=True, exist_ok=True)
+
+            # Generate timestamp and filename
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            safe_doi = doi.replace("/", "_").replace(":", "_")
+            filename = f"{timestamp}-{stage}-{safe_doi}.png"
+            screenshot_path = screenshots_dir / filename
+
+            if page:
+                # Use existing page
+                await page.screenshot(
+                    path=str(screenshot_path), full_page=True, timeout=10000
+                )
+                logger.info(
+                    f"DOI workflow screenshot: {stage} -> {screenshot_path}"
+                )
+            else:
+                # Create new browser for screenshot
+                async with async_playwright() as p:
+                    browser = await p.chromium.launch(headless=True)
+                    new_page = await browser.new_page()
+
+                    try:
+                        await new_page.goto(
+                            url, wait_until="networkidle", timeout=30000
+                        )
+                        await new_page.screenshot(
+                            path=str(screenshot_path),
+                            full_page=True,
+                            timeout=10000,
+                        )
+
+                        # Save page info
+                        info_file = screenshot_path.with_suffix(".txt")
+                        with open(info_file, "w", encoding="utf-8") as f:
+                            f.write(f"DOI: {doi}\n")
+                            f.write(f"URL: {url}\n")
+                            f.write(f"Stage: {stage}\n")
+                            f.write(f"Timestamp: {timestamp}\n")
+                            f.write(f"Page Title: {await new_page.title()}\n")
+                            f.write(f"Final URL: {new_page.url}\n")
+
+                        logger.info(
+                            f"DOI workflow screenshot: {stage} -> {screenshot_path}"
+                        )
+                    finally:
+                        await browser.close()
+
+        except Exception as e:
+            logger.debug(
+                f"Failed to capture DOI workflow screenshot for {stage}: {e}"
+            )
+
+    def _load_cache(self) -> Dict[str, Dict[str, any]]:
+        """Load URL cache from disk."""
+        if self.cache_file.exists():
+            try:
+                with open(self.cache_file, "r") as f:
+                    return json.load(f)
+            except Exception as e:
+                logger.warning(f"Failed to load cache: {e}")
+        return {}
+
+    def _save_cache(self):
+        """Save URL cache to disk."""
+        try:
+            with open(self.cache_file, "w") as f:
+                json.dump(self.cache, f, indent=2)
+        except Exception as e:
+            logger.error(f"Failed to save cache: {e}")
+
+    def _extract_doi_info(self, doi: str) -> Dict[str, str]:
+        """Extract publisher and article ID from DOI."""
+        # Common DOI patterns
+        patterns = {
+            "elsevier": r"10\.1016/(.+)",
+            "springer": r"10\.1007/(.+)",
+            "nature": r"10\.1038/(.+)",
+            "wiley": r"10\.1002/(.+)",
+            "ieee": r"10\.1109/(.+)",
+            "acs": r"10\.1021/(.+)",
+            "rsc": r"10\.1039/(.+)",
+            "plos": r"10\.1371/(.+)",
+            "frontiers": r"10\.3389/(.+)",
+            "mdpi": r"10\.3390/(.+)",
+            "oxford": r"10\.1093/(.+)",
+            "sage": r"10\.1177/(.+)",
+            "taylor_francis": r"10\.1080/(.+)",
+            "apa": r"10\.1037/(.+)",
+            "iop": r"10\.1088/(.+)",
+        }
+
+        for publisher, pattern in patterns.items():
+            match = re.match(pattern, doi)
+            if match:
+                return {
+                    "publisher": publisher,
+                    "article_id": match.group(1),
+                    "doi": doi,
+                }
+
+        # Generic pattern
+        match = re.match(r"(10\.\d+)/(.+)", doi)
+        if match:
+            return {
+                "publisher": "unknown",
+                "prefix": match.group(1),
+                "article_id": match.group(2),
+                "doi": doi,
+            }
+
+        return {"doi": doi, "publisher": "unknown"}
+
+    def _build_direct_urls(self, doi: str) -> List[str]:
+        """Build potential direct publisher URLs for a DOI."""
+        info = self._extract_doi_info(doi)
+        urls = []
+
+        # Always include standard DOI URL
+        urls.append(f"https://doi.org/{doi}")
+
+        # Publisher-specific patterns
+        if info["publisher"] == "elsevier":
+            # ScienceDirect pattern
+            urls.append(
+                f"https://www.sciencedirect.com/science/article/pii/{info['article_id']}"
+            )
+
+        elif info["publisher"] == "springer":
+            # SpringerLink pattern
+            urls.append(f"https://link.springer.com/article/{doi}")
+            urls.append(f"https://link.springer.com/chapter/{doi}")
+
+        elif info["publisher"] == "nature":
+            # Nature pattern
+            urls.append(
+                f"https://www.nature.com/articles/{info['article_id']}"
+            )
+
+        elif info["publisher"] == "wiley":
+            # Wiley Online Library pattern
+            urls.append(f"https://onlinelibrary.wiley.com/doi/abs/{doi}")
+            urls.append(f"https://onlinelibrary.wiley.com/doi/full/{doi}")
+
+        elif info["publisher"] == "ieee":
+            # IEEE Xplore pattern (needs document ID)
+            urls.append(
+                f"https://ieeexplore.ieee.org/document/{info['article_id']}"
+            )
+
+        elif info["publisher"] == "oxford":
+            # Oxford Academic pattern
+            urls.append(f"https://academic.oup.com/article-lookup/doi/{doi}")
+
+        return urls
+
+    async def resolve_single_async(
+        self, doi: str, use_openurl: bool = True, verify_access: bool = True
+    ) -> Optional[Dict[str, any]]:
+        """
+        Resolve a single DOI to accessible URL.
+
+        Args:
+            doi: DOI to resolve
+            use_openurl: Whether to use OpenURL resolver
+            verify_access: Whether to verify PDF access
+
+        Returns:
+            Dict with 'url', 'access_type', 'verified' fields if successful
+        """
+        # Check cache first
+        if doi in self.cache:
+            logger.debug(f"Using cached URL for {doi}")
+            return self.cache[doi]
+
+        result = None
+
+        try:
+            # Try OpenURL resolver first if configured
+            openurl_resolver_url = self.config.resolve(
+                "openurl_resolver_url", None, None, type=str
+            )
+            if use_openurl and openurl_resolver_url:
+                logger.info(f"Trying OpenURL resolver for {doi}")
+
+                # Capture screenshot of initial DOI URL
+                doi_url = f"https://doi.org/{doi}"
+                await self._capture_workflow_screenshot_async(
+                    doi, doi_url, "01_initial_doi"
+                )
+
+                openurl_result = await self._try_openurl_async(doi)
+                if openurl_result:
+                    result = openurl_result
+                    # Capture screenshot of OpenURL result
+                    if result.get("url"):
+                        await self._capture_workflow_screenshot_async(
+                            doi, result["url"], "02_openurl_resolved"
+                        )
+
+            # Try direct publisher URLs
+            if not result:
+                logger.info(f"Trying direct publisher URLs for {doi}")
+                direct_result = await self._try_direct_urls_async(
+                    doi, verify_access
+                )
+                if direct_result:
+                    result = direct_result
+                    # Capture screenshot of direct URL result
+                    if result.get("url"):
+                        await self._capture_workflow_screenshot_async(
+                            doi, result["url"], "03_direct_publisher"
+                        )
+
+            # Cache successful result
+            if result:
+                self.cache[doi] = result
+                self._save_cache()
+                logger.success(f"Resolved {doi} to {result['url']}")
+            else:
+                logger.warning(f"Failed to resolve {doi}")
+
+            return result
+
+        except Exception as e:
+            logger.error(f"Error resolving {doi}: {e}")
+            return None
+
+    async def _try_openurl_async(self, doi: str) -> Optional[Dict[str, any]]:
+        """Try to resolve DOI using OpenURL."""
+        try:
+            # Build OpenURL query
+            params = {
+                "rft_id": f"info:doi/{doi}",
+                "rft.genre": "article",
+                "rft_val_fmt": "info:ofi/fmt:kev:mtx:journal",
+                "req_dat": "format=pdf",
+            }
+
+            openurl_resolver_url = self.config.resolve(
+                "openurl_resolver_url", None, None, type=str
+            )
+            openurl = f"{openurl_resolver_url}?{urlencode(params)}"
+
+            # Use the OpenURL resolver to navigate
+            async with async_playwright() as p:
+                browser = await p.chromium.launch(headless=True)
+                page = await browser.new_page()
+
+                try:
+                    # Navigate to OpenURL
+                    await page.goto(
+                        openurl, wait_until="networkidle", timeout=30000
+                    )
+
+                    # Wait for potential redirects
+                    await page.wait_for_timeout(3000)
+
+                    # Get final URL
+                    final_url = page.url
+
+                    # Check if we reached a publisher page
+                    if "doi.org" not in final_url and any(
+                        domain in final_url
+                        for domain in [
+                            "sciencedirect",
+                            "springer",
+                            "nature",
+                            "wiley",
+                            "ieee",
+                        ]
+                    ):
+                        # Check for PDF access
+                        pdf_available = await self._check_pdf_access_async(
+                            page
+                        )
+
+                        return {
+                            "url": final_url,
+                            "access_type": "openurl",
+                            "pdf_available": pdf_available,
+                            "verified": True,
+                        }
+
+                finally:
+                    await browser.close()
+
+        except Exception as e:
+            logger.debug(f"OpenURL resolution failed for {doi}: {e}")
+
+        return None
+
+    async def _try_direct_urls_async(
+        self, doi: str, verify_access: bool = True
+    ) -> Optional[Dict[str, any]]:
+        """Try direct publisher URLs."""
+        urls = self._build_direct_urls(doi)
+
+        for url in urls:
+            try:
+                if verify_access:
+                    # Verify with browser
+                    result = await self._verify_url_access_async(url)
+                    if result:
+                        return {
+                            "url": url,
+                            "access_type": "direct",
+                            "pdf_available": result.get(
+                                "pdf_available", False
+                            ),
+                            "verified": True,
+                        }
+                else:
+                    # Just check if URL responds
+                    async with aiohttp.ClientSession() as session:
+                        async with session.head(
+                            url, allow_redirects=True
+                        ) as resp:
+                            if resp.status == 200:
+                                return {
+                                    "url": url,
+                                    "access_type": "direct",
+                                    "verified": False,
+                                }
+
+            except Exception as e:
+                logger.debug(f"Failed to access {url}: {e}")
+                continue
+
+        return None
+
+    async def _verify_url_access_async(
+        self, url: str
+    ) -> Optional[Dict[str, any]]:
+        """Verify URL provides article access."""
+        try:
+            async with async_playwright() as p:
+                browser = await p.chromium.launch(headless=True)
+                page = await browser.new_page()
+
+                try:
+                    # Navigate to URL
+                    await page.goto(
+                        url, wait_until="networkidle", timeout=30000
+                    )
+
+                    # Check for common paywall indicators
+                    paywall_indicators = [
+                        "purchase",
+                        "buy",
+                        "subscribe",
+                        "access denied",
+                        "log in",
+                        "sign in",
+                        "institutional login",
+                    ]
+
+                    page_text = await page.content()
+                    page_text_lower = page_text.lower()
+
+                    has_paywall = any(
+                        indicator in page_text_lower
+                        for indicator in paywall_indicators
+                    )
+
+                    # Check for PDF access
+                    pdf_available = await self._check_pdf_access_async(page)
+
+                    if not has_paywall or pdf_available:
+                        return {
+                            "pdf_available": pdf_available,
+                            "has_paywall": has_paywall,
+                        }
+
+                finally:
+                    await browser.close()
+
+        except Exception as e:
+            logger.debug(f"Failed to verify {url}: {e}")
+
+        return None
+
+    async def _check_pdf_access_async(self, page: Page) -> bool:
+        """Check if PDF download is available on the page."""
+        try:
+            # Look for PDF download links/buttons
+            pdf_selectors = [
+                'a[href*=".pdf"]',
+                'a[href*="/pdf/"]',
+                'button:has-text("Download PDF")',
+                'a:has-text("Download PDF")',
+                'a:has-text("View PDF")',
+                'a:has-text("Full Text PDF")',
+                ".pdf-download",
+                '[class*="pdf-link"]',
+            ]
+
+            for selector in pdf_selectors:
+                elements = await page.query_selector_all(selector)
+                if elements:
+                    return True
+
+            # Check for embedded PDF viewer
+            pdf_viewers = await page.query_selector_all(
+                'iframe[src*="pdf"], embed[type="application/pdf"]'
+            )
+            if pdf_viewers:
+                return True
+
+        except Exception as e:
+            logger.debug(f"Error checking PDF access: {e}")
+
+        return False
+
+    async def resolve_batch_async(
+        self, dois: List[str], max_concurrent: int = 3, progress_callback=None
+    ) -> Dict[str, Dict[str, any]]:
+        """
+        Resolve multiple DOIs concurrently.
+
+        Args:
+            dois: List of DOIs to resolve
+            max_concurrent: Maximum concurrent resolutions
+            progress_callback: Optional callback for progress updates
+
+        Returns:
+            Dict mapping DOIs to resolution results
+        """
+        results = {}
+        semaphore = asyncio.Semaphore(max_concurrent)
+
+        async def resolve_with_limit_async(doi: str, index: int):
+            async with semaphore:
+                if progress_callback:
+                    progress_callback(index, len(dois), f"Resolving {doi}")
+
+                result = await self.resolve_single_async(doi)
+                results[doi] = result
+                return doi, result
+
+        # Create tasks
+        tasks = [
+            resolve_with_limit_async(doi, i) for i, doi in enumerate(dois)
+        ]
+
+        # Process all DOIs
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+        return results
+
+    def resolve_from_bibtex(
+        self, bibtex_path: Path, output_path: Optional[Path] = None
+    ) -> Dict[str, Dict[str, any]]:
+        """
+        Resolve URLs for all DOIs in a BibTeX file.
+
+        Args:
+            bibtex_path: Path to BibTeX file
+            output_path: Optional path for updated BibTeX
+
+        Returns:
+            Dict mapping DOIs to resolution results
+        """
+        import bibtexparser
+
+        # Load BibTeX
+        with open(bibtex_path, "r", encoding="utf-8") as f:
+            bib_db = bibtexparser.load(f)
+
+        # Extract DOIs
+        dois = []
+        doi_to_entry = {}
+
+        for entry in bib_db.entries:
+            if "doi" in entry:
+                doi = entry["doi"]
+                dois.append(doi)
+                doi_to_entry[doi] = entry
+
+        logger.info(f"Found {len(dois)} DOIs in {bibtex_path}")
+
+        # Resolve URLs
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        try:
+            results = loop.run_until_complete(self.resolve_batch_async(dois))
+        finally:
+            loop.close()
+
+        # Update BibTeX entries with URLs
+        success_count = 0
+        for doi, result in results.items():
+            if result and result.get("url"):
+                entry = doi_to_entry[doi]
+                entry["url"] = result["url"]
+                entry["url_source"] = result["access_type"]
+                if result.get("pdf_available"):
+                    entry["pdf_available"] = "yes"
+                success_count += 1
+
+        logger.info(f"Resolved URLs for {success_count}/{len(dois)} DOIs")
+
+        # Save updated BibTeX if requested
+        if output_path:
+            with open(output_path, "w", encoding="utf-8") as f:
+                bibtexparser.dump(bib_db, f)
+            logger.info(f"Saved updated BibTeX to {output_path}")
+
+        return results
+
+
+async def main():
+    """Command-line interface for DOI to URL resolution."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Resolve DOIs to accessible publisher URLs",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Resolve single DOI
+  python -m scitex.scholar.open_url.resolve_urls --doi "10.1038/nature12373"
+
+  # Resolve DOIs from BibTeX file
+  python -m scitex.scholar.open_url.resolve_urls --bibtex papers.bib
+
+  # Save URLs to new BibTeX file
+  python -m scitex.scholar.open_url.resolve_urls --bibtex papers.bib --output papers-with-urls.bib
+        """,
+    )
+
+    input_group = parser.add_mutually_exclusive_group(required=True)
+
+    input_group.add_argument("--doi", type=str, help="Single DOI to resolve")
+
+    input_group.add_argument(
+        "--bibtex", "-b", type=str, help="BibTeX file containing DOIs"
+    )
+
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=str,
+        help="Output BibTeX file (for --bibtex mode)",
+    )
+
+    parser.add_argument(
+        "--no-verify", action="store_true", help="Skip access verification"
+    )
+
+    args = parser.parse_args()
+
+    # Initialize resolver
+    resolver = DOIToURLResolver()
+
+    if args.doi:
+        # Single DOI mode
+        result = await resolver.resolve_single_async(
+            args.doi, verify_access=not args.no_verify
+        )
+
+        if result:
+            print(f"\nResolved URL: {result['url']}")
+            print(f"Access type: {result['access_type']}")
+            if "pdf_available" in result:
+                print(
+                    f"PDF available: {'Yes' if result['pdf_available'] else 'No'}"
+                )
+        else:
+            print("\nFailed to resolve DOI")
+
+    else:
+        # BibTeX mode
+        results = resolver.resolve_from_bibtex(
+            Path(args.bibtex), Path(args.output) if args.output else None
+        )
+
+        # Print summary
+        success = sum(1 for r in results.values() if r and r.get("url"))
+        print(f"\nResolved {success}/{len(results)} DOIs")
+
+        # Show first few results
+        for doi, result in list(results.items())[:5]:
+            if result:
+                print(f"\n{doi}:")
+                print(f"  URL: {result['url']}")
+                print(f"  Type: {result['access_type']}")
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(asyncio.run(main()))
+
+# EOF