scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/scholar/url_finder/strategies/find_pdf_urls_by_navigation.py

@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-10-11 01:19:48 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/url/strategies/find_pdf_urls_by_navigation.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = (
+    "./src/scitex/scholar/url/strategies/find_pdf_urls_by_navigation.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+"""
+Find PDF URLs by navigating to PDF links and following redirects.
+
+Handles publishers like ScienceDirect that require navigation
+through redirect chains to reach the actual PDF URL.
+"""
+
+from typing import List, Optional
+from urllib.parse import urljoin
+
+from playwright.async_api import Page
+
+from scitex.browser.debugging import browser_logger
+from scitex.scholar import ScholarConfig
+from scitex.scholar.browser.utils import wait_redirects
+
+
+async def find_pdf_urls_by_navigation(
+    page: Page,
+    url: str = None,
+    config: ScholarConfig = None,
+    func_name: str = "find_pdf_urls_by_navigation",
+) -> List[str]:
+    """
+    Find PDF URLs by navigating to PDF links and capturing final URLs.
+
+    This handles cases like ScienceDirect where:
+    1. Direct PDF links exist (/pdfft? endpoints)
+    2. Navigating to them triggers redirects
+    3. Final destination is the actual PDF on pdf.sciencedirectassets.com
+
+    Args:
+        page: Playwright page object
+        url: Current page URL (unused, for signature consistency)
+        config: ScholarConfig instance
+        func_name: Function name for logging
+
+    Returns:
+        List of PDF URLs found
+    """
+    config = config or ScholarConfig()
+    pdf_urls = []
+
+    try:
+        # Check if we already have direct PDF links
+        current_url = page.url.lower()
+        is_sciencedirect = any(
+            domain in current_url
+            for domain in [
+                "sciencedirect.com",
+                "cell.com",
+                "elsevier.com",
+                "ssrn.com",
+            ]
+        )
+
+        if not is_sciencedirect:
+            return []
+
+        # Look for existing PDF links on the page
+        pdf_link_selectors = [
+            'a[href*="/pdfft?"]',  # ScienceDirect PDF endpoint
+            'a[aria-label*="Download PDF"]',
+            'a[aria-label*="Download This Paper"]',
+            'a:has-text("View PDF")',
+            "a.pdf-link",
+            'a[href*="/pdf/"]',
+        ]
+
+        pdf_href = None
+        for selector in pdf_link_selectors:
+            try:
+                element = await page.query_selector(selector)
+                if element:
+                    href = await element.get_attribute("href")
+                    if href and ("/pdfft?" in href or "/pdf/" in href):
+                        # Make absolute URL if needed
+                        if href.startswith("/"):
+                            href = urljoin(page.url, href)
+                        pdf_href = href
+                        await browser_logger.debug(
+                            page,
+                            f"{func_name}: Found PDF link: {href[:80]}...",
+                        )
+                        break
+            except Exception as e:
+                await browser_logger.debug(
+                    page,
+                    f"{func_name}: Error checking selector {selector}: {e}",
+                )
+                continue
+
+        if not pdf_href:
+            await browser_logger.debug(
+                page, f"{func_name}: No PDF links found on page"
+            )
+            return []
+
+        # Navigate to PDF URL in a new page to capture final URL
+        context = page.context
+        new_page = None
+
+        try:
+            await browser_logger.debug(
+                page,
+                f"{func_name}: Navigating to PDF URL to capture final destination...",
+            )
+            new_page = await context.new_page()
+
+            # Navigate and wait for redirects - be patient!
+            await new_page.goto(pdf_href, wait_until="commit", timeout=60000)
+
+            # Wait for redirects to complete with longer timeout
+            redirect_result = await wait_redirects(
+                new_page,
+                timeout=60000,  # 60 seconds timeout
+                show_progress=False,
+                track_chain=True,
+                auth_aware=True,
+                wait_for_idle=True,  # Wait for network idle
+            )
+
+            final_url = redirect_result.get("final_url", new_page.url)
+            redirect_chain = redirect_result.get("redirect_chain", [])
+
+            # Additional wait to ensure PDF loads
+            await new_page.wait_for_timeout(2000)
+
+            # Check again for final URL
+            final_url = new_page.url
+
+            await browser_logger.debug(
+                page,
+                f"{func_name}: Redirect complete after {len(redirect_chain)} steps",
+            )
+            await browser_logger.debug(
+                page, f"{func_name}: Final URL: {final_url[:80]}..."
+            )
+
+            # Check if final URL is a PDF
+            if any(
+                indicator in final_url
+                for indicator in [
+                    "pdf.sciencedirectassets.com",
+                    ".pdf",
+                    "application/pdf",
+                    "/pdf/",
+                    "pdfft?",
+                ]
+            ):
+                pdf_urls.append(final_url)
+                await browser_logger.debug(
+                    page, f"{func_name}: Captured final PDF URL via navigation"
+                )
+            else:
+                await browser_logger.debug(
+                    page,
+                    f"{func_name}: Final URL doesn't appear to be a PDF: {final_url}",
+                )
+
+        except Exception as e:
+            await browser_logger.error(
+                page, f"{func_name}: Error navigating to PDF: {e}"
+            )
+        finally:
+            if new_page:
+                try:
+                    await new_page.close()
+                except:
+                    pass
+
+        return pdf_urls
+
+    except Exception as e:
+        await browser_logger.error(
+            page, f"{func_name}: Error finding PDFs via navigation: {e}"
+        )
+        return []
+
+
+async def find_pdf_url_from_sciencedirect_api(
+    page: Page, func_name="find_pdf_url_from_sciencedirect_api"
+) -> Optional[str]:
+    """
+    Extract PDF URL from ScienceDirect page using their JavaScript context.
+
+    ScienceDirect pages often have the PDF URL in JavaScript variables.
+    """
+    try:
+        # Try to extract from page's JavaScript context
+        pdf_info = await page.evaluate(
+            """
+            () => {
+                // Look for PDF URL in various places
+
+                // Check for download URL in page data
+                const pdfLinks = document.querySelectorAll('a[href*="/pdfft?"]');
+                if (pdfLinks.length > 0) {
+                    return pdfLinks[0].href;
+                }
+
+                // Check window.SDM object (ScienceDirect)
+                if (window.SDM && window.SDM.pdfUrl) {
+                    return window.SDM.pdfUrl;
+                }
+
+                // Check for entitlement info
+                if (window.SD && window.SD.article && window.SD.article.pdfDownloadUrl) {
+                    return window.SD.article.pdfDownloadUrl;
+                }
+
+                // Look for View PDF button
+                const viewPdfBtn = document.querySelector('a[aria-label*="View PDF"]');
+                if (viewPdfBtn && viewPdfBtn.href) {
+                    return viewPdfBtn.href;
+                }
+
+                // Check meta tags
+                const pdfMeta = document.querySelector('meta[name="citation_pdf_url"]');
+                if (pdfMeta && pdfMeta.content) {
+                    return pdfMeta.content;
+                }
+
+                return null;
+            }
+        """
+        )
+
+        if pdf_info:
+            await browser_logger.debug(
+                page,
+                f"{func_name}: Found PDF info from page context: {pdf_info[:80]}...",
+            )
+            return pdf_info
+
+    except Exception as e:
+        await browser_logger.debug(
+            page, f"Could not extract PDF info from page context: {e}"
+        )
+
+    return None
+
+# EOF

scitex/scholar/url_finder/strategies/find_pdf_urls_by_publisher_patterns.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-10-10 01:33:13 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/url/strategies/publisher_patterns.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = (
+    "./src/scitex/scholar/url/strategies/publisher_patterns.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+__FILE__ = __file__
+
+import re
+from typing import List
+
+from scitex.browser.debugging import browser_logger
+
+
+async def find_pdf_urls_by_publisher_patterns(
+    page,
+    url: str = None,
+    config=None,
+    func_name: str = "find_pdf_urls_by_publisher_patterns"
+) -> List[str]:
+    """
+    Generate PDF URLs based on publisher-specific URL patterns.
+
+    Args:
+        page: Playwright page object (unused, for signature consistency)
+        url: Page URL to analyze (defaults to page.url if not provided)
+        config: ScholarConfig instance (unused, for signature consistency)
+        func_name: Function name for logging
+
+    Returns:
+        List of PDF URLs generated from patterns
+    """
+    url = url or page.url
+    urls_pdf = []
+
+    # Nature
+    if "nature.com" in url and not url.endswith(".pdf"):
+        urls_pdf.append(url.rstrip("/") + ".pdf")
+
+    # Science
+    elif "science.org" in url and "/doi/10." in url and "/pdf/" not in url:
+        urls_pdf.append(url.replace("/doi/", "/doi/pdf/"))
+
+    # Elsevier/ScienceDirect
+    elif "sciencedirect.com" in url and "/pii/" in url:
+        pii = url.split("/pii/")[-1].split("/")[0].split("?")[0]
+        urls_pdf.append(
+            f"https://www.sciencedirect.com/science/article/pii/{pii}/pdfft"
+        )
+
+    # Wiley
+    elif "wiley.com" in url and "/doi/" in url and "/pdfdirect" not in url:
+        urls_pdf.append(url.replace("/doi/", "/doi/pdfdirect/"))
+
+    # Frontiers
+    elif "frontiersin.org" in url and "/full" in url:
+        urls_pdf.append(url.replace("/full", "/pdf"))
+
+    # Springer
+    elif (
+        "springer.com" in url or "link.springer.com" in url
+    ) and "/article/" in url:
+        if not url.endswith(".pdf"):
+            urls_pdf.append(url.rstrip("/") + ".pdf")
+
+    # IEEE
+    elif "ieee.org" in url and "/document/" in url:
+        doc_id = url.split("/document/")[-1].split("/")[0]
+        urls_pdf.append(
+            f"https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber={doc_id}"
+        )
+
+    # IOP Publishing
+    elif "iopscience.iop.org" in url and "/article/" in url:
+        # Pattern: /article/10.1088/1741-2552/aaf92e
+        # PDF URL: /article/10.1088/1741-2552/aaf92e/pdf
+        if not url.endswith("/pdf"):
+            urls_pdf.append(url.rstrip("/") + "/pdf")
+        # Also try: /article/10.1088/1741-2552/aaf92e/pdf/metrics
+        # Some IOP articles have different PDF locations
+        doi_match = re.search(r"/article/(10\.\d+/[\w\-\.]+)", url)
+        if doi_match:
+            doi = doi_match.group(1)
+            # Alternative PDF locations for IOP
+            urls_pdf.append(f"https://iopscience.iop.org/article/{doi}/pdf")
+            urls_pdf.append(
+                f"https://iopscience.iop.org/article/{doi}/pdf/metrics"
+            )
+
+    # MDPI
+    elif "mdpi.com" in url and "/htm" in url:
+        urls_pdf.append(url.replace("/htm", "/pdf"))
+
+    # BMC
+    elif "biomedcentral.com" in url and "/articles/" in url:
+        urls_pdf.append(url.replace("/articles/", "/track/pdf/"))
+
+    # PLOS
+    elif "plos.org" in url and "/article" in url:
+        if "?id=" in url:
+            article_id = url.split("?id=")[-1].split("&")[0]
+            base_url = url.split("/article")[0]
+            urls_pdf.append(
+                f"{base_url}/article/file?id={article_id}&type=printable"
+            )
+        elif "/article/" in url:
+            urls_pdf.append(
+                url.replace("/article/", "/article/file?id=").split("?")[0]
+                + "&type=printable"
+            )
+
+    # Journal of Neuroscience
+    if "jneurosci.org" in url and "/content/" in url:
+        # Extract volume/issue/page numbers
+        match = re.search(r"/content/(\d+)/(\d+)/(\d+)", url)
+        if match:
+            vol, issue, page = match.groups()
+            urls_pdf.append(
+                f"https://www.jneurosci.org/content/jneuro/{vol}/{issue}/{page}.full.pdf"
+            )
+
+    # eNeuro
+    elif "eneuro.org" in url and "/content/" in url:
+        # Pattern: /content/3/6/ENEURO.0334-16.2016
+        match = re.search(r"/content/[^/]+/[^/]+/(ENEURO\.[^/]+)", url)
+        if match:
+            eneuro_id = match.group(1)
+            urls_pdf.append(
+                f"https://www.eneuro.org/content/eneuro/early/recent/{eneuro_id}.full.pdf"
+            )
+
+    # Oxford Academic
+    elif "academic.oup.com" in url:
+        urls_pdf.append(url.replace("/article/", "/article-pdf/"))
+
+    # Improve preprint handling
+    elif "biorxiv.org" in url or "medrxiv.org" in url:
+        # Handle versioned URLs better
+        if "/v" in url:  # e.g., /v1, /v2
+            base_url = url.split("/v")[0]
+            urls_pdf.append(f"{base_url}.full.pdf")
+        else:
+            urls_pdf.append(url + ".full.pdf")
+
+    elif "arxiv.org" in url:
+        if "/abs/" in url:
+            arxiv_id = url.split("/abs/")[-1]
+            urls_pdf.append(f"https://arxiv.org/pdf/{arxiv_id}.pdf")
+
+    if urls_pdf:
+        await browser_logger.debug(
+            page,
+            f"{func_name}: Pattern matching found {len(urls_pdf)} URLs"
+        )
+
+    return urls_pdf
+
+# EOF

scitex/scholar/url_finder/strategies/find_pdf_urls_by_zotero_translators.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-10-13 06:32:08 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/url_finder/strategies/find_pdf_urls_by_zotero_translators.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = (
+    "./src/scitex/scholar/url_finder/strategies/find_pdf_urls_by_zotero_translators.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+__FILE__ = __file__
+
+"""
+Find PDF URLs using Python Zotero translators.
+
+This module uses the zotero-translators-python package instead of running
+JavaScript translators. It provides better performance, reliability, and
+maintainability compared to the JavaScript-based approach.
+
+Features:
+- 100+ Python translators available
+- No JavaScript execution overhead
+- Better error handling
+- Easier debugging
+- Type safety
+
+Usage:
+    from scitex.scholar.url_finder.strategies import find_pdf_urls_by_zotero_translators
+
+    pdf_urls = await find_pdf_urls_by_zotero_translators(page, url)
+"""
+
+from typing import List
+
+from playwright.async_api import Page
+from zotero_translators_python.core.registry import TranslatorRegistry
+
+from scitex import logging
+from scitex.browser import browser_logger
+
+logger = logging.getLogger(__name__)
+
+
+async def find_pdf_urls_by_zotero_translators(
+    page: Page,
+    url: str = None,
+    config=None,
+    func_name: str = "find_pdf_urls_by_zotero_translators",
+) -> List[str]:
+    """
+    Find PDF URLs using Python-based Zotero translators.
+
+    This is the preferred method over JavaScript translators due to:
+    - Better performance (no JS eval overhead)
+    - Better reliability (proper error handling)
+    - Better maintainability (Python codebase)
+    - Better debugging (Python stack traces)
+
+    Args:
+        page: Playwright page object with loaded content
+        url: Current page URL (defaults to page.url if not provided)
+        config: ScholarConfig instance (unused, for signature consistency)
+        func_name: Function name for logging
+
+    Returns:
+        List of PDF URLs extracted by matching translators
+        Empty list if no translator matches or extraction fails
+
+    Examples:
+        >>> async with async_playwright() as p:
+        ...     browser = await p.chromium.launch()
+        ...     page = await browser.new_page()
+        ...     await page.goto("https://www.nature.com/articles/nature12345")
+        ...     pdf_urls = await find_pdf_urls_by_zotero_translators(page)
+        ...     print(f"Found {len(pdf_urls)} PDF URLs")
+    """
+    url = url or page.url
+
+    try:
+        # Get registry of all available translators
+        registry = TranslatorRegistry()
+
+        # Find matching translator for this URL
+        matching_translator = registry.get_translator_for_url(url)
+
+        if not matching_translator:
+            await browser_logger.info(
+                page, f"{func_name}: No Python translator matches URL: {url}"
+            )
+            return []
+
+        # Try the matching translator
+        all_pdf_urls = []
+        try:
+            # logger.debug(
+            #     f"{func_name}: Trying {matching_translator.LABEL} translator..."
+            # )
+            # logger.debug(f"{func_name}: Page URL: {page.url}")
+            # logger.debug(f"{func_name}: Translator target URL: {url}")
+
+            # Extract PDF URLs using the translator
+            pdf_urls = await matching_translator.extract_pdf_urls_async(page)
+
+            # logger.debug(f"{func_name}: Translator returned: {pdf_urls}")
+
+            if pdf_urls:
+                await browser_logger.debug(
+                    page,
+                    f"{func_name}: {matching_translator.LABEL} found {len(pdf_urls)} PDF URL(s)",
+                )
+                for i_pdf, pdf_url in enumerate(pdf_urls, 1):
+                    await browser_logger.debug(
+                        page, f"{func_name}  {i_pdf}. {pdf_url}"
+                    )
+
+                all_pdf_urls.extend(pdf_urls)
+            else:
+                await browser_logger.warning(
+                    page,
+                    f"{func_name}: {matching_translator.LABEL} returned empty list - check if page loaded correctly",
+                )
+
+        except Exception as e:
+            import traceback
+
+            await browser_logger.error(
+                page,
+                f"{func_name}: {matching_translator.LABEL} extraction failed: {e}\nTraceback: {traceback.format_exc()}",
+            )
+
+            # logger.debug(f"{func_name}: Traceback: {traceback.format_exc()}")
+
+        # Remove duplicates while preserving order
+        seen = set()
+        unique_urls = []
+        for url_pdf in all_pdf_urls:
+            if url_pdf not in seen:
+                seen.add(url_pdf)
+                unique_urls.append(url_pdf)
+
+        if unique_urls:
+            await browser_logger.debug(
+                page,
+                f"{func_name}: ✓ Python Zotero found {len(unique_urls)} URLs",
+            )
+        else:
+            await browser_logger.debug(
+                page, f"{func_name}: No PDF URLs found by Python translators"
+            )
+
+        return unique_urls
+
+    except Exception as e:
+        await browser_logger.warning(
+            page,
+            f"{func_name}: Zotero strategy failed: {e}",
+        )
+        return []
+
+# EOF

scitex/scholar/url_finder/strategies/find_supplementary_urls_by_href.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-10-11 01:19:50 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/url/strategies/find_supplementary_urls_by_href.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = (
+    "./src/scitex/scholar/url/strategies/find_supplementary_urls_by_href.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+__FILE__ = __file__
+
+from typing import Dict, List
+
+from playwright.async_api import Page
+
+from scitex import logging
+from scitex.browser import browser_logger
+from scitex.scholar import ScholarConfig
+
+logger = logging.getLogger(__name__)
+
+
+async def find_supplementary_urls_by_href(
+    page: Page, config: ScholarConfig = None
+) -> List[Dict]:
+    """Find supplementary material URLs in a web page."""
+    await browser_logger.debug(page, "Finding Supplementary URLs...")
+
+    config = config or ScholarConfig()
+    supplementary_selectors = config.resolve(
+        "supplementary_selectors",
+        default=[
+            'a[href*="supplementary"]',
+            'a[href*="supplement"]',
+            'a[href*="additional"]',
+        ],
+    )
+
+    try:
+        supplementary = await page.evaluate(
+            f"""() => {{
+            const results = [];
+            const selectors = {supplementary_selectors};
+            const seen_urls = new Set();
+
+            selectors.forEach(selector => {{
+                document.querySelectorAll(selector).forEach(link => {{
+                    if (link.href && !seen_urls.has(link.href)) {{
+                        seen_urls.add(link.href);
+                        results.push({{
+                            url: link.href,
+                            description: link.textContent.trim(),
+                            source: 'href_pattern'
+                        }});
+                    }}
+                }});
+            }});
+            return results;
+        }}"""
+        )
+        return supplementary
+    except Exception as e:
+        logger.error(f"Error finding supplementary URLs: {e}")
+        return []
+
+# EOF

scitex/scholar/utils/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Scholar utilities - Organized by function.
+
+Structure:
+- text/: Text processing utilities (TextNormalizer)
+- bibtex/: BibTeX parsing utilities
+- cleanup/: Maintenance and cleanup scripts
+
+For backward compatibility, TextNormalizer is re-exported at top level.
+"""
+
+from .text import TextNormalizer
+from .bibtex import parse_bibtex
+
+__all__ = [
+    "TextNormalizer",  # Most commonly used
+    "parse_bibtex",
+]
+
+# EOF

scitex/scholar/utils/bibtex/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""BibTeX processing utilities for Scholar module."""
+
+from ._parse_bibtex import parse_bibtex
+
+__all__ = ["parse_bibtex"]
+
+# EOF