scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/.legacy/metadata/urls/__init__.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-08 01:22:00 (assistant)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/urls/__init__.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-"""
-URL Module - Unified URL handling for Scholar
-
-Provides a single abstracted interface for all URL operations.
-Internal functions (_finder, _resolver) are available but not exposed.
-"""
-
-# Import the main handler class
-from ._handler import URLHandler
-
-# Keep the existing URLMetadataHandler for backward compatibility
-from ._URLMetadataHandler import (
-    URLMetadataHandler,
-    add_urls_to_paper,
-    get_papers_for_download,
-)
-
-__all__ = [
-    # Main abstracted interface
-    "URLHandler",
-    # Backward compatibility
-    "URLMetadataHandler",
-    "add_urls_to_paper",
-    "get_papers_for_download",
-]
-
-# EOF

scitex/scholar/.legacy/metadata/urls/_finder.py

@@ -1,319 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-09 01:01:43 (ywatanabe)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/urls/_finder.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-"""
-URL Finder Functions
-
-Simple functions to find/extract URLs from web pages and metadata.
-No classes, just functions that do one thing well.
-"""
-
-from typing import Dict, List
-from urllib.parse import urljoin
-
-from playwright.async_api import Page
-
-from scitex import logging
-
-logger = logging.getLogger(__name__)
-
-
-async def find_pdf_urls(page: Page, base_url: str = None) -> List[Dict]:
-    """
-    Find PDF URLs in a web page using multiple strategies.
-
-    Args:
-        page: Playwright page object
-        base_url: Base URL for relative links
-
-    Returns:
-        List of dicts with url, source, and reliability info
-    """
-    if base_url is None:
-        base_url = page.url
-
-    pdf_urls = []
-    seen_urls = set()
-
-    # Strategy 1: Try Zotero translator FIRST (most reliable)
-    translator_urls = await _find_with_zotero_translator(page, base_url)
-    for url in translator_urls:
-        if url not in seen_urls:
-            seen_urls.add(url)
-            pdf_urls.append(
-                {
-                    "url": url,
-                    "source": "zotero_translator",
-                    "reliability": "high",
-                }
-            )
-
-    # Strategy 2: Find direct PDF links (fallback if no translator)
-    direct_links = await _find_direct_pdf_links(page)
-    for url in direct_links:
-        if url not in seen_urls:
-            seen_urls.add(url)
-            pdf_urls.append(
-                {"url": url, "source": "direct_link", "reliability": "medium"}
-            )
-
-    # Strategy 3: Check for publisher patterns (additional URLs)
-    pattern_urls = _get_publisher_pdf_patterns(base_url)
-    for url in pattern_urls:
-        if url not in seen_urls:
-            seen_urls.add(url)
-            pdf_urls.append(
-                {
-                    "url": url,
-                    "source": "publisher_pattern",
-                    "reliability": "low",
-                }
-            )
-
-    logger.success(f"Found {len(pdf_urls)} unique PDF URLs")
-
-    # Log breakdown by source
-    source_counts = {}
-    for item in pdf_urls:
-        source = item["source"]
-        source_counts[source] = source_counts.get(source, 0) + 1
-
-    for source, count in source_counts.items():
-        logger.info(f"  - {source}: {count} URLs")
-
-    return pdf_urls
-
-
-async def find_supplementary_urls(page: Page) -> List[Dict]:
-    """
-    Find supplementary material URLs in a web page.
-
-    Args:
-        page: Playwright page object
-
-    Returns:
-        List of dicts with url, description, type, source, and reliability
-    """
-    try:
-        supplementary = await page.evaluate(
-            """
-            () => {
-                const results = [];
-
-                // Common supplementary selectors
-                const selectors = [
-                    'a[href*="supplementary"]',
-                    'a[href*="supplement"]',
-                    'a[href*="additional"]',
-                    'a[href*="supporting"]',
-                    'a[href*="SI"]',
-                    'a[href*="ESM"]'
-                ];
-
-                const seen_urls = new Set();
-
-                selectors.forEach(selector => {
-                    document.querySelectorAll(selector).forEach(link => {
-                        if (link.href && !seen_urls.has(link.href)) {
-                            seen_urls.add(link.href);
-
-                            // Determine file type from URL
-                            let type = 'unknown';
-                            const url_lower = link.href.toLowerCase();
-                            if (url_lower.includes('.pdf')) type = 'pdf';
-                            else if (url_lower.includes('.xlsx') || url_lower.includes('.xls')) type = 'excel';
-                            else if (url_lower.includes('.docx') || url_lower.includes('.doc')) type = 'word';
-                            else if (url_lower.includes('.zip')) type = 'archive';
-                            else if (url_lower.includes('.mp4') || url_lower.includes('.avi')) type = 'video';
-
-                            results.push({
-                                url: link.href,
-                                description: link.textContent.trim(),
-                                type: type,
-                                source: 'href_pattern',
-                                reliability: 'low'
-                            });
-                        }
-                    });
-                });
-
-                return results;
-            }
-        """
-        )
-
-        logger.success(
-            f"Found {len(supplementary)} supplementary URLs by href pattern matching"
-        )
-        return supplementary
-
-    except Exception as e:
-        logger.error(f"Error finding supplementary URLs: {e}")
-        return []
-
-
-async def _find_direct_pdf_links(page: Page) -> List[str]:
-    """Find direct PDF links in the page."""
-    try:
-        pdf_urls = await page.evaluate(
-            """
-            () => {
-                const urls = new Set();
-
-                // Find all links ending with .pdf
-                document.querySelectorAll('a[href$=".pdf"]').forEach(link => {
-                    urls.add(link.href);
-                });
-
-                // Find download buttons/links
-                const downloadSelectors = [
-                    'a[data-track-action*="download"]',
-                    'button[data-track-action*="download"]',
-                    'a:has-text("Download PDF")',
-                    'button:has-text("Download PDF")',
-                    'a[download][href*="pdf"]',
-                    '.pdf-download-btn',
-                    'a[href*="/pdf/"]',
-                    'a[href*="/doi/pdf/"]'
-                ];
-
-                downloadSelectors.forEach(selector => {
-                    try {
-                        document.querySelectorAll(selector).forEach(elem => {
-                            const href = elem.getAttribute('href') || elem.dataset.href;
-                            if (href) {
-                                const fullUrl = new URL(href, window.location.href).href;
-                                if (fullUrl.includes('pdf')) {
-                                    urls.add(fullUrl);
-                                }
-                            }
-                        });
-                    } catch {}
-                });
-
-                return Array.from(urls);
-            }
-        """
-        )
-
-        return pdf_urls
-    except:
-        return []
-
-
-def _get_publisher_pdf_patterns(url: str) -> List[str]:
-    """Generate PDF URLs based on publisher patterns."""
-    pdf_urls = []
-
-    # Nature
-    if "nature.com" in url and not url.endswith(".pdf"):
-        pdf_urls.append(url.rstrip("/") + ".pdf")
-
-    # Science
-    elif "science.org" in url and "/doi/10." in url and "/pdf/" not in url:
-        pdf_urls.append(url.replace("/doi/", "/doi/pdf/"))
-
-    # Elsevier/ScienceDirect
-    elif "sciencedirect.com" in url and "/pii/" in url:
-        pii = url.split("/pii/")[-1].split("/")[0].split("?")[0]
-        pdf_urls.append(
-            f"https://www.sciencedirect.com/science/article/pii/{pii}/pdfft"
-        )
-
-    # Wiley
-    elif "wiley.com" in url and "/doi/" in url and "/pdfdirect" not in url:
-        pdf_urls.append(url.replace("/doi/", "/doi/pdfdirect/"))
-
-    # Frontiers
-    elif "frontiersin.org" in url and "/full" in url:
-        pdf_urls.append(url.replace("/full", "/pdf"))
-
-    # Springer
-    elif ("springer.com" in url or "link.springer.com" in url) and "/article/" in url:
-        if not url.endswith(".pdf"):
-            pdf_urls.append(url.rstrip("/") + ".pdf")
-
-    # IEEE
-    elif "ieee.org" in url and "/document/" in url:
-        doc_id = url.split("/document/")[-1].split("/")[0]
-        pdf_urls.append(
-            f"https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber={doc_id}"
-        )
-
-    # MDPI
-    elif "mdpi.com" in url and "/htm" in url:
-        pdf_urls.append(url.replace("/htm", "/pdf"))
-
-    # BMC
-    elif "biomedcentral.com" in url and "/articles/" in url:
-        pdf_urls.append(url.replace("/articles/", "/track/pdf/"))
-
-    if len(pdf_urls) > 0:
-        logger.success(
-            f"Publisher-specific pattern matching found {len(pdf_urls)} PDF URLs"
-        )
-    else:
-        logger.warning(f"Publisher-specific patterns did not match any PDF URLs")
-
-    return pdf_urls
-
-
-async def _find_with_zotero_translator(page: Page, url: str) -> List[str]:
-    """
-    Find PDF URLs using Zotero translator (FIRST strategy - most reliable).
-
-    Args:
-        page: Playwright page object with loaded content
-        url: Current page URL
-
-    Returns:
-        List of PDF URLs extracted by Zotero translator
-    """
-    try:
-        from ._ZoteroTranslatorRunner import ZoteroTranslatorRunner
-
-        runner = ZoteroTranslatorRunner()
-
-        # Execute translator if one matches this URL
-        pdf_urls = await runner.extract_pdf_urls_async(page)
-
-        if pdf_urls:
-            logger.success(f"Zotero translator found {len(pdf_urls)} PDF URLs")
-            for pdf_url in pdf_urls:
-                logger.debug(f"  - {pdf_url}")
-        else:
-            logger.warning(f"Zotero translator did not find any PDF URLs")
-
-        return pdf_urls
-
-    except ImportError as e:
-        logger.warning(f"ZoteroTranslatorRunner not available: {e}")
-        return []
-    except Exception as e:
-        logger.error(f"Error running Zotero translator: {e}")
-        return []
-
-
-async def find_all_urls(page: Page) -> Dict[str, List[Dict]]:
-    """
-    Find all URL types in a page.
-
-    Returns:
-        Dict with 'pdf' and 'supplementary' keys, each containing list of dicts with source info
-    """
-    return {
-        "pdf": await find_pdf_urls(page),
-        "supplementary": await find_supplementary_urls(page),
-    }
-
-
-# EOF

scitex/scholar/.legacy/metadata/urls/_handler.py

@@ -1,326 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-15 16:21:23 (ywatanabe)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/urls/_handler.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-"""
-URLHandler - Main entry point for URL operations
-
-Provides a clean API that wraps the functional modules.
-Users can use this for convenience or directly import the functions.
-"""
-
-import json
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-from playwright.async_api import BrowserContext, Page
-
-from scitex import logging
-from scitex.scholar.config import ScholarConfig
-
-# Import functional modules
-from ._finder import find_all_urls, find_pdf_urls, find_supplementary_urls
-from ._resolver import (
-    build_url_doi,
-    doi_to_url_publisher,
-    extract_doi_from_url,
-    generate_openurl_query,
-    resolve_all_urls,
-    resolve_openurl,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class URLHandler:
-    """
-    Main entry point for all URL operations.
-
-    Wraps functional modules for convenience while keeping them accessible
-    for direct use when needed.
-    """
-
-    def __init__(
-        self,
-        context: BrowserContext = None,
-        openurl_resolver_url=None,
-        config=None,
-    ):
-        """
-        Initialize URL handler.
-
-        Args:
-            context: Authenticated browser context (optional)
-        """
-        self.context = context
-        self.config = config or ScholarConfig()
-        self.openurl_resolver_url = self.config.resolve(
-            "openurl_resolver_url", openurl_resolver_url
-        )
-
-    async def get_all_urls(
-        self,
-        doi: str = None,
-        title: str = None,
-        metadata: Dict = None,
-        page: Page = None,
-    ) -> Dict[str, Any]:
-        """
-        Get all URL types for a paper.
-
-        Args:
-            doi: DOI string
-            title: Paper title (for OpenURL)
-            metadata: Full metadata dict (for OpenURL)
-            page: Current page (for PDF extraction)
-
-        Returns:
-            Dict with all resolved URLs
-        """
-        urls = {}
-
-        # If we have metadata, use it
-        if metadata:
-            # Use the functional resolver
-            urls = await resolve_all_urls(
-                metadata, self.openurl_resolver_url, self.context
-            )
-
-        # If we have a DOI, resolve it
-        elif doi:
-            urls["url_doi"] = build_url_doi(doi)
-
-            if self.context:
-                try:
-                    url_publisher = await doi_to_url_publisher(doi, self.context)
-                    if url_publisher:
-                        urls["url_publisher"] = url_publisher
-
-                        # Try to get PDF URLs from publisher page
-                        if not page:
-                            page = await self.context.new_page()
-                        try:
-                            await page.goto(
-                                url_publisher,
-                                wait_until="domcontentloaded",
-                                timeout=30000,
-                            )
-                            pdf_urls = await find_pdf_urls(page)
-                            if pdf_urls:
-                                urls["url_pdf"] = pdf_urls
-                        except Exception as e:
-                            logger.warning(f"Failed to navigate to publisher URL: {e}")
-                            # Try with networkidle instead
-                            try:
-                                await page.goto(
-                                    url_publisher,
-                                    wait_until="networkidle",
-                                    timeout=30000,
-                                )
-                                pdf_urls = await find_pdf_urls(page)
-                                if pdf_urls:
-                                    urls["url_pdf"] = pdf_urls
-                            except Exception as e2:
-                                logger.error(f"Could not access publisher page: {e2}")
-
-                        # Extract all URLs from page before closing
-                        if page:
-                            try:
-                                page_urls = await find_all_urls(page)
-                                if page_urls.get("pdf"):
-                                    urls["url_pdf"] = page_urls["pdf"]
-                                if page_urls.get("supplementary"):
-                                    urls["url_supplementary"] = page_urls[
-                                        "supplementary"
-                                    ]
-                            except Exception as e:
-                                logger.error(f"Error extracting URLs from page: {e}")
-
-                        # Now close the page
-                        try:
-                            await page.close()
-                        except:
-                            pass  # Page might already be closed
-
-                except Exception as e:
-                    logger.error(f"Error resolving DOI {doi}: {e}")
-
-        return urls
-
-    async def find_pdf_urls_async(self, page_or_url) -> List[Dict]:
-        """
-        Find PDF URLs from a page or URL.
-
-        Args:
-            page_or_url: Page object or URL string
-
-        Returns:
-            List of dicts with url, source, and reliability
-        """
-        if isinstance(page_or_url, str):
-            # Create a page and navigate
-            if not self.context:
-                logger.error("Browser context required to navigate to URL")
-                return []
-
-            page = await self.context.new_page()
-            try:
-                await page.goto(
-                    page_or_url, wait_until="domcontentloaded", timeout=30000
-                )
-                return await find_pdf_urls(page)
-            except Exception as e:
-                logger.warning(f"Failed with domcontentloaded, trying networkidle: {e}")
-                try:
-                    await page.goto(
-                        page_or_url, wait_until="networkidle", timeout=30000
-                    )
-                    return await find_pdf_urls(page)
-                except Exception as e2:
-                    logger.error(f"Could not navigate to {page_or_url}: {e2}")
-                    return []
-            finally:
-                await page.close()
-        else:
-            # Assume it's a Page object
-            try:
-                return await find_pdf_urls(page_or_url)
-            except Exception as e:
-                logger.error(f"Error finding PDF URLs: {e}")
-                return []
-
-    def generate_openurl(
-        self, metadata: Dict, openurl_resolver_url: str
-    ) -> Optional[str]:
-        """
-        Generate OpenURL query from metadata.
-
-        Args:
-            metadata: Paper metadata
-
-        Returns:
-            OpenURL query string
-        """
-        return generate_openurl_query(metadata, openurl_resolver_url)
-
-    async def resolve_openurl_async(self, openurl_query: str) -> Optional[str]:
-        """
-        Resolve OpenURL to final authenticated URL.
-
-        Args:
-            openurl_query: OpenURL query string
-
-        Returns:
-            Final resolved URL
-        """
-        if not self.context:
-            logger.error("Browser context required for OpenURL resolution")
-            return None
-
-        return await resolve_openurl(openurl_query, self.context)
-
-    def update_metadata(self, metadata_path: Path, urls: Dict) -> bool:
-        """
-        Update metadata file with resolved URLs.
-
-        Args:
-            metadata_path: Path to metadata.json
-            urls: URLs to add/update
-
-        Returns:
-            True if updated successfully
-        """
-        try:
-            # Load existing metadata
-            with open(metadata_path, "r") as f:
-                metadata = json.load(f)
-
-            # Initialize urls section if not present
-            if "urls" not in metadata:
-                metadata["urls"] = {}
-
-            # Update with new URLs
-            metadata["urls"].update(urls)
-
-            # Save back
-            with open(metadata_path, "w") as f:
-                json.dump(metadata, f, indent=2, default=str)
-
-            logger.success(f"Updated metadata: {metadata_path.parent.name}")
-            return True
-
-        except Exception as e:
-            logger.error(f"Failed to update metadata: {e}")
-            return False
-
-    def get_urls_from_metadata(self, metadata_path: Path) -> Dict:
-        """
-        Get URLs from metadata file.
-
-        Args:
-            metadata_path: Path to metadata.json
-
-        Returns:
-            URLs dict from metadata
-        """
-        try:
-            with open(metadata_path, "r") as f:
-                metadata = json.load(f)
-
-            return metadata.get("urls", {})
-
-        except Exception as e:
-            logger.error(f"Failed to read metadata: {e}")
-            return {}
-
-    @staticmethod
-    def extract_doi(url: str) -> Optional[str]:
-        """
-        Extract DOI from a URL.
-
-        Args:
-            url: Any URL that might contain a DOI
-
-        Returns:
-            DOI string if found
-        """
-        return extract_doi_from_url(url)
-
-
-# Convenience functions for direct use without instantiating the class
-
-
-async def get_all_urls(
-    doi: str = None,
-    title: str = None,
-    metadata: Dict = None,
-    context: BrowserContext = None,
-) -> Dict[str, Any]:
-    """
-    Convenience function to get all URLs without instantiating URLHandler.
-
-    See URLHandler.get_all_urls for details.
-    """
-    handler = URLHandler(context)
-    return await handler.get_all_urls(doi, title, metadata)
-
-
-def update_metadata_urls(metadata_path: Path, urls: Dict) -> bool:
-    """
-    Convenience function to update metadata without instantiating URLHandler.
-
-    See URLHandler.update_metadata for details.
-    """
-    handler = URLHandler()
-    return handler.update_metadata(metadata_path, urls)
-
-
-# EOF