scitex 2.17.3__py3-none-any.whl → 2.17.4__py3-none-any.whl

scitex/scholar/url_finder/.tmp/open_url/KNOWN_RESOLVERS.py

@@ -1,462 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2025-08-01 13:15:00"
-# Author: Claude
-# File: KNOWN_RESOLVERS.py
-
-"""
-Known OpenURL resolvers from various institutions worldwide.
-
-This module contains a curated list of OpenURL resolvers used by
-academic institutions for accessing scholarly content.
-
-Sources:
-- Zotero OpenURL Resolver Directory: https://www.zotero.org/openurl_resolvers
-- Individual institution library websites
-- Common resolver patterns
-"""
-
-from typing import Dict, List, Optional
-
-# Major OpenURL resolver vendors
-RESOLVER_VENDORS = {
-    "ExLibris": {
-        "patterns": ["sfx", "exlibrisgroup.com"],
-        "description": "Ex Libris SFX resolver (very common)"
-    },
-    "SerialsSolutions": {
-        "patterns": ["serialssolutions.com", "360link"],
-        "description": "ProQuest SerialsSolutions 360 Link"
-    },
-    "EBSCO": {
-        "patterns": ["ebscohost.com/openurlresolver", "linkssource.ebsco.com"],
-        "description": "EBSCO Full Text Finder"
-    },
-    "OCLC": {
-        "patterns": ["worldcat.org", "oclc.org"],
-        "description": "OCLC WorldCat resolver"
-    },
-    "Ovid": {
-        "patterns": ["ovid.com", "linksolver"],
-        "description": "Ovid LinkSolver"
-    }
-}
-
-# Known institutional OpenURL resolvers
-KNOWN_RESOLVERS: Dict[str, Dict[str, str]] = {
-    # United States
-    "Harvard University": {
-        "url": "https://sfx.hul.harvard.edu/sfx_local",
-        "country": "US",
-        "vendor": "ExLibris"
-    },
-    "MIT": {
-        "url": "https://owens.mit.edu/sfx_local",
-        "country": "US",
-        "vendor": "ExLibris"
-    },
-    "Stanford University": {
-        "url": "https://stanford.idm.oclc.org/login?url=",
-        "country": "US",
-        "vendor": "OCLC"
-    },
-    "Yale University": {
-        "url": "https://yale.idm.oclc.org/login?url=",
-        "country": "US",
-        "vendor": "OCLC"
-    },
-    "University of California, Berkeley": {
-        "url": "https://ucelinks.cdlib.org:8443/sfx_ucb",
-        "country": "US",
-        "vendor": "ExLibris"
-    },
-    "UCLA": {
-        "url": "https://ucelinks.cdlib.org:8443/sfx_ucla",
-        "country": "US",
-        "vendor": "ExLibris"
-    },
-    "Columbia University": {
-        "url": "https://resolver.library.columbia.edu/openurl",
-        "country": "US",
-        "vendor": "SerialsSolutions"
-    },
-    "Princeton University": {
-        "url": "https://princeton.idm.oclc.org/login?url=",
-        "country": "US",
-        "vendor": "OCLC"
-    },
-    "University of Chicago": {
-        "url": "https://proxy.uchicago.edu/login?url=",
-        "country": "US",
-        "vendor": "Custom"
-    },
-    "Johns Hopkins": {
-        "url": "https://openurl.library.jhu.edu",
-        "country": "US",
-        "vendor": "Custom"
-    },
-    
-    # United Kingdom
-    "University of Oxford": {
-        "url": "https://fs.oxfordjournals.org/openurl",
-        "country": "UK",
-        "vendor": "Custom"
-    },
-    "University of Cambridge": {
-        "url": "https://cambridge.idm.oclc.org/login?url=",
-        "country": "UK",
-        "vendor": "OCLC"
-    },
-    "Imperial College London": {
-        "url": "https://imperial.idm.oclc.org/login?url=",
-        "country": "UK",
-        "vendor": "OCLC"
-    },
-    "UCL": {
-        "url": "https://ucl.idm.oclc.org/login?url=",
-        "country": "UK",
-        "vendor": "OCLC"
-    },
-    "University of Edinburgh": {
-        "url": "https://discovered.ed.ac.uk/openurl",
-        "country": "UK",
-        "vendor": "Custom"
-    },
-    
-    # Canada
-    "University of Toronto": {
-        "url": "https://myaccess.library.utoronto.ca/login?url=",
-        "country": "CA",
-        "vendor": "Custom"
-    },
-    "McGill University": {
-        "url": "https://mcgill.on.worldcat.org/atoztitles/link",
-        "country": "CA",
-        "vendor": "OCLC"
-    },
-    "University of British Columbia": {
-        "url": "https://ubc.summon.serialssolutions.com/link",
-        "country": "CA",
-        "vendor": "SerialsSolutions"
-    },
-    
-    # Australia
-    "University of Melbourne": {
-        "url": "https://unimelb.hosted.exlibrisgroup.com/sfxlcl41",
-        "country": "AU",
-        "vendor": "ExLibris"
-    },
-    "University of Sydney": {
-        "url": "https://ap01.alma.exlibrisgroup.com/view/uresolver/61USYD_INST/openurl",
-        "country": "AU",
-        "vendor": "ExLibris"
-    },
-    "Australian National University": {
-        "url": "https://anu.hosted.exlibrisgroup.com/primo-explore/openurl",
-        "country": "AU",
-        "vendor": "ExLibris"
-    },
-    "University of Queensland": {
-        "url": "https://uq.summon.serialssolutions.com/link",
-        "country": "AU",
-        "vendor": "SerialsSolutions"
-    },
-    "Monash University": {
-        "url": "https://monash.hosted.exlibrisgroup.com/sfx_local",
-        "country": "AU",
-        "vendor": "ExLibris"
-    },
-    
-    # Germany
-    "Max Planck Society": {
-        "url": "http://sfx.mpg.de/sfx_local",
-        "country": "DE",
-        "vendor": "ExLibris"
-    },
-    "University of Munich (LMU)": {
-        "url": "https://sfx.bib.uni-muenchen.de/sfx_lmu",
-        "country": "DE",
-        "vendor": "ExLibris"
-    },
-    "Heidelberg University": {
-        "url": "https://sfx.bib.uni-heidelberg.de/sfx_heidelberg",
-        "country": "DE",
-        "vendor": "ExLibris"
-    },
-    
-    # Netherlands
-    "University of Amsterdam": {
-        "url": "https://vu-nl.idm.oclc.org/login?url=",
-        "country": "NL",
-        "vendor": "OCLC"
-    },
-    "Delft University of Technology": {
-        "url": "https://tudelft.idm.oclc.org/login?url=",
-        "country": "NL",
-        "vendor": "OCLC"
-    },
-    
-    # France
-    "Sorbonne University": {
-        "url": "https://accesdistant.sorbonne-universite.fr/login?url=",
-        "country": "FR",
-        "vendor": "Custom"
-    },
-    "École Polytechnique": {
-        "url": "https://portail.polytechnique.edu/openurl",
-        "country": "FR",
-        "vendor": "Custom"
-    },
-    
-    # Switzerland
-    "ETH Zurich": {
-        "url": "https://www.library.ethz.ch/openurl",
-        "country": "CH",
-        "vendor": "Custom"
-    },
-    "EPFL": {
-        "url": "https://sfx.epfl.ch/sfx_local",
-        "country": "CH",
-        "vendor": "ExLibris"
-    },
-    
-    # Japan
-    "University of Tokyo": {
-        "url": "https://vs2ga4mq9g.search.serialssolutions.com",
-        "country": "JP",
-        "vendor": "SerialsSolutions"
-    },
-    "Kyoto University": {
-        "url": "https://kuline.kulib.kyoto-u.ac.jp/portal/openurl",
-        "country": "JP",
-        "vendor": "Custom"
-    },
-    
-    # Singapore
-    "National University of Singapore": {
-        "url": "https://libproxy.nus.edu.sg/login?url=",
-        "country": "SG",
-        "vendor": "Custom"
-    },
-    "Nanyang Technological University": {
-        "url": "https://ap01.alma.exlibrisgroup.com/view/uresolver/65NTU_INST/openurl",
-        "country": "SG",
-        "vendor": "ExLibris"
-    },
-    
-    # China
-    "Tsinghua University": {
-        "url": "http://sfx.lib.tsinghua.edu.cn/sfx_local",
-        "country": "CN",
-        "vendor": "ExLibris"
-    },
-    "Peking University": {
-        "url": "http://sfx.lib.pku.edu.cn/sfx_pku",
-        "country": "CN",
-        "vendor": "ExLibris"
-    },
-    
-    # South Korea
-    "Seoul National University": {
-        "url": "https://sfx.snu.ac.kr/sfx_local",
-        "country": "KR",
-        "vendor": "ExLibris"
-    },
-    "KAIST": {
-        "url": "https://library.kaist.ac.kr/openurl",
-        "country": "KR",
-        "vendor": "Custom"
-    },
-    
-    # Brazil
-    "University of São Paulo": {
-        "url": "http://www.buscaintegrada.usp.br/openurl",
-        "country": "BR",
-        "vendor": "Custom"
-    },
-    
-    # Mexico
-    "UNAM": {
-        "url": "https://pbidi.unam.mx/login?url=",
-        "country": "MX",
-        "vendor": "Custom"
-    },
-    
-    # India
-    "IIT Delhi": {
-        "url": "https://libproxy.iitd.ac.in/login?url=",
-        "country": "IN",
-        "vendor": "Custom"
-    },
-    "Indian Institute of Science": {
-        "url": "https://library.iisc.ac.in/openurl",
-        "country": "IN",
-        "vendor": "Custom"
-    }
-}
-
-# Generic OpenURL resolver patterns
-GENERIC_PATTERNS = [
-    # ExLibris SFX patterns
-    r"https?://[^/]+/sfx[^/]*",
-    r"https?://sfx\.[^/]+",
-    r"https?://[^/]+\.exlibrisgroup\.com",
-    
-    # SerialsSolutions patterns
-    r"https?://[^/]+\.serialssolutions\.com",
-    r"https?://[^/]+/360link",
-    
-    # OCLC patterns
-    r"https?://[^/]+\.idm\.oclc\.org",
-    r"https?://[^/]+\.worldcat\.org",
-    
-    # Common proxy patterns
-    r"https?://[^/]+/login\?url=",
-    r"https?://libproxy\.[^/]+",
-    r"https?://proxy\.[^/]+",
-    
-    # OpenURL patterns
-    r"https?://[^/]+/openurl",
-    r"https?://[^/]+/openurlresolver",
-]
-
-
-def get_resolver_by_institution(institution_name: str) -> Optional[Dict[str, str]]:
-    """
-    Get OpenURL resolver information by institution name.
-    
-    Args:
-        institution_name: Name of the institution
-        
-    Returns:
-        Dict with 'url', 'country', and 'vendor' if found, None otherwise
-    """
-    # Try exact match first
-    if institution_name in KNOWN_RESOLVERS:
-        return KNOWN_RESOLVERS[institution_name].copy()
-    
-    # Try case-insensitive match
-    institution_lower = institution_name.lower()
-    for name, info in KNOWN_RESOLVERS.items():
-        if name.lower() == institution_lower:
-            return info.copy()
-    
-    # Try partial match
-    for name, info in KNOWN_RESOLVERS.items():
-        if institution_lower in name.lower() or name.lower() in institution_lower:
-            return info.copy()
-    
-    return None
-
-
-def get_resolvers_by_country(country_code: str) -> Dict[str, Dict[str, str]]:
-    """
-    Get all OpenURL resolvers for a specific country.
-    
-    Args:
-        country_code: Two-letter country code (e.g., 'US', 'UK', 'AU')
-        
-    Returns:
-        Dict of institution names to resolver info
-    """
-    country_code = country_code.upper()
-    return {
-        name: info 
-        for name, info in KNOWN_RESOLVERS.items() 
-        if info.get('country') == country_code
-    }
-
-
-def get_resolvers_by_vendor(vendor_name: str) -> Dict[str, Dict[str, str]]:
-    """
-    Get all OpenURL resolvers using a specific vendor.
-    
-    Args:
-        vendor_name: Vendor name (e.g., 'ExLibris', 'OCLC')
-        
-    Returns:
-        Dict of institution names to resolver info
-    """
-    return {
-        name: info 
-        for name, info in KNOWN_RESOLVERS.items() 
-        if info.get('vendor', '').lower() == vendor_name.lower()
-    }
-
-
-def validate_resolver_url(url: str) -> bool:
-    """
-    Check if a URL looks like a valid OpenURL resolver.
-    
-    Args:
-        url: URL to validate
-        
-    Returns:
-        True if URL matches known resolver patterns
-    """
-    import re
-
-    # Check against known resolver URLs
-    for info in KNOWN_RESOLVERS.values():
-        if url.startswith(info['url']):
-            return True
-    
-    # Check against generic patterns
-    for pattern in GENERIC_PATTERNS:
-        if re.match(pattern, url):
-            return True
-    
-    return False
-
-
-def get_all_resolvers() -> List[Dict[str, str]]:
-    """
-    Get all known resolvers as a list.
-    
-    Returns:
-        List of dicts with 'name', 'url', 'country', 'vendor'
-    """
-    return [
-        {
-            'name': name,
-            'url': info['url'],
-            'country': info.get('country', 'Unknown'),
-            'vendor': info.get('vendor', 'Unknown')
-        }
-        for name, info in KNOWN_RESOLVERS.items()
-    ]
-
-
-# Common test DOIs for different publishers
-TEST_DOIS = {
-    "Nature": "10.1038/nature12373",
-    "Science": "10.1126/science.1234567",
-    "Cell": "10.1016/j.cell.2020.01.001",
-    "Elsevier": "10.1016/j.neuroimage.2020.116584",
-    "Wiley": "10.1111/jnc.15327",
-    "Springer": "10.1007/s00401-021-02283-6",
-    "Oxford": "10.1093/brain/awaa123",
-    "IEEE": "10.1109/TPAMI.2020.2984611",
-    "ACS": "10.1021/acs.jmedchem.0c00606",
-    "PNAS": "10.1073/pnas.1921909117"
-}
-
-
-if __name__ == "__main__":
-    # Example usage
-    print(f"Total known resolvers: {len(KNOWN_RESOLVERS)}")
-    print(f"\nCountries represented: {len(set(info['country'] for info in KNOWN_RESOLVERS.values()))}")
-    print(f"Vendors: {set(info.get('vendor', 'Unknown') for info in KNOWN_RESOLVERS.values())}")
-    
-    # Example: Find resolver for an institution
-    resolver = get_resolver_by_institution("Harvard")
-    if resolver:
-        print(f"\nHarvard resolver: {resolver['url']}")
-    
-    # Example: Get all US resolvers
-    us_resolvers = get_resolvers_by_country("US")
-    print(f"\nUS institutions with resolvers: {len(us_resolvers)}")
-    
-    # Example: Get all ExLibris resolvers
-    exlibris = get_resolvers_by_vendor("ExLibris")
-    print(f"Institutions using ExLibris SFX: {len(exlibris)}")

scitex/scholar/url_finder/.tmp/open_url/README.md

@@ -1,223 +0,0 @@
-<!-- ---
-!-- Timestamp: 2025-08-03 00:51:52
-!-- Author: ywatanabe
-!-- File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/open_url/README.md
-!-- --- -->
-
-# OpenURL Resolvers
-
-This module provides OpenURL resolver implementations with automatic ZenRows integration when API key is present.
-
-**Key Feature**: ZenRows stealth browser is automatically enabled when `SCITEX_SCHOLAR_ZENROWS_API_KEY` is set, providing:
-- 🛡️ Anti-bot protection with residential IPs
-- 🌐 Full browser control for authentication
-- 🚀 Automatic bypass of rate limits and CAPTCHAs
-
-## 1. OpenURLResolver (Standard)
-
-The standard browser-based resolver using Playwright.
-
-**Best for:**
-- Authenticated access to paywalled content
-- Complex JavaScript-based authentication flows
-- Sites that require real browser interactions
-
-**Limitations:**
-- Can be blocked by anti-bot measures
-- May encounter CAPTCHAs or rate limits
-
-```python
-from scitex.scholar.open_url import OpenURLResolver
-from scitex.scholar.auth import AuthenticationManager
-
-auth_manager = AuthenticationManager(email_openathens="your@email.com")
-resolver = OpenURLResolver(auth_manager, "https://your.resolver.url/")
-
-result = await resolver.resolve_async(doi="10.1038/nature12373")
-```
-
-## 2. OpenURLResolverWithZenRows (API-based)
-
-Uses ZenRows API to bypass anti-bot detection while making HTTP requests.
-
-**Best for:**
-- High-volume resolution tasks
-- Bypassing rate limits and IP blocks
-- Open access content detection
-
-**Limitations:**
-- Cannot execute JavaScript (no popup handling)
-- Limited authentication cookie transfer to publishers
-- May show_async "Purchase" for paywalled content even with auth
-
-```python
-from scitex.scholar.open_url import OpenURLResolverWithZenRows
-
-resolver = OpenURLResolverWithZenRows(
-    auth_manager, 
-    resolver_url,
-    zenrows_api_key="your_api_key"  # or set SCITEX_SCHOLAR_ZENROWS_API_KEY
-)
-
-result = await resolver.resolve_async(doi="10.1038/nature12373")
-```
-
-## 3. ZenRowsOpenURLResolver (Browser-based)
-
-Uses ZenRows Scraping Browser service - cloud-based Chrome instances with anti-bot bypass.
-
-**Best for:**
-- Sites with aggressive anti-bot protection (e.g., PNAS)
-- Maintaining full authentication context
-- JavaScript-heavy authentication flows with anti-bot measures
-
-**Limitations:**
-- Requires ZenRows API key
-- Slightly slower due to remote browser
-- May have concurrency limits based on plan
-
-```python
-from scitex.scholar.open_url import ZenRowsOpenURLResolver
-
-resolver = ZenRowsOpenURLResolver(
-    auth_manager,
-    resolver_url,
-    zenrows_api_key="your_api_key"  # or set SCITEX_SCHOLAR_ZENROWS_API_KEY
-)
-
-result = await resolver.resolve_async(doi="10.1073/pnas.0608765104")
-```
-
-## Usage Example (Synchronous)
-
-```python
-from scitex.scholar.open_url import OpenURLResolver, ZenRowsOpenURLResolver
-from scitex.scholar.auth import AuthenticationManager
-import os
-from scitex import logging
-
-# Enable debug logging
-logger = logging.getLogger()
-logger.setLevel(logging.DEBUG)
-
-# Initialize authentication
-auth_manager = AuthenticationManager(
-    email_openathens=os.getenv("SCITEX_SCHOLAR_OPENATHENS_EMAIL")
-)
-is_authenticate_async = await auth_manager.is_authenticate_async()
-
-# Choose your resolver
-# Standard browser-based resolver
-resolver = OpenURLResolver(
-    auth_manager, 
-    os.getenv("SCITEX_SCHOLAR_OPENURL_RESOLVER_URL")
-)
-
-
-# # OR: ZenRows cloud browser resolver (for anti-bot bypass)
-# resolver = ZenRowsOpenURLResolver(
-#     auth_manager, 
-#     os.getenv("SCITEX_SCHOLAR_OPENURL_RESOLVER_URL"),
-#     os.getenv("SCITEX_SCHOLAR_ZENROWS_API_KEY"))
-
-
-# DOIs to resolve
-dois = [
-    "10.1038/nature12373",
-    "10.1016/j.neuron.2018.01.048",
-    "10.1126/science.1172133",
-    "10.1073/pnas.0608765104",
-]
-
-#    "10.1002/hipo.22488",
-# # Resolve single DOI
-# result = resolver._resolve_single(doi=dois[0])
-
-# Resolve multiple DOIs in parallel
-results = resolver.resolve(dois)
-```
-
-## Choosing the Right Resolver
-
-| Scenario | Recommended Resolver |
-|----------|---------------------|
-| General academic paper access | OpenURLResolver |
-| High-volume batch processing | OpenURLResolverWithZenRows |
-| Sites blocking normal browsers | ZenRowsOpenURLResolver |
-| PNAS, sites with "unusual traffic" errors | ZenRowsOpenURLResolver |
-| Need full JavaScript execution + anti-bot | ZenRowsOpenURLResolver |
-
-## Automatic Fallback Strategy
-
-You can implement automatic fallback between resolvers:
-
-```python
-async def resolve_with_fallback_async(doi, metadata):
-    # Try standard resolver first
-    result = await standard_resolver.resolve_async(doi=doi, **metadata)
-    
-    if result and result.get('success'):
-        return result
-    
-    # Check for anti-bot indicators
-    if result and result.get('access_type') in ['captcha_required', 'rate_limited']:
-        # Try ZenRows browser resolver
-        return await zenrows_browser_resolver.resolve_async(doi=doi, **metadata)
-    
-    return result
-```
-
-## NEW: Simplified ZenRows Stealth Browser (Recommended)
-
-As of the latest update, ZenRows stealth capabilities are automatically integrated when the API key is present:
-
-```python
-# Just set the API key - ZenRows stealth is automatically enabled!
-os.environ["SCITEX_SCHOLAR_ZENROWS_API_KEY"] = "your_api_key"
-
-from scitex.scholar import Scholar
-
-# Scholar automatically uses ZenRows stealth browser
-scholar = Scholar()
-
-# Download with automatic anti-bot protection
-papers = await scholar.download_pdf_asyncs_async(
-    ["10.1038/nature12373", "10.1073/pnas.0608765104"],
-    show_async_progress=True
-)
-```
-
-This provides:
-- **Local browser window** you can see and interact with
-- **ZenRows proxy** for clean residential IPs
-- **Manual login** capability for complex SSO/2FA
-- **Automatic anti-bot bypass** for all operations
-
-## Environment Variables
-
-- `SCITEX_SCHOLAR_ZENROWS_API_KEY`: Your ZenRows API key (auto-enables stealth)
-- `SCITEX_SCHOLAR_OPENATHENS_EMAIL`: Email for OpenAthens authentication
-- `SCITEX_SCHOLAR_OPENURL_RESOLVER_URL`: Your institutional OpenURL resolver
-
-## Architecture
-
-```
-OpenURL Resolvers
-├── _OpenURLResolver.py          # Base implementation with Playwright
-├── _OpenURLResolverWithZenRows.py  # API-based ZenRows integration
-├── _ZenRowsOpenURLResolver.py      # Browser-based ZenRows integration
-└── _ResolverLinkFinder.py          # Shared link detection logic
-
-Browser Managers
-├── _BrowserManager.py              # Standard local browser
-├── _ProxyBrowserManager.py         # Local browser + proxy routing
-└── _ZenRowsBrowserManager.py       # Cloud browser instances
-```
-
-The separation ensures:
-- Clean architecture with single responsibility
-- Easy switching between implementations
-- No interference with other browser-based operations
-- Flexibility to use different strategies for different papers
-
-<!-- EOF -->