scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/scholar/auth/core/__init__.py

@@ -0,0 +1,16 @@
+"""Core authentication modules."""
+
+from .AuthenticationGateway import AuthenticationGateway, URLContext
+from .BrowserAuthenticator import BrowserAuthenticator
+from .StrategyResolver import AuthenticationStrategyResolver, AuthenticationStrategy, AuthenticationMethod
+
+__all__ = [
+    "AuthenticationGateway",
+    "URLContext",
+    "BrowserAuthenticator",
+    "AuthenticationStrategyResolver",
+    "AuthenticationStrategy",
+    "AuthenticationMethod",
+]
+
+# EOF

scitex/scholar/auth/gateway/_OpenURLLinkFinder.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-10-10 02:00:31 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/auth/gateway/_OpenURLLinkFinder.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = (
+    "./src/scitex/scholar/auth/gateway/_OpenURLLinkFinder.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+__FILE__ = __file__
+
+import asyncio
+from typing import List
+
+from playwright.async_api import Locator, Page
+
+from scitex import logging
+from scitex.browser.debugging import highlight_element_async
+from scitex.scholar import ScholarConfig
+from scitex.scholar.browser.utils import click_and_wait
+
+logger = logging.getLogger(__name__)
+
+
+class OpenURLLinkFinder:
+    """Finds full-text links on resolver pages by publisher name."""
+
+    def __init__(self, config: ScholarConfig = None):
+        self.name = self.__class__.__name__
+        self.config = config or ScholarConfig()
+
+    async def find_link_elements(self, page: Page, doi: str) -> List[Locator]:
+        """Find and highlight publisher text on the page."""
+        try:
+            logger.info(f"{self.name}: Finding links from {page.url}")
+
+            openurl_available_from_patterns = self.config.resolve(
+                "openurl_available_from_patterns", None
+            )
+            seen_hrefs = set()
+            found_links = []
+
+            for pattern in openurl_available_from_patterns:
+                publisher = pattern[len("Available from ") :]
+                try:
+                    link_element = page.locator(
+                        f'a:has-text("{publisher}")'
+                    ).first
+                    if await link_element.count() > 0:
+                        href = await link_element.get_attribute("href")
+                        if href not in seen_hrefs:
+                            # logger.info(f"Found link elements for: {publisher}")
+                            await highlight_element_async(link_element, 500)
+                            found_links.append(
+                                {
+                                    "publisher": publisher,
+                                    "link_element": link_element,
+                                }
+                            )
+                            seen_hrefs.add(href)
+                except Exception as e:
+                    logger.debug(
+                        f"{self.name}: Could not find {publisher}: {e}"
+                    )
+
+            if found_links:
+                publishers = [
+                    found_link.get("publisher") for found_link in found_links
+                ]
+                logger.info(
+                    f"{self.name}: Found {len(publishers)} link elements for: {', '.join(publishers)}"
+                )
+                return found_links
+
+        except Exception as e:
+            logger.fail(
+                f"{self.name}: Did not find any urls from {page.url}: {e}"
+            )
+            return []
+
+
+if __name__ == "__main__":
+
+    async def main():
+        from scitex.scholar import (
+            ScholarAuthManager,
+            ScholarBrowserManager,
+            ScholarURLFinder,
+        )
+
+        auth_manager = ScholarAuthManager()
+        browser_manager = ScholarBrowserManager(
+            auth_manager=auth_manager,
+            browser_mode="interactive",
+            chrome_profile_name="system",
+        )
+        browser, context = (
+            await browser_manager.get_authenticated_browser_and_context_async()
+        )
+        page = await context.new_page()
+        url = "https://unimelb.hosted.exlibrisgroup.com/sfxlcl41?doi=10.1126/science.aao0702"
+        await page.goto(url, wait_until="networkidle", timeout=30_000)
+        await page.wait_for_timeout(3000)
+
+        finder = OpenURLLinkFinder()
+        links = await finder.find_link_elements(
+            page, "10.1126/science.aao0702"
+        )
+
+        result = await click_and_wait(links[0].get("link_element"))
+
+    asyncio.run(main())
+
+# python -m scitex.scholar.auth.gateway._OpenURLLinkFinder
+
+# EOF

scitex/scholar/auth/gateway/_OpenURLResolver.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-10-11 07:52:38 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/auth/gateway/_OpenURLResolver.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = (
+    "./src/scitex/scholar/auth/gateway/_OpenURLResolver.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+__FILE__ = __file__
+
+from typing import Optional
+from urllib.parse import quote
+
+from playwright.async_api import Locator, Page
+
+from scitex import logging
+from scitex.browser.debugging import browser_logger
+from scitex.scholar import ScholarConfig
+from scitex.scholar.browser.utils import click_and_wait
+
+from ._OpenURLLinkFinder import OpenURLLinkFinder
+
+logger = logging.getLogger(__name__)
+
+
+class OpenURLResolver:
+    """Handles OpenURL resolution and authentication flow."""
+
+    def __init__(self, config: ScholarConfig = None):
+        self.name = self.__class__.__name__
+        self.config = config or ScholarConfig()
+        self.resolver_url = self.config.resolve("openurl_resolver_url")
+        self.finder = OpenURLLinkFinder(config=config)
+
+    async def resolve_doi(self, doi: str, page: Page) -> Optional[str]:
+        """Main entry point: resolve DOI through OpenURL to authenticated URL."""
+        openurl_query = self._build_query(doi)
+        if not openurl_query:
+            return None
+        return await self._resolve_query(openurl_query, page, doi)
+
+    def _build_query(self, doi: str, title: str = None) -> Optional[str]:
+        """Build OpenURL query string."""
+        if not self.resolver_url:
+            return None
+
+        params = [f"doi={doi}"]
+        if title:
+            params.append(f"atitle={quote(title[:200])}")
+
+        built_query = f"{self.resolver_url}?{'&'.join(params)}"
+        logger.debug(f"{self.name}: Built query URL: {built_query}")
+        return built_query
+
+    async def _resolve_query(
+        self, query: str, page: Page, doi: str
+    ) -> Optional[str]:
+        """Resolve OpenURL query to final authenticated URL with retry."""
+        logger.debug(f"{self.name}: Resolving query URL: {query}...")
+
+        for attempt in range(3):
+            try:
+                await page.goto(
+                    query, wait_until="domcontentloaded", timeout=60000
+                )
+                await browser_logger.debug(
+                    page,
+                    f"{self.name}: Loaded resolver page at {page.url[:60]}",
+                )
+
+                # Visual: Waiting for page to stabilize
+                await browser_logger.debug(
+                    page,
+                    f"{self.name}: Waiting for resolver to load (networkidle)...",
+                )
+                try:
+                    await page.wait_for_load_state(
+                        "networkidle", timeout=15_000
+                    )
+                    await browser_logger.debug(
+                        page, f"{self.name}: ✓ Resolver page ready"
+                    )
+                except Exception:
+                    await browser_logger.info(
+                        page, f"{self.name}: Page still loading, continuing..."
+                    )
+                await page.wait_for_timeout(1000)
+
+                # Visual: Finding publisher links
+                await browser_logger.info(
+                    page, f"{self.name}: Searching for publisher links..."
+                )
+                found_links = await self.finder.find_link_elements(page, doi)
+
+                if not found_links:
+                    await browser_logger.info(
+                        page, f"{self.name}: No publisher links found"
+                    )
+                    await page.wait_for_timeout(2000)
+                    return None
+
+                # Visual: Found links
+                await browser_logger.debug(
+                    page,
+                    f"{self.name}: Found {len(found_links)} publisher link(s)",
+                )
+                await page.wait_for_timeout(1000)
+
+                # Visual: Try each publisher link
+                for i, found_link in enumerate(found_links, 1):
+                    publisher = found_link.get("publisher")
+                    link_element = found_link.get("link_element")
+
+                    await browser_logger.debug(
+                        page,
+                        f"{self.name}: Clicking {publisher} link ({i}/{len(found_links)})...",
+                    )
+
+                    result = await click_and_wait(
+                        link_element,
+                        f"Clicking {publisher} link for {doi[:20]}...",
+                    )
+
+                    if result.get("success"):
+                        final_url = result.get("final_url")
+                        await browser_logger.info(
+                            page,
+                            f"{self.name}: ✓ Landed at {final_url[:60]}",
+                        )
+                        await page.wait_for_timeout(2000)
+                        return final_url
+                    else:
+                        await browser_logger.info(
+                            page,
+                            f"{self.name}: ✗ {publisher} link failed, trying next...",
+                        )
+                        await page.wait_for_timeout(1000)
+
+                # All links failed
+                await browser_logger.info(
+                    page, f"{self.name}: ✗ All publisher links failed"
+                )
+                await page.wait_for_timeout(2000)
+                return None
+
+            except Exception as e:
+                if attempt < 2:
+                    wait_time = (attempt + 1) * 2
+                    logger.warning(
+                        f"OpenURL attempt {attempt + 1} failed: {e}, retrying in {wait_time}s"
+                    )
+                    await browser_logger.info(
+                        page,
+                        f"{self.name}: ✗ Attempt {attempt + 1} failed, retrying in {wait_time}s...",
+                    )
+                    await page.wait_for_timeout(wait_time * 1000)
+                    continue
+                else:
+                    logger.error(
+                        f"OpenURL resolution failed after 3 attempts: {e}"
+                    )
+                    await browser_logger.info(
+                        page,
+                        f"{self.name}: ✗ FAILED after 3 attempts: {str(e)[:80]}",
+                    )
+                    await page.wait_for_timeout(2000)
+                    await browser_logger.info(
+                        page, f"{self.name}: f{doi} - query not resolved"
+                    )
+                    return None
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    from scitex.scholar import ScholarAuthManager, ScholarBrowserManager
+
+    async def main():
+        # Initialize browser with authentication
+        auth_manager = ScholarAuthManager()
+        browser_manager = ScholarBrowserManager(
+            auth_manager=auth_manager,
+            browser_mode="stealth",
+            chrome_profile_name="system",
+        )
+
+        browser, context = (
+            await browser_manager.get_authenticated_browser_and_context_async()
+        )
+        page = await context.new_page()
+
+        # Test OpenURL resolver
+        resolver = OpenURLResolver()
+        doi = "10.1126/science.aao0702"
+
+        resolved_url = await resolver.resolve_doi(doi, page)
+
+        await browser.close()
+
+    asyncio.run(main())
+
+# python -m scitex.scholar.auth.gateway._OpenURLResolver
+
+# EOF

scitex/scholar/auth/gateway/__init__.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/auth/gateway/__init__.py
+# ----------------------------------------
+"""
+Authentication Gateway Module
+
+This module contains authentication gateway components that handle
+authentication mechanisms to access content behind paywalls and SSO systems.
+
+Components:
+- OpenURLResolver: Resolves DOIs through OpenURL resolvers
+- OpenURLLinkFinder: Finds publisher links on resolver pages
+- resolve_functions: Utility functions for URL resolution
+
+These were moved from url/helpers/resolvers as they are authentication
+mechanisms rather than URL discovery mechanisms.
+"""
+
+from ._OpenURLResolver import OpenURLResolver
+from ._OpenURLLinkFinder import OpenURLLinkFinder
+from ._resolve_functions import (
+    normalize_doi_as_http,
+    resolve_publisher_url_by_navigating_to_doi_page,
+    extract_doi_from_url,
+    resolve_openurl,
+)
+
+__all__ = [
+    "OpenURLResolver",
+    "OpenURLLinkFinder",
+    "normalize_doi_as_http",
+    "resolve_publisher_url_by_navigating_to_doi_page",
+    "extract_doi_from_url",
+    "resolve_openurl",
+]
+
+# EOF

scitex/scholar/auth/gateway/_resolve_functions.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-10-10 03:24:00 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/auth/gateway/_resolve_functions.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = (
+    "./src/scitex/scholar/auth/gateway/_resolve_functions.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+__FILE__ = __file__
+
+import re
+
+"""
+URL Resolver Functions
+
+Simple functions to resolve/convert between different URL types.
+No classes, just functions that do one thing well.
+"""
+
+import asyncio
+from typing import Optional
+
+from playwright.async_api import Page
+
+from scitex import logging
+from scitex.browser.debugging import browser_logger
+
+from ._OpenURLResolver import OpenURLResolver
+
+logger = logging.getLogger(__name__)
+
+
+async def resolve_publisher_url_by_navigating_to_doi_page(
+    doi: str,
+    page: Page,
+    func_name="resolve_publisher_url_by_navigating_to_doi_page",
+) -> Optional[str]:
+    """Resolve DOI to publisher URL by following redirects."""
+    url_doi = f"https://doi.org/{doi}" if not doi.startswith("http") else doi
+
+    await browser_logger.info(
+        page,
+        f"{func_name}: Finding Publisher URL by Navigating to DOI page...",
+    )
+
+    try:
+        logger.info(f"{func_name}: Resolving DOI: {doi}")
+        await page.goto(url_doi, wait_until="domcontentloaded", timeout=30000)
+        await asyncio.sleep(2)
+        url_publisher = page.url
+        logger.info(
+            f"{func_name}: Resolved Publisher URL by navigation: {doi}   ->   {url_publisher}"
+        )
+        return url_publisher
+    except Exception as e:
+        logger.error(
+            f"{func_name}: Publisher URL not resolved by navigating to {doi}: {e}"
+        )
+        from pathlib import Path
+
+        screenshot_dir = Path.home() / ".scitex/scholar/workspace/screenshots"
+        await browser_logger.info(
+            page,
+            f"{func_name}: {doi} - Publisher URL not resolved by navigating",
+            take_screenshot=True,
+            screenshot_category="Resolve",
+            screenshot_dir=screenshot_dir,
+        )
+        return None
+
+
+async def resolve_openurl(openurl_query: str, page: Page) -> Optional[str]:
+    """Resolve OpenURL to final authenticated URL."""
+    resolver = OpenURLResolver()
+    doi_match = re.search(r"doi=([^&]+)", openurl_query)
+    doi = doi_match.group(1) if doi_match else ""
+
+    return await resolver._resolve_query(openurl_query, page, doi)
+
+
+def normalize_doi_as_http(doi: str) -> str:
+    if doi.startswith("http"):
+        return doi
+    if doi.startswith("doi:"):
+        doi = doi[4:]
+    return f"https://doi.org/{doi}"
+
+
+def extract_doi_from_url(url: str) -> Optional[str]:
+    doi_pattern = r"10\.\d{4,}(?:\.\d+)*/[-._;()/:\w]+"
+    match = re.search(doi_pattern, url)
+    if match:
+        return match.group(0)
+    return None
+
+# EOF

scitex/scholar/auth/providers/BaseAuthenticator.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-08-07 13:58:10 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/auth/library/_BaseAuthenticator.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = __file__
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+"""
+Abstract base class for authenticators.
+
+This module provides the base interface that all authenticators
+(OpenAthens, Lean Library, etc.) must implement.
+"""
+
+"""Imports"""
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from scitex import logging
+
+from scitex.errors import AuthenticationError
+
+"""Logger"""
+logger = logging.getLogger(__name__)
+
+"""Classes"""
+
+
+class BaseAuthenticator(ABC):
+    """
+    Abstract base class for authentication providers.
+
+    All authentication providers (OpenAthens, EZProxy, Shibboleth, etc.)
+    should inherit from this class and implement the required methods.
+    """
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize authentication provider.
+
+        Args:
+            config: Authenticator-specific configuration
+        """
+        self.config = config or {}
+        self.name = self.__class__.__name__.replace("Authentication", "")
+
+    @abstractmethod
+    async def is_authenticate_async(self, verify_live: bool = False) -> bool:
+        """
+        Check if currently authenticate_async.
+
+        Args:
+            verify_live: If True, verify with actual request instead of just checking session
+
+        Returns:
+            True if authenticate_async, False otherwise
+        """
+        pass
+
+    @abstractmethod
+    async def authenticate_async(self, **kwargs) -> dict:
+        """
+        Perform authentication and return session data.
+
+        Args:
+            **kwargs: Authenticator-specific authentication parameters
+
+        Returns:
+            Dictionary containing session data (e.g., cookies, tokens)
+        """
+        pass
+
+    @abstractmethod
+    async def get_auth_headers_async(self) -> Dict[str, str]:
+        """
+        Get authentication headers for requests.
+
+        Returns:
+            Dictionary of headers to include in authenticate_async requests
+        """
+        pass
+
+    @abstractmethod
+    async def get_auth_cookies_async(self) -> List[Dict[str, Any]]:
+        """
+        Get authentication cookies for requests.
+
+        Returns:
+            List of cookie dictionaries
+        """
+        pass
+
+    @abstractmethod
+    async def logout_async(self) -> None:
+        """Log out and clear authentication state."""
+        pass
+
+    @abstractmethod
+    async def get_session_info_async(self) -> Dict[str, Any]:
+        """
+        Get information about current session.
+
+        Returns:
+            Dictionary containing session details (expiry, username, etc.)
+        """
+        pass
+
+    def __str__(self) -> str:
+        """String representation of provider."""
+        return f"{self.name}Authenticator"
+
+    def __repr__(self) -> str:
+        """Detailed representation of provider."""
+        return f"{self.name}Authenticator(config={self.config})>"
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    async def main():
+        """Example usage of BaseAuthenticator (through a concrete implementation)."""
+        # This is an abstract class, so we'll demonstrate with OpenAthensAuthenticator
+        from scitex.scholar.auth import OpenAthensAuthenticator
+
+        # Initialize authenticator
+        auth = OpenAthensAuthenticator(
+            institution="University Example",
+            username="your_username",
+            password="your_password",
+            browser_backend="local",  # or "zenrows"
+        )
+
+        try:
+            # Check if already authenticate_async
+            is_auth = await auth.is_authenticate_async()
+            print(f"Initially authenticate_async: {is_auth}")
+
+            if not is_auth:
+                # Perform authentication
+                print("Authenticating...")
+                success = await auth.authenticate_async()
+                print(f"Authentication successful: {success}")
+
+            # Get authentication headers/cookies for requests
+            headers = await auth.get_auth_headers_async()
+            cookies = await auth.get_auth_cookies_async()
+            print(f"Auth headers: {list(headers.keys())}")
+            print(f"Auth cookies: {list(cookies.keys())}")
+
+            # Get session info
+            session_info = await auth.get_session_info_async()
+            print(f"Session info: {session_info}")
+
+        finally:
+            # Always cleanup
+            await auth.logout_async()
+            print("Logged out successfully")
+
+    # Run the example
+    asyncio.run(main())
+
+# EOF