scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/.legacy/metadata/urls/_resolver.py

@@ -1,287 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-15 16:14:54 (ywatanabe)"
-# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/metadata/urls/_resolver.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-"""
-URL Resolver Functions
-
-Simple functions to resolve/convert between different URL types.
-No classes, just functions that do one thing well.
-"""
-
-import asyncio
-from typing import Dict, Optional
-from urllib.parse import quote
-
-from playwright.async_api import BrowserContext, Page
-
-from scitex import logging
-
-logger = logging.getLogger(__name__)
-
-
-async def doi_to_url_publisher(
-    doi: str, context: BrowserContext = None
-) -> Optional[str]:
-    """
-    Resolve DOI to publisher URL by following redirects.
-
-    Args:
-        doi: DOI string (with or without https://doi.org/ prefix)
-        context: Browser context for authenticated resolution
-
-    Returns:
-        Publisher URL after following DOI redirect
-    """
-    # Ensure DOI URL format
-    if not doi.startswith("http"):
-        url_doi = f"https://doi.org/{doi}"
-    else:
-        url_doi = doi
-
-    if context:
-        # Use browser to follow redirects (handles authentication)
-        page = await context.new_page()
-        try:
-            logger.info(f"Resolving DOI: {doi}")
-            await page.goto(url_doi, wait_until="domcontentloaded", timeout=30000)
-            await asyncio.sleep(2)  # Let redirects settle
-
-            url_publisher = page.url
-            logger.info(f"Resolved to: {url_publisher}")
-            return url_publisher
-
-        except Exception as e:
-            logger.error(f"Failed to resolve DOI: {e}")
-            return None
-        finally:
-            await page.close()
-    else:
-        # Simple HTTP redirect following (no authentication)
-        import httpx
-
-        try:
-            async with httpx.AsyncClient(follow_redirects=True) as client:
-                response = await client.get(url_doi)
-                return str(response.url)
-        except Exception as e:
-            logger.error(f"Failed to resolve DOI: {e}")
-            return None
-
-
-def generate_openurl_query(
-    metadata: Dict,
-    openurl_resolver_url: str,
-) -> Optional[str]:
-    """
-    Generate OpenURL query from metadata.
-
-    Args:
-        metadata: Paper metadata dict
-        openurl_resolver_url: Institution's OpenURL resolver base URL
-
-    Returns:
-        OpenURL query string
-    """
-    params = []
-
-    # Add source identifier
-    params.append("sid=scitex")
-
-    # Add DOI if available
-    if metadata.get("doi"):
-        params.append(f"doi={metadata['doi']}")
-
-    # Add title
-    if metadata.get("title"):
-        params.append(f"atitle={quote(metadata['title'][:200])}")
-
-    # Add journal
-    if metadata.get("journal"):
-        params.append(f"jtitle={quote(metadata['journal'])}")
-
-    # Add year
-    if metadata.get("year"):
-        params.append(f"date={metadata['year']}")
-
-    # Add volume/issue/pages
-    if metadata.get("volume"):
-        params.append(f"volume={metadata['volume']}")
-    if metadata.get("issue"):
-        params.append(f"issue={metadata['issue']}")
-    if metadata.get("pages"):
-        params.append(f"pages={metadata['pages']}")
-
-    # Add first author
-    if metadata.get("authors"):
-        authors = metadata["authors"]
-        if isinstance(authors, list) and authors:
-            first_author = authors[0]
-            if isinstance(first_author, str):
-                # Parse "Last, First" or "First Last" format
-                if "," in first_author:
-                    last, first = first_author.split(",", 1)
-                    params.append(f"aulast={quote(last.strip())}")
-                    params.append(f"aufirst={quote(first.strip())}")
-                else:
-                    parts = first_author.strip().split()
-                    if parts:
-                        params.append(f"aulast={quote(parts[-1])}")
-                        if len(parts) > 1:
-                            params.append(f"aufirst={quote(parts[0])}")
-            elif isinstance(first_author, dict):
-                if first_author.get("lastName"):
-                    params.append(f"aulast={quote(first_author['lastName'])}")
-                if first_author.get("firstName"):
-                    params.append(f"aufirst={quote(first_author['firstName'])}")
-
-    # Add ISSN if available
-    if metadata.get("issn"):
-        params.append(f"issn={metadata['issn']}")
-
-    if params:
-        return f"{openurl_resolver_url}?{'&'.join(params)}"
-
-    return None
-
-
-async def resolve_openurl(openurl_query: str, context: BrowserContext) -> Optional[str]:
-    """
-    Resolve OpenURL query to final authenticated URL.
-
-    Args:
-        openurl_query: OpenURL query string
-        context: Authenticated browser context
-
-    Returns:
-        Final URL after OpenURL resolution and authentication
-    """
-    if not context:
-        logger.error("Browser context required for OpenURL resolution")
-        return None
-
-    page = await context.new_page()
-    try:
-        logger.info("Resolving OpenURL...")
-        await page.goto(openurl_query, wait_until="networkidle", timeout=30000)
-
-        # Wait for any redirects/authentication
-        await asyncio.sleep(3)
-
-        # Check if we need to click a "Go" or "Access" button
-        go_buttons = [
-            'button:has-text("Go")',
-            'button:has-text("Access")',
-            'a:has-text("Full Text")',
-            'a:has-text("View Article")',
-        ]
-
-        for selector in go_buttons:
-            try:
-                button = page.locator(selector).first
-                if await button.is_visible():
-                    await button.click()
-                    await asyncio.sleep(3)
-                    break
-            except:
-                pass
-
-        resolved_url = page.url
-        logger.info(f"OpenURL resolved to: {resolved_url}")
-        return resolved_url
-
-    except Exception as e:
-        logger.error(f"OpenURL resolution failed: {e}")
-        return None
-    finally:
-        await page.close()
-
-
-def build_url_doi(doi: str) -> str:
-    """
-    Build standard DOI URL from DOI string.
-
-    Args:
-        doi: DOI string (with or without prefix)
-
-    Returns:
-        Full DOI URL
-    """
-    if doi.startswith("http"):
-        return doi
-    if doi.startswith("doi:"):
-        doi = doi[4:]
-    return f"https://doi.org/{doi}"
-
-
-def extract_doi_from_url(url: str) -> Optional[str]:
-    """
-    Extract DOI from a URL if present.
-
-    Args:
-        url: Any URL that might contain a DOI
-
-    Returns:
-        DOI string if found
-    """
-    import re
-
-    # Pattern for DOI
-    doi_pattern = r"10\.\d{4,}(?:\.\d+)*/[-._;()/:\w]+"
-
-    match = re.search(doi_pattern, url)
-    if match:
-        return match.group(0)
-
-    return None
-
-
-# Convenience function to resolve all URL types
-async def resolve_all_urls(
-    metadata: Dict, openurl_resolver_url: str, context: BrowserContext = None
-) -> Dict[str, any]:
-    """
-    Resolve all URL types from metadata.
-
-    Args:
-        metadata: Paper metadata
-        context: Browser context for authenticated resolution
-
-    Returns:
-        Dict with all resolved URLs
-    """
-    urls = {}
-
-    # Build DOI URL
-    if metadata.get("doi"):
-        urls["url_doi"] = build_url_doi(metadata["doi"])
-
-        # Resolve to publisher
-        if context:
-            url_publisher = await doi_to_url_publisher(metadata["doi"], context)
-            if url_publisher:
-                urls["url_publisher"] = url_publisher
-
-    # Generate OpenURL query
-    openurl_query = generate_openurl_query(metadata, openurl_resolver_url)
-    if openurl_query:
-        urls["url_openurl_query"] = openurl_query
-
-        # Resolve OpenURL if context available
-        if context:
-            resolved = await resolve_openurl(openurl_query, context)
-            if resolved:
-                urls["url_openurl_resolved"] = resolved
-
-    return urls
-
-
-# EOF

scitex/scholar/.legacy/metadata/urls/docs/CORE_URL_TYPES.md

@@ -1,187 +0,0 @@
-<!-- ---
-!-- Timestamp: 2025-08-08 08:37:40
-!-- Author: ywatanabe
-!-- File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/metadata/urls/CORE_URL_TYPES.md
-!-- --- -->
-
-# Core URL Types for Scholar Metadata
-
-## Essential URLs in Resolution Order
-
-## Usage
-
-``` python
-from scitex.scholar.metadata.urls import URLHandler
-from scitex.scholar.browser import BrowserManager
-from scitex.scholar.auth import AuthenticationManager
-
-browser_manager = BrowserManager(
-    chrome_profile_name="system",
-    browser_mode="interactive",
-    auth_manager=AuthenticationManager(),
-)
-
-browser, context = (
-    await browser_manager.get_authenticated_browser_and_context_async()
-)
-
-url_handler = URLHandler(context)
-urls = await url_handler.get_all_urls(doi="10.1523/jneurosci.2929-12.2012")
-print(urls)
-# {'url_doi': 'https://doi.org/10.1523/jneurosci.2929-12.2012',
-#  'url_publisher': 'https://www.jneurosci.org/content/32/44/15467',
-#  'url_pdf': ['https://www.jneurosci.org/content/jneuro/32/44/15467.full.pdf',
-#   'https://www.jneurosci.org/content/32/44.toc.pdf',
-#   'https://www.jneurosci.org/content/jneuro/32/44/local/advertising.pdf',
-#   'https://www.jneurosci.org/content/jneuro/32/44/local/ed-board.pdf',
-#   'https://www.jneurosci.org/content/jneuro/32/44/15467.full-text.pdf',
-#   'https://www.jneurosci.org/content/32/44/15467.full.pdf']}
-
-## Download article PDF
-from playwright.async_api import BrowserContext
-from pathlib import Path
-from scitex import logging
-
-logger = logging.getLogger(__name__)
-
-
-async def download_pdf_direct(context: BrowserContext, pdf_url: str, output_path: Path):
-    """
-    Download PDF using request context (bypasses Chrome PDF viewer).
-    
-    This sends HTTP requests with the browser's cookies/auth,
-    but doesn't render the response in the browser.
-    """
-    response = await context.request.get(pdf_url)
-    
-    if response.ok and response.headers.get('content-type', '').startswith('application/pdf'):
-        content = await response.body()
-        
-        with open(output_path, 'wb') as f:
-            f.write(content)
-        
-        return True
-    return False
-
-for ii_pdf, url_pdf in enumerate(urls["url_pdf"]):
-    success = await download_pdf_direct(context, url_pdf, f"/tmp/tmp_{ii_pdf}.pdf")
-    if success:
-        logger.success(f"Downloaded: {url_pdf}")
-    else:
-        logger.fail(f"Not downloaded {url_pdf}")
-
-
-```
-
-
-
-### 1. **DOI URL** (`url_doi`)
-- **What**: The DOI resolver URL
-- **Example**: `https://doi.org/10.1038/s41593-025-01990-7`
-- **Purpose**: Permanent identifier that redirects to current publisher location
-- **Source**: Extracted from metadata or constructed from DOI
-
-### 2. **Publisher URL** (`url_publisher`) 
-- **What**: The actual publisher's article page (after DOI redirect)
-- **Example**: `https://www.nature.com/articles/s41593-025-01990-7`
-- **Purpose**: The publisher's canonical article page
-- **Source**: Result of following DOI redirect
-
-### 3. **OpenURL Query URL** (`url_openurl_query`)
-- **What**: The OpenURL query with metadata parameters
-- **Example**: `https://unimelb.hosted.exlibrisgroup.com/openurl/61UNIMELB/61UNIMELB_INST?sid=google&auinit=S&aulast=Elyounssi&atitle=Addressing%20artifactual%20bias&jtitle=Nature%20Neuroscience&doi=10.1038/s41593-025-01990-7`
-- **Purpose**: Query URL containing bibliographic metadata for resolver
-- **Source**: Generated from article metadata using OpenURL 1.0 standard
-
-### 4. **OpenURL Resolved URL** (`url_openurl_resolved`)
-- **What**: The final URL after OpenURL resolver processes the query
-- **Example**: `https://www-nature-com.eu1.proxy.openathens.net/articles/s41593-025-01990-7`
-- **Purpose**: The authenticated publisher URL via institutional proxy
-- **Source**: Result of OpenURL resolver's "Go" button or automatic redirect
-
-### 5. **Final PDF URL** (`url_final_pdf`)
-- **What**: The actual working PDF download URL
-- **Example**: `https://www-nature-com.eu1.proxy.openathens.net/articles/s41593-025-01990-7.pdf`
-- **Purpose**: Direct link to download the PDF (with authentication)
-- **Source**: Discovered through various methods (Zotero, patterns, page scraping)
-
-## Simplified Metadata Structure
-
-```json
-{
-  "scitex_id": "F99329E1",
-  "title": "Addressing artifactual bias in large...",
-  
-  "doi": "10.1038/s41593-025-01990-7",
-  "url_doi": "https://doi.org/10.1038/s41593-025-01990-7",
-  "url_doi_source": "CrossRef",
-  
-  "url_publisher": "https://www.nature.com/articles/s41593-025-01990-7",
-  "url_publisher_source": "DOI_resolution",
-  
-  "url_openurl_query": "https://unimelb.hosted.exlibrisgroup.com/openurl/61UNIMELB/61UNIMELB_INST?sid=scitex&doi=10.1038/s41593-025-01990-7",
-  "url_openurl_query_source": "OpenURLGenerator",
-  
-  "url_openurl_resolved": "https://www-nature-com.eu1.proxy.openathens.net/articles/s41593-025-01990-7",
-  "url_openurl_resolved_source": "OpenURLResolver",
-  
-  "url_final_pdf": "https://www-nature-com.eu1.proxy.openathens.net/articles/s41593-025-01990-7.pdf",
-  "url_final_pdf_source": "DirectRequest",
-  
-  "pdf_urls": [
-    "https://www.nature.com/articles/s41593-025-01990-7.pdf",
-    "https://www-nature-com.eu1.proxy.openathens.net/articles/s41593-025-01990-7.pdf"
-  ],
-  "pdf_urls_source": "MultipleStrategies",
-  
-  "pdf_downloaded": false,
-  "pdf_download_attempts": [],
-  
-  "urls_updated_at": "2025-08-08T00:00:00Z"
-}
-```
-
-## Resolution Pipeline
-
-```
-1. DOI → url_doi
-   ↓
-2. Resolve DOI → url_publisher
-   ↓
-3. Build OpenURL query → url_openurl_query
-   ↓
-4. Submit to resolver → url_openurl_resolved (authenticated)
-   ↓
-5. Find PDF URLs (multiple strategies) → pdf_urls[]
-   ↓
-6. Test & download PDFs → url_final_pdf
-```
-
-## OpenURL Query Parameters
-
-The `url_openurl_query` typically includes:
-- `sid`: Source identifier (e.g., "scitex", "google")
-- `doi`: Digital Object Identifier
-- `atitle`: Article title
-- `jtitle`: Journal title
-- `aulast`: Author last name
-- `aufirst`: Author first name
-- `date`: Publication date
-- `volume`: Volume number
-- `issue`: Issue number
-- `spage`: Start page
-- `epage`: End page
-- `issn`: Journal ISSN
-
-## Key Differences
-
-- **`url_openurl_query`**: The constructed query URL with metadata parameters - what you send TO the resolver
-- **`url_openurl_resolved`**: The authenticated URL you get BACK from the resolver after it processes your query
-
-This distinction is important because:
-1. The query URL is what we generate from metadata
-2. The resolved URL is what we get after authentication/proxy routing
-3. Sometimes the resolver shows an intermediate page with multiple options
-4. The resolved URL may vary based on institutional subscriptions
-
-<!-- EOF -->

scitex/scholar/.legacy/metadata/urls/docs/URL_SCHEMA.md

@@ -1,223 +0,0 @@
-# URL Schema for Scholar Metadata
-
-## Overview
-Each paper entry in the Scholar library needs to track multiple types of URLs for different purposes. This document defines the comprehensive URL schema.
-
-## URL Types and Their Purposes
-
-### 1. Identification URLs
-- **`url_doi`**: The DOI resolver URL (e.g., `https://doi.org/10.1038/s41593-025-01990-7`)
-  - Purpose: Permanent identifier, always redirects to current publisher location
-  - Source: DOI resolver services (CrossRef, DataCite)
-  
-- **`canonical_url`**: The canonical/preferred URL for the article
-  - Purpose: The "official" URL that should be cited
-  - Source: Publisher metadata or DOI resolution
-
-### 2. Access URLs
-- **`article_url`**: The article landing page URL
-  - Purpose: Human-readable page with abstract, metadata, and download options
-  - Example: `https://www.nature.com/articles/s41593-025-01990-7`
-  - Source: Original input, DOI resolution, or search results
-
-- **`url_publisher`**: The final publisher URL after authentication
-  - Purpose: The actual URL after SSO/proxy authentication
-  - Example: `https://www-nature-com.eu1.proxy.openathens.net/articles/...`
-  - Source: Browser after authentication
-
-- **`openurl`**: The library OpenURL resolver link
-  - Purpose: Institution-specific link that handles authentication
-  - Example: `https://unimelb.hosted.exlibrisgroup.com/openurl/61UNIMELB/...`
-  - Source: OpenURL resolver
-
-### 3. Download URLs
-- **`pdf_urls`**: Array of direct PDF download URLs
-  - Purpose: Direct links to PDF files (may require authentication)
-  - Examples: 
-    - `https://www.nature.com/articles/s41593-025-01990-7.pdf`
-    - `https://www.nature.com/articles/s41593-025-01990-7.pdf?proof=t`
-  - Source: Zotero translators, page scraping, publisher patterns
-
-- **`pdf_viewer_url`**: URL of the PDF viewer page
-  - Purpose: Browser-based PDF viewer (not direct download)
-  - Example: `https://www.nature.com/articles/s41593-025-01990-7.pdf#view`
-  - Source: Browser navigation
-
-### 4. Alternative Access URLs
-- **`preprint_url`**: Link to preprint version (if available)
-  - Purpose: Free access to early version
-  - Examples: 
-    - `https://arxiv.org/abs/2303.12345`
-    - `https://www.biorxiv.org/content/10.1101/...`
-  - Source: DOI metadata, CrossRef, arXiv API
-
-- **`author_manuscript_url`**: Link to author's accepted manuscript
-  - Purpose: Open access version from institutional repository
-  - Example: `https://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567/`
-  - Source: PubMed Central, institutional repositories
-
-- **`alternate_urls`**: Array of alternative access points
-  - Purpose: Mirror sites, archived versions, etc.
-  - Source: Various databases
-
-### 5. Supplementary Material URLs
-- **`supplementary_urls`**: Array of supplementary material links
-  - Structure:
-    ```json
-    {
-      "url": "https://...",
-      "description": "Supplementary Tables",
-      "type": "xlsx",
-      "size": 1234567
-    }
-    ```
-  - Purpose: Additional data, figures, videos, code
-  - Source: Article page, Zotero translators
-
-### 6. Related URLs
-- **`dataset_urls`**: Links to associated datasets
-  - Purpose: Research data repositories
-  - Examples:
-    - `https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE123456`
-    - `https://zenodo.org/record/1234567`
-  - Source: Article metadata, data availability statements
-
-- **`code_repository_url`**: Link to associated code
-  - Purpose: Reproducibility
-  - Example: `https://github.com/author/project`
-  - Source: Article metadata, code availability statements
-
-### 7. Metadata/API URLs
-- **`crossref_api_url`**: CrossRef API endpoint for this article
-  - Purpose: Programmatic metadata access
-  - Example: `https://api.crossref.org/works/10.1038/s41593-025-01990-7`
-  - Source: Generated from DOI
-
-- **`semantic_scholar_url`**: Semantic Scholar page
-  - Purpose: Citation network, related papers
-  - Example: `https://www.semanticscholar.org/paper/...`
-  - Source: Semantic Scholar API
-
-- **`google_scholar_url`**: Google Scholar page
-  - Purpose: Citations, versions
-  - Source: Google Scholar search
-
-### 8. Archive/Preservation URLs
-- **`wayback_url`**: Internet Archive snapshot
-  - Purpose: Permanent archive in case of link rot
-  - Example: `https://web.archive.org/web/20250101/...`
-  - Source: Wayback Machine API
-
-- **`perma_cc_url`**: Perma.cc permanent link
-  - Purpose: Legal/scholarly citation preservation
-  - Source: Perma.cc service
-
-## Metadata Structure Example
-
-```json
-{
-  "scitex_id": "F99329E1",
-  "title": "Example Paper Title",
-  
-  "urls": {
-    "identification": {
-      "url_doi": "https://doi.org/10.1038/...",
-      "url_doi_source": "CrossRef",
-      "canonical_url": "https://www.nature.com/articles/...",
-      "canonical_url_source": "DOI_resolution"
-    },
-    
-    "access": {
-      "article_url": "https://www.nature.com/articles/...",
-      "article_url_source": "user_input",
-      "url_publisher": "https://www-nature-com.proxy.openathens.net/...",
-      "url_publisher_source": "browser_authentication",
-      "openurl": "https://unimelb.hosted.exlibrisgroup.com/...",
-      "openurl_source": "OpenURLResolver"
-    },
-    
-    "download": {
-      "pdf_urls": [
-        {
-          "url": "https://www.nature.com/articles/...pdf",
-          "source": "zotero_translator",
-          "verified": true,
-          "last_checked": "2025-08-08T00:00:00Z"
-        }
-      ],
-      "pdf_viewer_url": "https://www.nature.com/articles/...pdf#view",
-      "pdf_viewer_url_source": "browser_navigation"
-    },
-    
-    "alternative": {
-      "preprint_url": "https://arxiv.org/abs/...",
-      "preprint_url_source": "CrossRef",
-      "author_manuscript_url": "https://www.ncbi.nlm.nih.gov/pmc/...",
-      "author_manuscript_url_source": "PubMed"
-    },
-    
-    "supplementary": [
-      {
-        "url": "https://...",
-        "description": "Supplementary Data",
-        "type": "xlsx",
-        "size": 1234567,
-        "source": "article_page"
-      }
-    ],
-    
-    "related": {
-      "dataset_urls": ["https://..."],
-      "code_repository_url": "https://github.com/..."
-    },
-    
-    "metadata_api": {
-      "crossref_api_url": "https://api.crossref.org/works/...",
-      "semantic_scholar_url": "https://www.semanticscholar.org/paper/..."
-    },
-    
-    "archive": {
-      "wayback_url": "https://web.archive.org/web/...",
-      "perma_cc_url": "https://perma.cc/..."
-    }
-  },
-  
-  "url_resolution_status": {
-    "doi_resolved": true,
-    "openurl_resolved": true,
-    "pdf_urls_found": true,
-    "pdf_downloaded": false,
-    "last_updated": "2025-08-08T00:00:00Z"
-  }
-}
-```
-
-## URL Resolution Pipeline
-
-1. **Input Stage**: User provides article URL, DOI, or title
-2. **DOI Resolution**: Resolve DOI to get canonical URL
-3. **Metadata Extraction**: Use Zotero translators to get article metadata and PDF URLs
-4. **OpenURL Resolution**: Generate OpenURL for institutional access
-5. **PDF URL Discovery**: Find all possible PDF URLs using multiple strategies
-6. **Authentication**: Navigate through SSO/proxy to get authenticated URLs
-7. **Download**: Attempt PDF download using discovered URLs
-8. **Verification**: Verify downloaded PDF is valid and complete
-9. **Archive**: Store successful URLs and download status
-
-## Priority Order for PDF Download
-
-1. Direct PDF URLs from Zotero translators
-2. Publisher-specific PDF patterns
-3. OpenURL resolved PDF link
-4. Preprint/OA versions
-5. Author manuscript versions
-6. Print-to-PDF as last resort
-
-## URL Validation Rules
-
-- All URLs should be validated before storage
-- HTTPS preferred over HTTP
-- Remove tracking parameters
-- Normalize URLs (trailing slashes, fragments)
-- Check for redirects and store final URLs
-- Timestamp all URL discoveries