scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/pdf_download/_cli.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# Timestamp: "2026-01-22 (ywatanabe)"
+# File: src/scitex/scholar/pdf_download/_cli.py
+"""CLI entry point for ScholarPDFDownloader."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+from pathlib import Path
+
+from scitex import logging
+
+logger = logging.getLogger(__name__)
+
+__FILE__ = __file__
+
+
+async def main_async(args):
+    """Example usage showing decoupled URL resolution and downloading."""
+    from scitex.scholar import (
+        ScholarAuthManager,
+        ScholarBrowserManager,
+        ScholarURLFinder,
+    )
+    from scitex.scholar.auth import AuthenticationGateway
+    from scitex.scholar.pdf_download import ScholarPDFDownloader
+
+    # Authenticated Browser and Context
+    auth_manager = ScholarAuthManager()
+    browser_manager = ScholarBrowserManager(
+        chrome_profile_name="system",
+        browser_mode=args.browser_mode,
+        auth_manager=auth_manager,
+        use_zenrows_proxy=False,
+    )
+    (
+        browser,
+        context,
+    ) = await browser_manager.get_authenticated_browser_and_context_async()
+
+    # Authentication Gateway
+    auth_gateway = AuthenticationGateway(
+        auth_manager=auth_manager,
+        browser_manager=browser_manager,
+    )
+    url_context = await auth_gateway.prepare_context_async(
+        doi=args.doi, context=context
+    )
+
+    # URL Resolution
+    url_finder = ScholarURLFinder(context)
+    resolved_url = url_context.url if url_context else None
+    if resolved_url:
+        logger.info(f"Using resolved URL from auth_gateway: {resolved_url}")
+        urls = await url_finder.find_pdf_urls(resolved_url)
+    else:
+        logger.info(f"No resolved URL, using DOI: {args.doi}")
+        urls = await url_finder.find_pdf_urls(args.doi)
+
+    # Extract URL strings from list of dicts
+    pdf_urls = []
+    for entry in urls:
+        if isinstance(entry, dict):
+            pdf_urls.append(entry.get("url"))
+        elif isinstance(entry, str):
+            pdf_urls.append(entry)
+
+    if not pdf_urls:
+        logger.error(f"No PDF URLs found for DOI: {args.doi}")
+        return
+
+    logger.info(f"Found {len(pdf_urls)} PDF URL(s) for DOI: {args.doi}")
+
+    # PDF Download
+    pdf_downloader = ScholarPDFDownloader(context)
+    if len(pdf_urls) == 1:
+        await pdf_downloader.download_from_url(pdf_urls[0], args.output)
+    else:
+        output_dir = Path(args.output).parent
+        await pdf_downloader.download_from_urls(
+            pdf_urls, output_dir=output_dir, max_concurrent=3
+        )
+
+
+def main(args):
+    asyncio.run(main_async(args))
+    return 0
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Download a PDF using DOI with authentication support"
+    )
+    parser.add_argument(
+        "--doi",
+        type=str,
+        required=True,
+        help="DOI of the paper (e.g., 10.1088/1741-2552/aaf92e)",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="~/.scitex/scholar/library/downloads/downloaded_paper.pdf",
+        help="Output path for the PDF",
+    )
+    parser.add_argument(
+        "--browser-mode",
+        type=str,
+        choices=["stealth", "interactive"],
+        default="stealth",
+        help="Browser mode (default: stealth)",
+    )
+    return parser.parse_args()
+
+
+def run_main() -> None:
+    """Initialize scitex framework, run main function, and cleanup."""
+    global CONFIG, CC, sys, plt, rng
+
+    import sys
+
+    import matplotlib.pyplot as plt
+
+    import scitex as stx
+
+    args = parse_args()
+
+    CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
+        sys,
+        plt,
+        args=args,
+        file=__FILE__,
+        sdir_suffix=None,
+        verbose=False,
+        agg=True,
+    )
+
+    exit_status = main(args)
+
+    stx.session.close(
+        CONFIG,
+        verbose=False,
+        notify=False,
+        message="",
+        exit_status=exit_status,
+    )
+
+
+if __name__ == "__main__":
+    run_main()
+
+# EOF

scitex/scholar/pdf_download/strategies/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """PDF Download Strategies
 
 This module contains different strategies for downloading PDFs from academic publishers.
@@ -9,21 +8,24 @@ Each strategy is tried in sequence until one succeeds.
 # Download strategies
 from .chrome_pdf_viewer import try_download_chrome_pdf_viewer_async
 from .direct_download import try_download_direct_async
-from .response_body import try_download_response_body_async
-from .manual_download_fallback import try_download_manual_async
-from .open_access_download import (
-    try_download_open_access_async,
-    try_download_open_access_sync,
+from .manual_download_fallback import (
+    handle_manual_download_on_page_async,
+    try_download_manual_async,
 )
 
 # Manual download utilities
 from .manual_download_utils import (
     DownloadMonitorAndSync,
     FlexibleFilenameGenerator,
-    show_stop_automation_button_async,
-    show_manual_download_button_async,
     complete_manual_download_workflow_async,
+    show_manual_download_button_async,
+    show_stop_automation_button_async,
+)
+from .open_access_download import (
+    try_download_open_access_async,
+    try_download_open_access_sync,
 )
+from .response_body import try_download_response_body_async
 
 __all__ = [
     # Download strategies
@@ -33,6 +35,7 @@ __all__ = [
     "try_download_manual_async",
     "try_download_open_access_async",
     "try_download_open_access_sync",
+    "handle_manual_download_on_page_async",
     # Manual download utilities
     "DownloadMonitorAndSync",
     "FlexibleFilenameGenerator",

scitex/scholar/pdf_download/strategies/chrome_pdf_viewer.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 # Timestamp: "2025-10-13 08:18:35 (ywatanabe)"
 # File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/pdf_download/strategies/chrome_pdf_viewer.py
 # ----------------------------------------
 from __future__ import annotations
+
 import os
 
 __FILE__ = "./src/scitex/scholar/pdf_download/strategies/chrome_pdf_viewer.py"
@@ -230,12 +230,12 @@ async def try_download_chrome_pdf_viewer_async(
         new_files = files_after - files_before
         download_duration = time.time() - download_start_time
 
-        logger.info(f"{func_name}: Checking download result...")
-        logger.info(f"{func_name}:  is_downloaded (Playwright): {is_downloaded}")
-        logger.info(f"{func_name}:  output_path: {output_path}")
-        logger.info(f"{func_name}:  Files before: {len(files_before)}")
-        logger.info(f"{func_name}:  Files after: {len(files_after)}")
-        logger.info(f"{func_name}:  New files: {len(new_files)}")
+        logger.debug(f"{func_name}: Checking download result...")
+        logger.debug(f"{func_name}:  is_downloaded (Playwright): {is_downloaded}")
+        logger.debug(f"{func_name}:  output_path: {output_path}")
+        logger.debug(f"{func_name}:  Files before: {len(files_before)}")
+        logger.debug(f"{func_name}:  Files after: {len(files_after)}")
+        logger.debug(f"{func_name}:  New files: {len(new_files)}")
 
         if new_files:
             # Found new file(s) in download directory
@@ -243,10 +243,10 @@ async def try_download_chrome_pdf_viewer_async(
             file_size = downloaded_file.stat().st_size
             file_size_mb = file_size / (1024 * 1024)
 
-            logger.info(f"{func_name}: Found downloaded file: {downloaded_file.name}")
-            logger.info(f"{func_name}:  Size: {file_size_mb:.2f} MB")
-            logger.info(f"{func_name}:  Duration: {download_duration:.1f}s")
-            logger.info(f"{func_name}:  Location: {downloaded_file}")
+            logger.debug(f"{func_name}: Found downloaded file: {downloaded_file.name}")
+            logger.debug(f"{func_name}:  Size: {file_size_mb:.2f} MB")
+            logger.debug(f"{func_name}:  Duration: {download_duration:.1f}s")
+            logger.debug(f"{func_name}:  Location: {downloaded_file}")
 
             if file_size > 1000:  # At least 1KB
                 # Rename to desired output filename

scitex/scholar/pdf_download/strategies/manual_download_fallback.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 # Timestamp: "2025-10-13 08:00:08 (ywatanabe)"
 # File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/pdf_download/strategies/manual_download_fallback.py
 # ----------------------------------------
 from __future__ import annotations
+
 import os
 
 __FILE__ = "./src/scitex/scholar/pdf_download/strategies/manual_download_fallback.py"
@@ -21,7 +21,6 @@ from scitex.scholar import ScholarConfig
 from scitex.scholar.browser import browser_logger
 from scitex.scholar.pdf_download.strategies.manual_download_utils import (
     DownloadMonitorAndSync,
-    complete_manual_download_workflow_async,
 )
 
 logger = logging.getLogger(__name__)
@@ -51,7 +50,8 @@ async def try_download_manual_async(
         config: Scholar configuration
         doi: Optional DOI for filename generation
 
-    Returns:
+    Returns
+    -------
         Path to downloaded file, or None if failed
     """
     config = config or ScholarConfig()
@@ -164,4 +164,81 @@ async def try_download_manual_async(
         return None
 
 
+async def handle_manual_download_on_page_async(
+    page,
+    pdf_url: str,
+    output_path: Path,
+    func_name: str = "handle_manual_download",
+    config: ScholarConfig = None,
+    doi: Optional[str] = None,
+) -> Optional[Path]:
+    """Handle manual download on an already-open page.
+
+    Unlike try_download_manual_async, this uses an existing page
+    (e.g., from the stop automation button workflow).
+
+    Args:
+        page: Already-open Playwright page
+        pdf_url: URL of the PDF
+        output_path: Target output path
+        config: Scholar configuration
+        doi: Optional DOI for metadata
+
+    Returns
+    -------
+        Path to downloaded file, or None if failed
+    """
+    config = config or ScholarConfig()
+    downloads_dir = config.get_library_downloads_dir()
+
+    # Extract DOI from URL if not provided
+    if not doi and "doi.org/" in pdf_url:
+        doi = pdf_url.split("doi.org/")[-1].split("?")[0].split("#")[0]
+
+    await browser_logger.info(page, f"{func_name}: Manual download mode activated")
+    await browser_logger.info(
+        page, f"{func_name}: Please download the PDF manually from this page"
+    )
+
+    # Monitor for download
+    monitor = DownloadMonitorAndSync(downloads_dir, downloads_dir)
+
+    def log_progress(msg: str):
+        logger.info(f"{func_name}: {msg}")
+
+    temp_file = await monitor.monitor_for_new_download_async(
+        timeout_sec=120, logger_func=log_progress
+    )
+
+    if not temp_file:
+        await browser_logger.error(
+            page, f"{func_name}: No new PDF detected in downloads directory"
+        )
+        return None
+
+    await browser_logger.info(
+        page,
+        f"{func_name}: Detected PDF: {temp_file.name} ({temp_file.stat().st_size / 1e6:.1f} MB)",
+    )
+
+    # Save minimal metadata
+    if doi:
+        import json
+
+        metadata_file = temp_file.parent / f"{temp_file.name}.meta.json"
+        metadata = {"doi": doi, "pdf_url": pdf_url, "pdf_file": temp_file.name}
+        with open(metadata_file, "w") as f:
+            json.dump(metadata, f, indent=2)
+
+    await browser_logger.info(
+        page, f"{func_name}: Manual download complete - saved in downloads/"
+    )
+
+    logger.info(f"{func_name}: PDF: {temp_file}")
+    if doi:
+        logger.info(f"{func_name}: DOI: {doi} (saved in {temp_file.name}.meta.json)")
+
+    return temp_file
+
+
 # EOF

scitex/scholar/pipelines/ScholarPipelineBibTeX.py

@@ -1,14 +1,8 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# Timestamp: "2025-10-16 01:47:39 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/pipelines/ScholarPipelineBibTeX.py
-# ----------------------------------------
-from __future__ import annotations
-import os
+# Timestamp: "2026-01-22 16:32:41 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/scholar/pipelines/ScholarPipelineBibTeX.py
 
-__FILE__ = "./src/scitex/scholar/pipelines/ScholarPipelineBibTeX.py"
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
 
 """
 Functionalities:
@@ -33,12 +27,12 @@ IO:
     - {input_bibtex}_processed.bib (enriched BibTeX with download status)
 """
 
-"""Imports"""
-import argparse
 import asyncio
 from pathlib import Path
-from typing import Optional, Union
+from typing import Optional
+from typing import Union
 
+import scitex as stx
 from scitex import logging
 from scitex.scholar.core import Papers
 from scitex.scholar.pipelines.ScholarPipelineParallel import (
@@ -49,8 +43,6 @@ from scitex.scholar.storage import BibTeXHandler
 logger = logging.getLogger(__name__)
 
 """Functions & Classes"""
-
-
 class ScholarPipelineBibTeX:
     """Processes BibTeX files through parallel paper acquisition pipeline"""
 
@@ -96,7 +88,8 @@ class ScholarPipelineBibTeX:
             project: Project name for symlinking (optional)
             output_bibtex_path: Path to save enriched BibTeX (optional, defaults to {input}_processed.bib)
 
-        Returns:
+        Returns
+        -------
             Papers collection with processed papers
         """
         bibtex_path = Path(bibtex_path)
@@ -145,7 +138,9 @@ class ScholarPipelineBibTeX:
         logger.success(
             f"{self.name}: Processed {len(processed_papers)}/{len(papers)} papers"
         )
-        logger.success(f"{self.name}: Saved enriched BibTeX: {output_bibtex_path}")
+        logger.success(
+            f"{self.name}: Saved enriched BibTeX: {output_bibtex_path}"
+        )
 
         # Update project bibliography if project specified
         if project:
@@ -161,7 +156,9 @@ class ScholarPipelineBibTeX:
                     bibtex_files=[bibtex_path, output_bibtex_path],
                 )
 
-                logger.success(f"{self.name}: Updated project bibliography: {project}")
+                logger.success(
+                    f"{self.name}: Updated project bibliography: {project}"
+                )
             except Exception as e:
                 logger.warning(f"Failed to update bibliography: {e}")
 
@@ -180,7 +177,8 @@ class ScholarPipelineBibTeX:
             project: Project name for symlinking (optional)
             output_bibtex_path: Path to save enriched BibTeX (optional)
 
-        Returns:
+        Returns
+        -------
             Papers collection with processed papers
         """
         logger.info(f"{self.name}: Processing BibTeX text content")
@@ -193,7 +191,9 @@ class ScholarPipelineBibTeX:
             logger.warning(f"{self.name}: No papers found in BibTeX text")
             return Papers([], project=project)
 
-        logger.info(f"{self.name}: Loaded {len(papers)} papers from BibTeX text")
+        logger.info(
+            f"{self.name}: Loaded {len(papers)} papers from BibTeX text"
+        )
 
         # Step 2: Process papers in parallel
         papers_collection = Papers(papers, project=project)
@@ -213,7 +213,9 @@ class ScholarPipelineBibTeX:
                 processed_collection,
                 output_path=output_bibtex_path,
             )
-            logger.success(f"{self.name}: Saved enriched BibTeX: {output_bibtex_path}")
+            logger.success(
+                f"{self.name}: Saved enriched BibTeX: {output_bibtex_path}"
+            )
 
         logger.success(
             f"{self.name}: Processed {len(processed_papers)}/{len(papers)} papers"
@@ -222,134 +224,84 @@ class ScholarPipelineBibTeX:
         return processed_collection
 
 
-def main(args):
-    """Run BibTeX pipeline"""
-
-    if not args.bibtex:
+@stx.session
+def main(
+    bibtex: str = None,
+    project: str = None,
+    output: str = None,
+    num_workers: int = 4,
+    browser_mode: str = "stealth",
+    chrome_profile: str = "system",
+    CONFIG=stx.INJECTED,
+    logger=stx.INJECTED,
+) -> int:
+    """Process BibTeX files through parallel paper acquisition pipeline.
+
+    Parameters
+    ----------
+    bibtex : str
+        Path to BibTeX file (required)
+    project : str
+        Project name for symlinking (optional)
+    output : str
+        Output BibTeX path (default: {input}_processed.bib)
+    num_workers : int
+        Number of parallel workers (default: 4)
+    browser_mode : str
+        Browser mode: 'stealth' or 'interactive' (default: stealth)
+    chrome_profile : str
+        Base Chrome profile name to sync from (default: system)
+
+    Returns
+    -------
+    int
+        Exit status code (0 for success)
+    """
+    if not bibtex:
         logger.error("No BibTeX file provided. Use --bibtex")
         return 1
 
-    bibtex_path = Path(args.bibtex)
+    bibtex_path = Path(bibtex)
     if not bibtex_path.exists():
         logger.error(f"BibTeX file not found: {bibtex_path}")
         return 1
 
     logger.info(f"Processing BibTeX file: {bibtex_path}")
-    logger.info(f"Workers: {args.num_workers}")
-    logger.info(f"Project: {args.project or 'None'}")
+    logger.info(f"Workers: {num_workers}")
+    logger.info(f"Project: {project or 'None'}")
 
     # Create BibTeX pipeline
     bibtex_pipeline = ScholarPipelineBibTeX(
-        num_workers=args.num_workers,
-        browser_mode=args.browser_mode,
-        base_chrome_profile=args.chrome_profile,
+        num_workers=num_workers,
+        browser_mode=browser_mode,
+        base_chrome_profile=chrome_profile,
     )
 
     # Run pipeline
     papers = asyncio.run(
         bibtex_pipeline.process_bibtex_file_async(
             bibtex_path=bibtex_path,
-            project=args.project,
-            output_bibtex_path=args.output,
+            project=project,
+            output_bibtex_path=output,
         )
     )
 
-    logger.success(f"BibTeX processing complete: {len(papers)} papers processed")
-    return 0
-
-
-def parse_args() -> argparse.Namespace:
-    """Parse command line arguments."""
-    parser = argparse.ArgumentParser(
-        description="Process BibTeX files through parallel paper acquisition pipeline"
-    )
-    parser.add_argument(
-        "--bibtex",
-        type=str,
-        required=True,
-        help="Path to BibTeX file",
-    )
-    parser.add_argument(
-        "--project",
-        type=str,
-        default=None,
-        help="Project name for symlinking (optional)",
-    )
-    parser.add_argument(
-        "--output",
-        type=str,
-        default=None,
-        help="Output BibTeX path (default: {input}_processed.bib)",
-    )
-    parser.add_argument(
-        "--num-workers",
-        type=int,
-        default=4,
-        help="Number of parallel workers (default: 4)",
-    )
-    parser.add_argument(
-        "--browser-mode",
-        type=str,
-        choices=["stealth", "interactive"],
-        default="stealth",
-        help="Browser mode (default: stealth)",
-    )
-    parser.add_argument(
-        "--chrome-profile",
-        type=str,
-        default="system",
-        help="Base Chrome profile name to sync from (default: system)",
-    )
-    args = parser.parse_args()
-    return args
-
-
-def run_main() -> None:
-    """Initialize scitex framework, run main function, and cleanup."""
-    global CONFIG, CC, sys, plt, rng
-
-    import sys
-
-    import matplotlib.pyplot as plt
-
-    import scitex as stx
-
-    args = parse_args()
-
-    CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
-        sys,
-        plt,
-        args=args,
-        file=__FILE__,
-        sdir_suffix=None,
-        verbose=False,
-        agg=True,
-    )
-
-    exit_status = main(args)
-
-    stx.session.close(
-        CONFIG,
-        verbose=False,
-        notify=False,
-        message="",
-        exit_status=exit_status,
+    logger.success(
+        f"BibTeX processing complete: {len(papers)} papers processed"
     )
+    return 0
 
 
 if __name__ == "__main__":
-    run_main()
+    main()
 
 """
-Usage:
-
-    # Process BibTeX file with 8 workers
-    python -m scitex.scholar.pipelines.ScholarPipelineBibTeX \
-        --bibtex ./data/scholar/bib_files/neurovista.bib \
-        --project neurovista \
-        --num-workers 8 \
-        --chrome-profile system \
-        --browser-mode stealth
+python -m scitex.scholar.pipelines.ScholarPipelineBibTeX \
+    --bibtex ./data/scholar/bib_files/neurovista.bib \
+    --project neurovista \
+    --num-workers 8 \
+    --chrome-profile system \
+    --browser-mode interactive
 """
 
 # EOF