scitex 2.14.0__py3-none-any.whl → 2.16.0__py3-none-any.whl

scitex/scholar/pipelines/ScholarPipelineParallel.py

@@ -1,15 +1,6 @@
 #!/usr/bin/env python3
-# Timestamp: "2025-10-13 06:13:41 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/pipelines/ScholarPipelineParallel.py
-# ----------------------------------------
-from __future__ import annotations
-
-import os
-
-__FILE__ = "./src/scitex/scholar/pipelines/ScholarPipelineParallel.py"
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
+# Timestamp: "2026-01-22 (ywatanabe)"
+# File: src/scitex/scholar/pipelines/ScholarPipelineParallel.py
 """
 Functionalities:
   - Orchestrates parallel paper acquisition using multiple browser profiles
@@ -33,11 +24,12 @@ IO:
     - library/{project}/{paper_id} -> ../MASTER/{paper_id} (multiple symlinks)
 """
 
-"""Imports"""
-import argparse
+from __future__ import annotations
+
 import asyncio
 from typing import List, Optional
 
+import scitex as stx
 from scitex import logging
 from scitex.browser.core import ChromeProfileManager
 from scitex.scholar.auth import ScholarAuthManager
@@ -79,7 +71,8 @@ class ScholarPipelineParallel:
     async def _verify_authentication_async(self) -> bool:
         """Pre-verify authentication once before spawning workers.
 
-        Returns:
+        Returns
+        -------
             True if authenticated, False otherwise
         """
         logger.info(f"{self.name}: Verifying authentication...")
@@ -110,7 +103,8 @@ class ScholarPipelineParallel:
         Args:
             num_workers: Number of workers to prepare (defaults to self.num_workers)
 
-        Returns:
+        Returns
+        -------
             List of worker profile names
         """
         workers_to_prepare = (
@@ -157,7 +151,8 @@ class ScholarPipelineParallel:
             worker_id: Worker ID for logging
             worker_profile: Chrome profile name for this worker
 
-        Returns:
+        Returns
+        -------
             Paper object if successful, None otherwise
         """
         logger.info(
@@ -200,7 +195,8 @@ class ScholarPipelineParallel:
             doi_or_title_list: List of DOI or title strings
             project: Project name for symlinking (optional)
 
-        Returns:
+        Returns
+        -------
             List of successfully processed Paper objects
         """
         if not doi_or_title_list:
@@ -291,7 +287,8 @@ class ScholarPipelineParallel:
             papers: Papers collection
             project: Project name for symlinking (optional, uses papers.project if None)
 
-        Returns:
+        Returns
+        -------
             List of successfully processed Paper objects
         """
         # Extract DOIs or titles from papers
@@ -314,34 +311,64 @@ class ScholarPipelineParallel:
         )
 
 
-def main(args):
-    """Run parallel pipeline"""
-
+@stx.session
+def main(
+    dois: str = None,
+    titles: str = None,
+    project: str = None,
+    num_workers: int = 4,
+    browser_mode: str = "stealth",
+    chrome_profile: str = "system",
+    CONFIG=stx.INJECTED,
+    logger=stx.INJECTED,
+) -> int:
+    """Orchestrate parallel paper acquisition pipeline.
+
+    Parameters
+    ----------
+    dois : str
+        Comma-separated DOIs (e.g., '10.1038/...,10.1016/...')
+    titles : str
+        Comma-separated paper titles
+    project : str
+        Project name for symlinking (optional)
+    num_workers : int
+        Number of parallel workers (default: 4)
+    browser_mode : str
+        Browser mode: 'stealth' or 'interactive' (default: stealth)
+    chrome_profile : str
+        Base Chrome profile name to sync from (default: system)
+
+    Returns
+    -------
+    int
+        Exit status code (0 for success)
+    """
     # Parse input queries
     queries = []
-    if args.dois:
-        queries.extend(args.dois.split(","))
-    if args.titles:
-        queries.extend(args.titles.split(","))
+    if dois:
+        queries.extend(dois.split(","))
+    if titles:
+        queries.extend(titles.split(","))
 
     if not queries:
         logger.error("No queries provided. Use --dois or --titles")
         return 1
 
-    logger.info(f"Processing {len(queries)} queries with {args.num_workers} workers")
+    logger.info(f"Processing {len(queries)} queries with {num_workers} workers")
 
     # Create parallel pipeline
     parallel_pipeline = ScholarPipelineParallel(
-        num_workers=args.num_workers,
-        browser_mode=args.browser_mode,
-        base_chrome_profile=args.chrome_profile,
+        num_workers=num_workers,
+        browser_mode=browser_mode,
+        base_chrome_profile=chrome_profile,
     )
 
     # Run pipeline
     papers = asyncio.run(
         parallel_pipeline.process_papers_from_list_async(
             doi_or_title_list=queries,
-            project=args.project,
+            project=project,
         )
     )
 
@@ -349,114 +376,29 @@ def main(args):
     return 0
 
 
-def parse_args() -> argparse.Namespace:
-    """Parse command line arguments."""
-    parser = argparse.ArgumentParser(
-        description="Orchestrate parallel paper acquisition pipeline"
-    )
-    parser.add_argument(
-        "--dois",
-        type=str,
-        default=None,
-        help="Comma-separated DOIs (e.g., '10.1038/...,10.1016/...')",
-    )
-    parser.add_argument(
-        "--titles",
-        type=str,
-        default=None,
-        help="Comma-separated paper titles",
-    )
-    parser.add_argument(
-        "--project",
-        type=str,
-        default=None,
-        help="Project name for symlinking (optional)",
-    )
-    parser.add_argument(
-        "--num-workers",
-        type=int,
-        default=4,
-        help="Number of parallel workers (default: 4)",
-    )
-    parser.add_argument(
-        "--browser-mode",
-        type=str,
-        choices=["stealth", "interactive"],
-        default="stealth",
-        help="Browser mode (default: stealth)",
-    )
-    parser.add_argument(
-        "--chrome-profile",
-        type=str,
-        default="system",
-        help="Base Chrome profile name to sync from (default: system)",
-    )
-    args = parser.parse_args()
-    return args
-
-
-def run_main() -> None:
-    """Initialize scitex framework, run main function, and cleanup."""
-    global CONFIG, CC, sys, plt, rng
-
-    import sys
-
-    import matplotlib.pyplot as plt
-
-    import scitex as stx
-
-    args = parse_args()
-
-    CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
-        sys,
-        plt,
-        args=args,
-        file=__FILE__,
-        sdir_suffix=None,
-        verbose=False,
-        agg=True,
-    )
-
-    exit_status = main(args)
-
-    stx.session.close(
-        CONFIG,
-        verbose=False,
-        notify=False,
-        message="",
-        exit_status=exit_status,
-    )
-
-
 if __name__ == "__main__":
-    run_main()
-
-"""
-Usage:
-    # With DOIs (4 workers)
-    python -m scitex.scholar.pipelines.ScholarPipelineParallel \
-        --dois "10.1212/wnl.0000000000200348,10.1038/s41598-017-02626-y" \
-        --project neurovista \
-        --num-workers 4 \
-        --browser-mode stealth \
-        --chrome-profile system
-
-    # With titles (2 workers)
-    python -m scitex.scholar.pipelines.ScholarPipelineParallel \
-        --titles "Attention Is All You Need,BERT: Pre-training of Deep Bidirectional Transformers" \
-        --project transformers \
-        --num-workers 2 \
-        --browser-mode stealth \
-        --chrome-profile system
-
-    # Mixed DOIs and titles (8 workers)
-    python -m scitex.scholar.pipelines.ScholarPipelineParallel \
-        --dois "10.1038/s41593-025-01990-7" \
-        --titles "Neural State Monitoring in the Treatment of Epilepsy" \
-        --project epilepsy \
-        --num-workers 8 \
-        --browser-mode stealth \
-        --chrome-profile system
-"""
+    main()
+
+# Usage:
+#     # With DOIs (4 workers)
+#     python -m scitex.scholar.pipelines.ScholarPipelineParallel \
+#         --dois "10.1212/wnl.0000000000200348,10.1038/s41598-017-02626-y" \
+#         --project neurovista \
+#         --num-workers 4 \
+#         --browser-mode stealth \
+#         --chrome-profile system
+#
+#     # With titles (2 workers)
+#     python -m scitex.scholar.pipelines.ScholarPipelineParallel \
+#         --titles "Attention Is All You Need,BERT: Pre-training" \
+#         --project transformers \
+#         --num-workers 2
+#
+#     # Mixed DOIs and titles (8 workers)
+#     python -m scitex.scholar.pipelines.ScholarPipelineParallel \
+#         --dois "10.1038/s41593-025-01990-7" \
+#         --titles "Neural State Monitoring in the Treatment of Epilepsy" \
+#         --project epilepsy \
+#         --num-workers 8
 
 # EOF

scitex/scholar/pipelines/ScholarPipelineSingle.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Timestamp: "2026-01-14 (ywatanabe)"
+# Timestamp: "2026-01-22 (ywatanabe)"
 # File: src/scitex/scholar/pipelines/ScholarPipelineSingle.py
 """
 Single paper acquisition pipeline orchestrator.
@@ -21,10 +21,10 @@ IO:
 
 from __future__ import annotations
 
-import argparse
 import asyncio
 from typing import Optional
 
+import scitex as stx
 from scitex import logging
 from scitex.scholar.storage import PaperIO
 
@@ -90,10 +90,10 @@ class ScholarPipelineSingle(PipelineStepsMixin, PipelineHelpersMixin):
             )
             if context:
                 await self._step_06_find_pdf_urls(
-                    paper, io, context, auth_gateway, force
+                    paper, io, context, auth_gateway, force, browser_manager
                 )
                 await self._step_07_download_pdf(
-                    paper, io, context, auth_gateway, force
+                    paper, io, context, auth_gateway, force, browser_manager
                 )
             if browser_manager:
                 await browser_manager.close()
@@ -108,75 +108,55 @@ class ScholarPipelineSingle(PipelineStepsMixin, PipelineHelpersMixin):
             return paper, symlink_path
 
 
-def main(args):
-    """Run single paper pipeline."""
+@stx.session
+def main(
+    doi_or_title: str = None,
+    project: str = None,
+    browser_mode: str = "stealth",
+    chrome_profile: str = "system",
+    force: bool = False,
+    CONFIG=stx.INJECTED,
+    logger=stx.INJECTED,
+) -> int:
+    """Orchestrate full paper acquisition pipeline.
+
+    Parameters
+    ----------
+    doi_or_title : str
+        DOI or paper title (required)
+    project : str
+        Project name for symlinking (optional)
+    browser_mode : str
+        Browser mode: 'stealth' or 'interactive' (default: stealth)
+    chrome_profile : str
+        Chrome profile name (default: system)
+    force : bool
+        Force fresh processing (default: False)
+
+    Returns
+    -------
+    int
+        Exit status code (0 for success)
+    """
+    if not doi_or_title:
+        logger.error("--doi-or-title is required")
+        return 1
+
     pipeline = ScholarPipelineSingle(
-        browser_mode=args.browser_mode, chrome_profile=args.chrome_profile
+        browser_mode=browser_mode, chrome_profile=chrome_profile
     )
     paper, symlink_path = asyncio.run(
         pipeline.process_single_paper(
-            doi_or_title=args.doi_or_title,
-            project=args.project,
-            force=args.force,
+            doi_or_title=doi_or_title,
+            project=project,
+            force=force,
         )
     )
     return 0
 
 
-def parse_args() -> argparse.Namespace:
-    """Parse command line arguments."""
-    parser = argparse.ArgumentParser(
-        description="Orchestrate full paper acquisition pipeline"
-    )
-    parser.add_argument(
-        "--doi-or-title", type=str, required=True, help="DOI or paper title"
-    )
-    parser.add_argument(
-        "--project", type=str, default=None, help="Project name for symlinking"
-    )
-    parser.add_argument(
-        "--browser-mode",
-        type=str,
-        choices=["stealth", "interactive"],
-        default="stealth",
-        help="Browser mode (default: stealth)",
-    )
-    parser.add_argument(
-        "--chrome-profile",
-        type=str,
-        required=True,
-        help="Chrome profile name (default: system)",
-    )
-    parser.add_argument(
-        "--force",
-        "-f",
-        action="store_true",
-        default=False,
-        help="Force fresh processing",
-    )
-    return parser.parse_args()
-
-
-def run_main() -> None:
-    """Initialize scitex framework, run main function, and cleanup."""
-    import sys
-
-    import matplotlib.pyplot as plt
-
-    import scitex as stx
-
-    args = parse_args()
-    CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
-        sys, plt, args=args, file=__file__, sdir_suffix=None, verbose=False, agg=True
-    )
-    exit_status = main(args)
-    stx.session.close(
-        CONFIG, verbose=False, notify=False, message="", exit_status=exit_status
-    )
-
-
 if __name__ == "__main__":
-    run_main()
+    main()
 
 # Usage:
 # python -m scitex.scholar.pipelines.ScholarPipelineSingle \

scitex/scholar/pipelines/_single_steps.py

@@ -115,7 +115,9 @@ class PipelineStepsMixin:
         )
         return browser_manager, context, auth_gateway
 
-    async def _step_06_find_pdf_urls(self, paper, io, context, auth_gateway, force):
+    async def _step_06_find_pdf_urls(
+        self, paper, io, context, auth_gateway, force, browser_manager=None
+    ):
         if not paper.metadata.url.pdfs or force:
             logger.info(f"{self.name}: Finding PDF URLs...")
             try:
@@ -127,6 +129,9 @@ class PipelineStepsMixin:
                 )
             except Exception as e:
                 logger.warning(f"{self.name}: Auth gateway failed: {e}")
+                await self._capture_screenshot(
+                    browser_manager, context, io, "auth_gateway_failed"
+                )
                 publisher_url = paper.metadata.id.doi
             from scitex.scholar import ScholarURLFinder
 
@@ -135,23 +140,34 @@ class PipelineStepsMixin:
             paper.metadata.url.pdfs = urls
             paper.metadata.url.pdfs_engines = ["ScholarURLFinder"]
             io.save_metadata()
+            if not urls:
+                await self._capture_screenshot(
+                    browser_manager, context, io, "no_pdf_urls_found"
+                )
             logger.info(f"{self.name}: Found {len(urls)} PDF URL(s)")
         else:
             logger.info(f"{self.name}: PDF URLs exist ({len(paper.metadata.url.pdfs)})")
 
-    async def _step_07_download_pdf(self, paper, io, context, auth_gateway, force):
+    async def _step_07_download_pdf(
+        self, paper, io, context, auth_gateway, force, browser_manager=None
+    ):
         if (not io.has_pdf() or force) and paper.metadata.url.pdfs:
             logger.info(f"{self.name}: Downloading PDF...")
             from scitex.scholar.pdf_download import ScholarPDFDownloader
 
             downloader = ScholarPDFDownloader(context)
-            downloaded, temp_path = await self._download_pdf_from_url(
+            downloaded, temp_path, download_method = await self._download_pdf_from_url(
                 paper, io, context, auth_gateway, downloader
             )
             if downloaded:
-                self._handle_downloaded_pdf(paper, io, downloaded, temp_path)
+                self._handle_downloaded_pdf(
+                    paper, io, downloaded, temp_path, download_method
+                )
             else:
-                self._check_manual_download(io)
+                await self._capture_screenshot(
+                    browser_manager, context, io, "pdf_download_failed"
+                )
+                self._check_manual_download(io, paper)
         elif io.has_pdf():
             logger.info(f"{self.name}: PDF already exists, skipping download")
 
@@ -211,24 +227,35 @@ class PipelineStepsMixin:
         downloaded_file = await downloader.download_from_url(
             pdf_url, output_path=temp_pdf_path, doi=paper.metadata.id.doi
         )
-        return downloaded_file, temp_pdf_path
-
-    def _handle_downloaded_pdf(self, paper, io, downloaded_file, temp_pdf_path):
+        # Track download method based on context flags
+        download_method = "unknown"
+        if downloaded_file:
+            is_manual = getattr(context, "_scitex_is_manual_mode", False)
+            download_method = "manual_download" if is_manual else "automated"
+        return downloaded_file, temp_pdf_path, download_method
+
+    def _handle_downloaded_pdf(
+        self, paper, io, downloaded_file, temp_pdf_path, download_method="unknown"
+    ):
         import shutil
 
         if downloaded_file == temp_pdf_path and temp_pdf_path.exists():
             main_pdf = io.get_pdf_path()
             shutil.move(str(temp_pdf_path), str(main_pdf))
             paper.metadata.path.pdfs = [str(main_pdf)]
+            paper.metadata.path.pdfs_engines = [download_method]
             paper.container.pdf_size_bytes = main_pdf.stat().st_size
             io.save_metadata()
-            logger.success(f"{self.name}: PDF downloaded to MASTER")
+            logger.success(
+                f"{self.name}: PDF downloaded to MASTER via {download_method}"
+            )
         else:
             io.save_pdf(downloaded_file)
+            paper.metadata.path.pdfs_engines = [download_method]
             io.save_metadata()
         logger.info(f"{self.name}: PDF saved ({str(downloaded_file)})")
 
-    def _check_manual_download(self, io):
+    def _check_manual_download(self, io, paper=None):
         import time
 
         from scitex.scholar import ScholarConfig
@@ -249,6 +276,9 @@ class PipelineStepsMixin:
             latest_pdf = recent_pdfs[0][0]
             logger.info(f"{self.name}: Found recent PDF: {latest_pdf.name}")
             io.save_pdf(latest_pdf)
+            # Track as manual download
+            if paper:
+                paper.metadata.path.pdfs_engines = ["manual_download"]
             io.save_metadata()
             logger.success(f"{self.name}: Manual PDF saved to MASTER")
         else:
@@ -258,39 +288,44 @@ class PipelineStepsMixin:
 class PipelineHelpersMixin:
     """Mixin containing helper methods for single paper pipeline."""
 
+    async def _capture_screenshot(self, browser_manager, context, io, description):
+        """Capture screenshot for debugging when issues occur."""
+        if not browser_manager or not context:
+            return
+        try:
+            from datetime import datetime
+
+            screenshots_dir = io.paper_dir / "screenshots"
+            screenshots_dir.mkdir(parents=True, exist_ok=True)
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            screenshot_path = screenshots_dir / f"{timestamp}_{description}.png"
+            pages = context.pages
+            if pages:
+                page = pages[0]
+                await browser_manager.take_screenshot_async(
+                    page, str(screenshot_path), full_page=True
+                )
+                logger.info(f"{self.name}: Screenshot saved: {screenshot_path.name}")
+        except Exception as e:
+            logger.debug(f"{self.name}: Screenshot capture failed: {e}")
+
     def _generate_paper_id(self, doi: str) -> str:
         """Generate 8-digit library ID from DOI."""
         return hashlib.md5(f"DOI:{doi}".encode()).hexdigest()[:8].upper()
 
     def _link_to_project(self, paper: Paper, project: str, io: PaperIO) -> Path:
-        """Create human-readable symlink in project directory."""
-        from scitex.scholar import ScholarConfig
+        """Create human-readable symlink in project directory using LibraryManager."""
+        from scitex.scholar.storage import LibraryManager
 
-        config = ScholarConfig()
-        project_dir = config.path_manager.get_library_project_dir(project)
-        pdf_files = list(io.paper_dir.glob("*.pdf"))
-        entry_name = config.path_manager.get_library_project_entry_dirname(
-            n_pdfs=len(pdf_files),
-            citation_count=paper.metadata.citation_count.total or 0,
-            impact_factor=int(paper.metadata.publication.impact_factor or 0),
-            year=paper.metadata.basic.year or 0,
-            first_author=(
-                paper.metadata.basic.authors[0].split()[-1]
-                if paper.metadata.basic.authors
-                else "Unknown"
-            ),
-            journal_name=(
-                paper.metadata.publication.short_journal
-                or paper.metadata.publication.journal
-                or "Unknown"
-            ),
+        library_manager = LibraryManager()
+        symlink_path = library_manager.update_symlink(
+            master_storage_path=io.paper_dir,
+            project=project,
         )
-        symlink_path = project_dir / entry_name
-        target_path = Path("../MASTER") / paper.container.library_id
-        if symlink_path.exists() or symlink_path.is_symlink():
-            symlink_path.unlink()
-        symlink_path.symlink_to(target_path)
-        logger.success(f"{self.name}: Created symlink: {project}/{entry_name}")
+        if symlink_path:
+            logger.success(
+                f"{self.name}: Created symlink: {project}/{symlink_path.name}"
+            )
         return symlink_path
 
     def _enrich_impact_factor(self, paper: Paper) -> None: