scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/scholar/pdf_download/strategies/response_body.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-10-13 08:02:32 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/pdf_download/strategies/response_body.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = (
+    "./src/scitex/scholar/pdf_download/strategies/response_body.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+"""Response Body Download Strategy"""
+
+from pathlib import Path
+from typing import Optional
+
+from playwright.async_api import BrowserContext
+
+from scitex import logging
+from scitex.scholar.browser import browser_logger
+
+logger = logging.getLogger(__name__)
+
+
+async def try_download_response_body_async(
+    context: BrowserContext,
+    pdf_url: str,
+    output_path: Path,
+    func_name: str = "try_download_response_body_async",
+) -> Optional[Path]:
+    """Download PDF from HTTP response body."""
+    # Check if manual mode is active - skip immediately
+    if (
+        hasattr(context, "_scitex_is_manual_mode")
+        and context._scitex_is_manual_mode
+    ):
+        logger.info(f"{func_name}: Skipping because manual mode is active...")
+        return None
+
+    page = None
+    try:
+        page = await context.new_page()
+        await browser_logger.info(
+            page,
+            f"{func_name}: Trying to download {pdf_url} from response body",
+        )
+
+        # Step 1: Navigate
+        await browser_logger.info(
+            page,
+            f"{func_name}: Navigating to {pdf_url[:60]}...",
+        )
+
+        download_path = None
+        download_handler_active = True
+
+        async def handle_download(download):
+            nonlocal download_path
+            # Only handle downloads if NOT in manual mode
+            if (
+                hasattr(context, "_scitex_is_manual_mode")
+                and context._scitex_is_manual_mode
+            ):
+                logger.info(
+                    f"{func_name}: Ignoring download (manual mode active)"
+                )
+                return
+
+            if download_handler_active:
+                await download.save_as(output_path)
+                download_path = output_path
+
+        page.on("download", handle_download)
+
+        response = await page.goto(pdf_url, wait_until="load", timeout=60_000)
+
+        await browser_logger.info(
+            page,
+            f"{func_name}: Loaded, waiting for auto-download (60s)...",
+        )
+
+        # Wait for download, but check for manual mode activation every second
+        for i in range(60):
+            # Check if manual mode was activated - ABORT IMMEDIATELY
+            if (
+                hasattr(context, "_scitex_is_manual_mode")
+                and context._scitex_is_manual_mode
+            ):
+                logger.info(f"{func_name}: Manual mode activated, aborting")
+                download_handler_active = False  # Disable handler
+                page.remove_listener(
+                    "download", handle_download
+                )  # Remove listener
+                await page.close()
+                return None
+
+            await page.wait_for_timeout(1000)  # Wait 1 second
+
+            # Check if download already happened
+            if download_path and download_path.exists():
+                break
+
+        # Check if auto-download occurred
+        if download_path and download_path.exists():
+            size_MiB = download_path.stat().st_size / 1024 / 1024
+            await browser_logger.info(
+                page,
+                f"{func_name}: Auto-download: from {pdf_url} to {output_path} ({size_MiB:.2f} MiB)",
+            )
+            await browser_logger.info(
+                page,
+                f"{func_name}: ✓ Auto-download {size_MiB:.2f} MB",
+            )
+            await page.wait_for_timeout(2000)
+            await page.close()
+            return output_path
+
+        # Step 2: Check response
+        await browser_logger.info(
+            page,
+            f"{func_name}: Checking response (status: {response.status})...",
+        )
+
+        if not response.ok:
+            await browser_logger.fail(
+                page,
+                f"{func_name}: Page not reached: {pdf_url} (reason: {response.status})",
+            )
+            await browser_logger.fail(
+                page,
+                f"{func_name}: ✗ HTTP {response.status}",
+            )
+            await page.wait_for_timeout(2000)
+            await page.close()
+            return None
+
+        # Step 3: Extract from response body
+        await browser_logger.info(
+            page,
+            f"{func_name}: Extracting PDF from response body...",
+        )
+        content = await response.body()
+        content_type = response.headers.get("content-type", "")
+
+        is_pdf = content[:4] == b"%PDF" or "application/pdf" in content_type
+
+        is_html = (
+            content[:15].lower().startswith(b"<!doctype html")
+            or content[:6].lower().startswith(b"<html")
+            or "text/html" in content_type
+        )
+
+        if is_pdf and not is_html and len(content) > 1024:
+            with open(output_path, "wb") as file_:
+                file_.write(content)
+            size_MiB = len(content) / 1024 / 1024
+            await browser_logger.info(
+                page,
+                f"{func_name}: Response body download: from {pdf_url} to {output_path} ({size_MiB:.2f} MiB)",
+            )
+            await browser_logger.info(
+                page,
+                f"{func_name}: ✓ Extracted {size_MiB:.2f} MB",
+            )
+            await page.wait_for_timeout(2000)
+            await page.close()
+            return output_path
+
+        await browser_logger.warning(
+            page, f"{func_name}: Failed download from response body"
+        )
+        await browser_logger.warning(
+            page,
+            f"{func_name}: ✗ Not PDF (type: {content_type}, size: {len(content)})",
+        )
+        await page.wait_for_timeout(2000)
+        await page.close()
+        return None
+
+    except Exception as ee:
+        if page is not None:
+            await browser_logger.warning(
+                page,
+                f"{func_name}: Failed download from response body: {ee}",
+            )
+            try:
+                await browser_logger.info(
+                    page,
+                    f"{func_name}: ✗ EXCEPTION: {str(ee)[:100]}",
+                )
+                await page.wait_for_timeout(2000)
+            except Exception as popup_error:
+                logger.debug(
+                    f"{func_name}: Could not show error popup: {popup_error}"
+                )
+            finally:
+                try:
+                    await page.close()
+                except Exception as close_error:
+                    logger.debug(
+                        f"{func_name}: Error closing page: {close_error}"
+                    )
+        return None
+
+# EOF

scitex/scholar/pipelines/ScholarPipelineBibTeX.py

@@ -0,0 +1,364 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-10-16 01:47:39 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/pipelines/ScholarPipelineBibTeX.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = (
+    "./src/scitex/scholar/pipelines/ScholarPipelineBibTeX.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+"""
+Functionalities:
+  - Processes BibTeX files through parallel paper acquisition pipeline
+  - Loads papers from BibTeX parallel download update BibTeX with results
+  - Supports resumable processing (skips already downloaded papers)
+  - Updates BibTeX with enriched metadata and download status
+
+Dependencies:
+  - packages:
+    - playwright
+    - asyncio
+
+IO:
+  - input-files:
+    - BibTeX files (.bib)
+
+  - output-files:
+    - library/MASTER/{paper_id}/metadata.json (multiple papers)
+    - library/MASTER/{paper_id}/main.pdf (multiple papers)
+    - library/{project}/{paper_id} -> ../MASTER/{paper_id} (multiple symlinks)
+    - {input_bibtex}_processed.bib (enriched BibTeX with download status)
+"""
+
+"""Imports"""
+import argparse
+import asyncio
+from pathlib import Path
+from typing import Optional, Union
+
+from scitex import logging
+from scitex.scholar.core import Papers
+from scitex.scholar.pipelines.ScholarPipelineParallel import (
+    ScholarPipelineParallel,
+)
+from scitex.scholar.storage import BibTeXHandler
+
+logger = logging.getLogger(__name__)
+
+"""Functions & Classes"""
+class ScholarPipelineBibTeX:
+    """Processes BibTeX files through parallel paper acquisition pipeline"""
+
+    def __init__(
+        self,
+        num_workers: int = 4,
+        browser_mode: str = "stealth",
+        base_chrome_profile: str = "system",
+    ):
+        """Initialize BibTeX pipeline.
+
+        Args:
+            num_workers: Number of parallel workers (default: 4)
+            browser_mode: Browser mode for all workers ('stealth' or 'interactive')
+            base_chrome_profile: Base chrome profile to sync from (default: 'system')
+        """
+        self.name = self.__class__.__name__
+        self.num_workers = num_workers
+        self.browser_mode = browser_mode
+        self.base_chrome_profile = base_chrome_profile
+
+        # Initialize parallel pipeline
+        self.parallel_pipeline = ScholarPipelineParallel(
+            num_workers=num_workers,
+            browser_mode=browser_mode,
+            base_chrome_profile=base_chrome_profile,
+        )
+
+        logger.info(
+            f"{self.name}: Initialized with {num_workers} workers, mode={browser_mode}"
+        )
+
+    async def process_bibtex_file_async(
+        self,
+        bibtex_path: Union[str, Path],
+        project: Optional[str] = None,
+        output_bibtex_path: Optional[Union[str, Path]] = None,
+    ) -> Papers:
+        """Process all papers from a BibTeX file in parallel.
+
+        Args:
+            bibtex_path: Path to input BibTeX file
+            project: Project name for symlinking (optional)
+            output_bibtex_path: Path to save enriched BibTeX (optional, defaults to {input}_processed.bib)
+
+        Returns:
+            Papers collection with processed papers
+        """
+        bibtex_path = Path(bibtex_path)
+        if not bibtex_path.exists():
+            raise FileNotFoundError(f"BibTeX file not found: {bibtex_path}")
+
+        logger.info(f"{self.name}: Processing BibTeX file: {bibtex_path}")
+
+        # Step 1: Load papers from BibTeX
+        bibtex_handler = BibTeXHandler(project=project)
+        papers = bibtex_handler.papers_from_bibtex(bibtex_path)
+
+        if not papers:
+            logger.warning(f"{self.name}: No papers found in BibTeX file")
+            return Papers([], project=project)
+
+        logger.info(f"{self.name}: Loaded {len(papers)} papers from BibTeX")
+
+        # Step 2: Process papers in parallel using ScholarPipelineParallel
+        papers_collection = Papers(papers, project=project)
+        processed_papers = (
+            await self.parallel_pipeline.process_papers_from_collection_async(
+                papers=papers_collection,
+                project=project,
+            )
+        )
+
+        # Step 3: Save enriched BibTeX with processing results
+        if output_bibtex_path is None:
+            # Default: save as {input}_processed.bib
+            output_bibtex_path = (
+                bibtex_path.parent / f"{bibtex_path.stem}_processed.bib"
+            )
+        else:
+            output_bibtex_path = Path(output_bibtex_path)
+
+        # Convert processed papers to Papers collection
+        processed_collection = Papers(processed_papers, project=project)
+
+        # Save enriched BibTeX
+        bibtex_handler.papers_to_bibtex(
+            processed_collection,
+            output_path=output_bibtex_path,
+        )
+
+        logger.success(
+            f"{self.name}: Processed {len(processed_papers)}/{len(papers)} papers"
+        )
+        logger.success(
+            f"{self.name}: Saved enriched BibTeX: {output_bibtex_path}"
+        )
+
+        # Update project bibliography if project specified
+        if project:
+            try:
+                from scitex.scholar.config import ScholarConfig
+
+                config = ScholarConfig()
+                bib_handler = BibTeXHandler(project=project, config=config)
+
+                # Setup bibliography with original and processed files
+                bib_handler.setup_project_bibliography(
+                    project=project,
+                    bibtex_files=[bibtex_path, output_bibtex_path],
+                )
+
+                logger.success(
+                    f"{self.name}: Updated project bibliography: {project}"
+                )
+            except Exception as e:
+                logger.warning(f"Failed to update bibliography: {e}")
+
+        return processed_collection
+
+    async def process_bibtex_text_async(
+        self,
+        bibtex_text: str,
+        project: Optional[str] = None,
+        output_bibtex_path: Optional[Union[str, Path]] = None,
+    ) -> Papers:
+        """Process papers from BibTeX text content in parallel.
+
+        Args:
+            bibtex_text: BibTeX content as string
+            project: Project name for symlinking (optional)
+            output_bibtex_path: Path to save enriched BibTeX (optional)
+
+        Returns:
+            Papers collection with processed papers
+        """
+        logger.info(f"{self.name}: Processing BibTeX text content")
+
+        # Step 1: Load papers from BibTeX text
+        bibtex_handler = BibTeXHandler(project=project)
+        papers = bibtex_handler.papers_from_bibtex(bibtex_text)
+
+        if not papers:
+            logger.warning(f"{self.name}: No papers found in BibTeX text")
+            return Papers([], project=project)
+
+        logger.info(
+            f"{self.name}: Loaded {len(papers)} papers from BibTeX text"
+        )
+
+        # Step 2: Process papers in parallel
+        papers_collection = Papers(papers, project=project)
+        processed_papers = (
+            await self.parallel_pipeline.process_papers_from_collection_async(
+                papers=papers_collection,
+                project=project,
+            )
+        )
+
+        # Step 3: Save enriched BibTeX if path provided
+        processed_collection = Papers(processed_papers, project=project)
+
+        if output_bibtex_path:
+            output_bibtex_path = Path(output_bibtex_path)
+            bibtex_handler.papers_to_bibtex(
+                processed_collection,
+                output_path=output_bibtex_path,
+            )
+            logger.success(
+                f"{self.name}: Saved enriched BibTeX: {output_bibtex_path}"
+            )
+
+        logger.success(
+            f"{self.name}: Processed {len(processed_papers)}/{len(papers)} papers"
+        )
+
+        return processed_collection
+
+
+def main(args):
+    """Run BibTeX pipeline"""
+
+    if not args.bibtex:
+        logger.error("No BibTeX file provided. Use --bibtex")
+        return 1
+
+    bibtex_path = Path(args.bibtex)
+    if not bibtex_path.exists():
+        logger.error(f"BibTeX file not found: {bibtex_path}")
+        return 1
+
+    logger.info(f"Processing BibTeX file: {bibtex_path}")
+    logger.info(f"Workers: {args.num_workers}")
+    logger.info(f"Project: {args.project or 'None'}")
+
+    # Create BibTeX pipeline
+    bibtex_pipeline = ScholarPipelineBibTeX(
+        num_workers=args.num_workers,
+        browser_mode=args.browser_mode,
+        base_chrome_profile=args.chrome_profile,
+    )
+
+    # Run pipeline
+    papers = asyncio.run(
+        bibtex_pipeline.process_bibtex_file_async(
+            bibtex_path=bibtex_path,
+            project=args.project,
+            output_bibtex_path=args.output,
+        )
+    )
+
+    logger.success(
+        f"BibTeX processing complete: {len(papers)} papers processed"
+    )
+    return 0
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Process BibTeX files through parallel paper acquisition pipeline"
+    )
+    parser.add_argument(
+        "--bibtex",
+        type=str,
+        required=True,
+        help="Path to BibTeX file",
+    )
+    parser.add_argument(
+        "--project",
+        type=str,
+        default=None,
+        help="Project name for symlinking (optional)",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=None,
+        help="Output BibTeX path (default: {input}_processed.bib)",
+    )
+    parser.add_argument(
+        "--num-workers",
+        type=int,
+        default=4,
+        help="Number of parallel workers (default: 4)",
+    )
+    parser.add_argument(
+        "--browser-mode",
+        type=str,
+        choices=["stealth", "interactive"],
+        default="stealth",
+        help="Browser mode (default: stealth)",
+    )
+    parser.add_argument(
+        "--chrome-profile",
+        type=str,
+        default="system",
+        help="Base Chrome profile name to sync from (default: system)",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def run_main() -> None:
+    """Initialize scitex framework, run main function, and cleanup."""
+    global CONFIG, CC, sys, plt, rng
+
+    import sys
+
+    import matplotlib.pyplot as plt
+
+    import scitex as stx
+
+    args = parse_args()
+
+    CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
+        sys,
+        plt,
+        args=args,
+        file=__FILE__,
+        sdir_suffix=None,
+        verbose=False,
+        agg=True,
+    )
+
+    exit_status = main(args)
+
+    stx.session.close(
+        CONFIG,
+        verbose=False,
+        notify=False,
+        message="",
+        exit_status=exit_status,
+    )
+
+
+if __name__ == "__main__":
+    run_main()
+
+"""
+Usage:
+
+    # Process BibTeX file with 8 workers
+    python -m scitex.scholar.pipelines.ScholarPipelineBibTeX \
+        --bibtex ./data/scholar/bib_files/neurovista.bib \
+        --project neurovista \
+        --num-workers 8 \
+        --chrome-profile system \
+        --browser-mode stealth
+"""
+
+# EOF