scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/pdf_download/ScholarPDFDownloader.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/pdf_download/ScholarPDFDownloader.py"
-)
+
+__FILE__ = "./src/scitex/scholar/pdf_download/ScholarPDFDownloader.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -184,7 +183,9 @@ class ScholarPDFDownloader:
         if result:
             logger.info(f"{self.name}: Successfully downloaded OA PDF to {result}")
         else:
-            logger.debug(f"{self.name}: OA download failed, may need browser-based download")
+            logger.debug(
+                f"{self.name}: OA download failed, may need browser-based download"
+            )
 
         return result
 
@@ -215,15 +216,15 @@ class ScholarPDFDownloader:
             output_path = Path(str(output_path) + ".pdf")
 
         # Extract metadata
-        meta = paper.metadata if hasattr(paper, 'metadata') else paper
-        access = getattr(meta, 'access', None)
-        url_meta = getattr(meta, 'url', None)
-        id_meta = getattr(meta, 'id', None)
+        meta = paper.metadata if hasattr(paper, "metadata") else paper
+        access = getattr(meta, "access", None)
+        url_meta = getattr(meta, "url", None)
+        id_meta = getattr(meta, "id", None)
 
-        is_open_access = getattr(access, 'is_open_access', False) if access else False
-        oa_url = getattr(access, 'oa_url', None) if access else None
-        pdf_urls = getattr(url_meta, 'pdfs', []) if url_meta else []
-        doi = getattr(id_meta, 'doi', None) if id_meta else None
+        is_open_access = getattr(access, "is_open_access", False) if access else False
+        oa_url = getattr(access, "oa_url", None) if access else None
+        pdf_urls = getattr(url_meta, "pdfs", []) if url_meta else []
+        doi = getattr(id_meta, "doi", None) if id_meta else None
 
         logger.info(f"{self.name}: Smart download for DOI={doi}, OA={is_open_access}")
 
@@ -239,7 +240,7 @@ class ScholarPDFDownloader:
 
         # Strategy 2: Try available PDF URLs
         for pdf_entry in pdf_urls:
-            pdf_url = pdf_entry.get('url') if isinstance(pdf_entry, dict) else pdf_entry
+            pdf_url = pdf_entry.get("url") if isinstance(pdf_entry, dict) else pdf_entry
             if pdf_url:
                 logger.info(f"{self.name}: Trying PDF URL: {pdf_url[:60]}...")
                 result = await self.download_from_url(pdf_url, output_path, doi=doi)
@@ -280,9 +281,7 @@ class ScholarPDFDownloader:
         """
 
         if not pdf_url:
-            logger.warning(
-                f"{self.name}: PDF URL passed but not valid: {pdf_url}"
-            )
+            logger.warning(f"{self.name}: PDF URL passed but not valid: {pdf_url}")
             return None
 
         if isinstance(output_path, str):
@@ -302,9 +301,7 @@ class ScholarPDFDownloader:
         stop_event = asyncio.Event()
 
         # Add manual mode flag to context (shared across all strategies)
-        self.context._scitex_is_manual_mode = (
-            False  # Flag strategies can check
-        )
+        self.context._scitex_is_manual_mode = False  # Flag strategies can check
         self.context._scitex_manual_mode_event = (
             stop_event  # Event for internal monitoring
         )
@@ -333,9 +330,7 @@ class ScholarPDFDownloader:
             )
 
         async def direct_download_wrapper(url, path):
-            return await try_download_direct_async(
-                self.context, url, path, self.name
-            )
+            return await try_download_direct_async(self.context, url, path, self.name)
 
         async def response_body_wrapper(url, path):
             return await try_download_response_body_async(
@@ -398,12 +393,8 @@ class ScholarPDFDownloader:
                         f"{self.name}: {method_name} returned None (failed or not applicable)"
                     )
             except Exception as e:
-                logger.warning(
-                    f"{self.name}: {method_name} raised exception: {e}"
-                )
-                logger.debug(
-                    f"{self.name}: Traceback: {traceback.format_exc()}"
-                )
+                logger.warning(f"{self.name}: {method_name} raised exception: {e}")
+                logger.debug(f"{self.name}: Traceback: {traceback.format_exc()}")
 
         # If user chose manual download or all automation failed
         if stop_event.is_set():
@@ -560,9 +551,10 @@ async def main_async(args):
         auth_manager=auth_manager,
         use_zenrows_proxy=False,
     )
-    browser, context = (
-        await browser_manager.get_authenticated_browser_and_context_async()
-    )
+    (
+        browser,
+        context,
+    ) = await browser_manager.get_authenticated_browser_and_context_async()
 
     # Authentication Gateway
     auth_gateway = AuthenticationGateway(
@@ -581,15 +573,11 @@ async def main_async(args):
     # Use the resolved URL from auth_gateway to avoid duplicate OpenURL resolution
     resolved_url = url_context.url if url_context else None
     if resolved_url:
-        logger.info(
-            f"{__name__}: Using resolved URL from auth_gateway: {resolved_url}"
-        )
+        logger.info(f"{__name__}: Using resolved URL from auth_gateway: {resolved_url}")
         urls = await url_finder.find_pdf_urls(resolved_url)
     else:
         logger.info(f"{__name__}: No resolved URL, using DOI: {args.doi}")
-        urls = await url_finder.find_pdf_urls(
-            args.doi
-        )  # Will resolve DOI internally
+        urls = await url_finder.find_pdf_urls(args.doi)  # Will resolve DOI internally
 
     # Extract URL strings from list of dicts
     pdf_urls = []

scitex/scholar/pdf_download/strategies/chrome_pdf_viewer.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/pdf_download/strategies/chrome_pdf_viewer.py"
-)
+
+__FILE__ = "./src/scitex/scholar/pdf_download/strategies/chrome_pdf_viewer.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -232,9 +231,7 @@ async def try_download_chrome_pdf_viewer_async(
         download_duration = time.time() - download_start_time
 
         logger.info(f"{func_name}: Checking download result...")
-        logger.info(
-            f"{func_name}:  is_downloaded (Playwright): {is_downloaded}"
-        )
+        logger.info(f"{func_name}:  is_downloaded (Playwright): {is_downloaded}")
         logger.info(f"{func_name}:  output_path: {output_path}")
         logger.info(f"{func_name}:  Files before: {len(files_before)}")
         logger.info(f"{func_name}:  Files after: {len(files_after)}")
@@ -246,9 +243,7 @@ async def try_download_chrome_pdf_viewer_async(
             file_size = downloaded_file.stat().st_size
             file_size_mb = file_size / (1024 * 1024)
 
-            logger.info(
-                f"{func_name}: Found downloaded file: {downloaded_file.name}"
-            )
+            logger.info(f"{func_name}: Found downloaded file: {downloaded_file.name}")
             logger.info(f"{func_name}:  Size: {file_size_mb:.2f} MB")
             logger.info(f"{func_name}:  Duration: {download_duration:.1f}s")
             logger.info(f"{func_name}:  Location: {downloaded_file}")
@@ -324,9 +319,7 @@ async def try_download_chrome_pdf_viewer_async(
                 await page.close()
                 return output_path
 
-        await browser_logger.warning(
-            page, f"{func_name}: ✗ Download did not complete"
-        )
+        await browser_logger.warning(page, f"{func_name}: ✗ Download did not complete")
         logger.warning(
             f"{func_name}: Download did not complete (is_downloaded={is_downloaded}, exists={output_path.exists()})"
         )
@@ -348,7 +341,9 @@ async def try_download_chrome_pdf_viewer_async(
 
     except Exception as ee:
         # Log error safely without browser popup (avoids recursive errors)
-        error_msg = f"{func_name}: Chrome PDF Viewer failed: {type(ee).__name__}: {str(ee)}"
+        error_msg = (
+            f"{func_name}: Chrome PDF Viewer failed: {type(ee).__name__}: {str(ee)}"
+        )
         logger.error(error_msg)
         logger.debug(f"  URL: {pdf_url}")
         logger.debug(f"  Output: {output_path}")
@@ -361,16 +356,13 @@ async def try_download_chrome_pdf_viewer_async(
                 )
                 await page.wait_for_timeout(2000)  # Show error for 2s
             except Exception as popup_error:
-                logger.debug(
-                    f"{func_name}: Could not show error popup: {popup_error}"
-                )
+                logger.debug(f"{func_name}: Could not show error popup: {popup_error}")
             finally:
                 try:
                     await page.close()
                 except Exception as close_error:
-                    logger.debug(
-                        f"{func_name}: Error closing page: {close_error}"
-                    )
+                    logger.debug(f"{func_name}: Error closing page: {close_error}")
         return None
 
+
 # EOF

scitex/scholar/pdf_download/strategies/direct_download.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/pdf_download/strategies/direct_download.py"
-)
+
+__FILE__ = "./src/scitex/scholar/pdf_download/strategies/direct_download.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 """Direct Download Strategy"""
@@ -116,16 +115,13 @@ async def try_download_direct_async(
                 )
                 await page.wait_for_timeout(2000)
             except Exception as popup_error:
-                logger.debug(
-                    f"{func_name}: Could not show error popup: {popup_error}"
-                )
+                logger.debug(f"{func_name}: Could not show error popup: {popup_error}")
             finally:
                 try:
                     await page.close()
                 except Exception as close_error:
-                    logger.debug(
-                        f"{func_name}: Error closing page: {close_error}"
-                    )
+                    logger.debug(f"{func_name}: Error closing page: {close_error}")
         return None
 
+
 # EOF

scitex/scholar/pdf_download/strategies/manual_download_fallback.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/pdf_download/strategies/manual_download_fallback.py"
-)
+
+__FILE__ = "./src/scitex/scholar/pdf_download/strategies/manual_download_fallback.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 """Manual Download Fallback Strategy"""
@@ -164,4 +163,5 @@ async def try_download_manual_async(
                 pass
         return None
 
+
 # EOF

scitex/scholar/pdf_download/strategies/manual_download_utils.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/pdf_download/strategies/manual_download_utils.py"
-)
+
+__FILE__ = "./src/scitex/scholar/pdf_download/strategies/manual_download_utils.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -78,9 +77,7 @@ class FlexibleFilenameGenerator:
             from urllib.parse import urlparse
 
             parsed = urlparse(url)
-            base = f"{parsed.netloc}_{parsed.path}".replace("/", "_").replace(
-                ".", "_"
-            )
+            base = f"{parsed.netloc}_{parsed.path}".replace("/", "_").replace(".", "_")
             base = re.sub(r'[<>:"|?*]', "", base)
         else:
             # Last resort: timestamp-based
@@ -194,10 +191,7 @@ class DownloadMonitorAndSync:
                 return None
 
             # Report progress periodically
-            if (
-                logger_func
-                and (elapsed - last_progress_time) >= progress_interval
-            ):
+            if logger_func and (elapsed - last_progress_time) >= progress_interval:
                 current_file_count = len(self._get_current_files())
                 logger_func(
                     f"{self.name}: Still waiting for download... ({remaining:.0f}s remaining, "
@@ -571,9 +565,7 @@ async def show_stop_automation_button_async(
         )
 
     except Exception as e:
-        logger.error(
-            f"show_stop_automation_button_async: Failed to inject button: {e}"
-        )
+        logger.error(f"show_stop_automation_button_async: Failed to inject button: {e}")
         return
 
     # Wait for DOUBLE-CLICK (no timeout - always available)
@@ -993,4 +985,5 @@ async def wait_for_manual_mode_activation_async(
     except Exception as e:
         pass
 
+
 # EOF

scitex/scholar/pdf_download/strategies/open_access_download.py

@@ -19,25 +19,29 @@ logger = logging.getLogger(__name__)
 
 # Known OA source patterns and their handlers
 OA_SOURCE_PATTERNS = {
-    'arxiv': {
-        'patterns': ['arxiv.org'],
-        'pdf_transform': lambda url: url.replace('/abs/', '/pdf/') + '.pdf' if '/abs/' in url else url,
+    "arxiv": {
+        "patterns": ["arxiv.org"],
+        "pdf_transform": lambda url: url.replace("/abs/", "/pdf/") + ".pdf"
+        if "/abs/" in url
+        else url,
     },
-    'pmc': {
-        'patterns': ['ncbi.nlm.nih.gov/pmc', 'europepmc.org'],
-        'pdf_transform': lambda url: url,  # PMC links are usually direct
+    "pmc": {
+        "patterns": ["ncbi.nlm.nih.gov/pmc", "europepmc.org"],
+        "pdf_transform": lambda url: url,  # PMC links are usually direct
     },
-    'biorxiv': {
-        'patterns': ['biorxiv.org', 'medrxiv.org'],
-        'pdf_transform': lambda url: url + '.full.pdf' if not url.endswith('.pdf') else url,
+    "biorxiv": {
+        "patterns": ["biorxiv.org", "medrxiv.org"],
+        "pdf_transform": lambda url: url + ".full.pdf"
+        if not url.endswith(".pdf")
+        else url,
     },
-    'doaj': {
-        'patterns': ['doaj.org'],
-        'pdf_transform': lambda url: url,
+    "doaj": {
+        "patterns": ["doaj.org"],
+        "pdf_transform": lambda url: url,
     },
-    'zenodo': {
-        'patterns': ['zenodo.org'],
-        'pdf_transform': lambda url: url,
+    "zenodo": {
+        "patterns": ["zenodo.org"],
+        "pdf_transform": lambda url: url,
     },
 }
 
@@ -46,7 +50,7 @@ def _identify_oa_source(url: str) -> Optional[str]:
     """Identify which OA source a URL belongs to."""
     url_lower = url.lower()
     for source_name, config in OA_SOURCE_PATTERNS.items():
-        for pattern in config['patterns']:
+        for pattern in config["patterns"]:
             if pattern in url_lower:
                 return source_name
     return None
@@ -55,7 +59,7 @@ def _identify_oa_source(url: str) -> Optional[str]:
 def _transform_to_pdf_url(url: str, source: str) -> str:
     """Transform URL to direct PDF URL based on source."""
     if source in OA_SOURCE_PATTERNS:
-        transform_func = OA_SOURCE_PATTERNS[source]['pdf_transform']
+        transform_func = OA_SOURCE_PATTERNS[source]["pdf_transform"]
         return transform_func(url)
     return url
 
@@ -91,7 +95,9 @@ async def try_download_open_access_async(
     source = _identify_oa_source(oa_url)
     pdf_url = _transform_to_pdf_url(oa_url, source) if source else oa_url
 
-    logger.info(f"{func_name}: Attempting OA download from {source or 'unknown'}: {pdf_url[:80]}...")
+    logger.info(
+        f"{func_name}: Attempting OA download from {source or 'unknown'}: {pdf_url[:80]}..."
+    )
 
     try:
         # Create output directory if needed
@@ -101,23 +107,29 @@ async def try_download_open_access_async(
         # Use aiohttp for async download
         async with aiohttp.ClientSession() as session:
             headers = {
-                'User-Agent': 'SciTeX/1.0 (Academic Research Tool; mailto:contact@scitex.io)',
-                'Accept': 'application/pdf,*/*',
+                "User-Agent": "SciTeX/1.0 (Academic Research Tool; mailto:contact@scitex.io)",
+                "Accept": "application/pdf,*/*",
             }
 
-            async with session.get(pdf_url, headers=headers, timeout=aiohttp.ClientTimeout(total=timeout)) as response:
+            async with session.get(
+                pdf_url, headers=headers, timeout=aiohttp.ClientTimeout(total=timeout)
+            ) as response:
                 if response.status != 200:
-                    logger.warning(f"{func_name}: HTTP {response.status} from {pdf_url}")
+                    logger.warning(
+                        f"{func_name}: HTTP {response.status} from {pdf_url}"
+                    )
                     return None
 
-                content_type = response.headers.get('Content-Type', '')
+                content_type = response.headers.get("Content-Type", "")
 
                 # Verify we're getting a PDF
-                if 'pdf' not in content_type.lower() and not pdf_url.endswith('.pdf'):
+                if "pdf" not in content_type.lower() and not pdf_url.endswith(".pdf"):
                     # Some servers don't set content-type correctly, check magic bytes
                     first_bytes = await response.content.read(5)
-                    if first_bytes != b'%PDF-':
-                        logger.warning(f"{func_name}: Response is not a PDF (content-type: {content_type})")
+                    if first_bytes != b"%PDF-":
+                        logger.warning(
+                            f"{func_name}: Response is not a PDF (content-type: {content_type})"
+                        )
                         return None
                     # Reset for full download
                     content = first_bytes + await response.content.read()
@@ -126,19 +138,25 @@ async def try_download_open_access_async(
 
                 # Validate PDF content
                 if len(content) < 1000:  # PDF should be at least 1KB
-                    logger.warning(f"{func_name}: Downloaded content too small ({len(content)} bytes)")
+                    logger.warning(
+                        f"{func_name}: Downloaded content too small ({len(content)} bytes)"
+                    )
                     return None
 
-                if not content.startswith(b'%PDF-'):
-                    logger.warning(f"{func_name}: Downloaded content is not a valid PDF")
+                if not content.startswith(b"%PDF-"):
+                    logger.warning(
+                        f"{func_name}: Downloaded content is not a valid PDF"
+                    )
                     return None
 
                 # Save to file
-                with open(output_path, 'wb') as f:
+                with open(output_path, "wb") as f:
                     f.write(content)
 
                 size_mb = len(content) / 1024 / 1024
-                logger.info(f"{func_name}: Successfully downloaded {size_mb:.2f} MB to {output_path}")
+                logger.info(
+                    f"{func_name}: Successfully downloaded {size_mb:.2f} MB to {output_path}"
+                )
                 return output_path
 
     except aiohttp.ClientError as e:

scitex/scholar/pdf_download/strategies/response_body.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/pdf_download/strategies/response_body.py"
-)
+
+__FILE__ = "./src/scitex/scholar/pdf_download/strategies/response_body.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -32,10 +31,7 @@ async def try_download_response_body_async(
 ) -> Optional[Path]:
     """Download PDF from HTTP response body."""
     # Check if manual mode is active - skip immediately
-    if (
-        hasattr(context, "_scitex_is_manual_mode")
-        and context._scitex_is_manual_mode
-    ):
+    if hasattr(context, "_scitex_is_manual_mode") and context._scitex_is_manual_mode:
         logger.info(f"{func_name}: Skipping because manual mode is active...")
         return None
 
@@ -63,9 +59,7 @@ async def try_download_response_body_async(
                 hasattr(context, "_scitex_is_manual_mode")
                 and context._scitex_is_manual_mode
             ):
-                logger.info(
-                    f"{func_name}: Ignoring download (manual mode active)"
-                )
+                logger.info(f"{func_name}: Ignoring download (manual mode active)")
                 return
 
             if download_handler_active:
@@ -90,9 +84,7 @@ async def try_download_response_body_async(
             ):
                 logger.info(f"{func_name}: Manual mode activated, aborting")
                 download_handler_active = False  # Disable handler
-                page.remove_listener(
-                    "download", handle_download
-                )  # Remove listener
+                page.remove_listener("download", handle_download)  # Remove listener
                 await page.close()
                 return None
 
@@ -192,16 +184,13 @@ async def try_download_response_body_async(
                 )
                 await page.wait_for_timeout(2000)
             except Exception as popup_error:
-                logger.debug(
-                    f"{func_name}: Could not show error popup: {popup_error}"
-                )
+                logger.debug(f"{func_name}: Could not show error popup: {popup_error}")
             finally:
                 try:
                     await page.close()
                 except Exception as close_error:
-                    logger.debug(
-                        f"{func_name}: Error closing page: {close_error}"
-                    )
+                    logger.debug(f"{func_name}: Error closing page: {close_error}")
         return None
 
+
 # EOF

scitex/scholar/pipelines/ScholarPipelineBibTeX.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/pipelines/ScholarPipelineBibTeX.py"
-)
+
+__FILE__ = "./src/scitex/scholar/pipelines/ScholarPipelineBibTeX.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -50,6 +49,8 @@ from scitex.scholar.storage import BibTeXHandler
 logger = logging.getLogger(__name__)
 
 """Functions & Classes"""
+
+
 class ScholarPipelineBibTeX:
     """Processes BibTeX files through parallel paper acquisition pipeline"""
 
@@ -144,9 +145,7 @@ class ScholarPipelineBibTeX:
         logger.success(
             f"{self.name}: Processed {len(processed_papers)}/{len(papers)} papers"
         )
-        logger.success(
-            f"{self.name}: Saved enriched BibTeX: {output_bibtex_path}"
-        )
+        logger.success(f"{self.name}: Saved enriched BibTeX: {output_bibtex_path}")
 
         # Update project bibliography if project specified
         if project:
@@ -162,9 +161,7 @@ class ScholarPipelineBibTeX:
                     bibtex_files=[bibtex_path, output_bibtex_path],
                 )
 
-                logger.success(
-                    f"{self.name}: Updated project bibliography: {project}"
-                )
+                logger.success(f"{self.name}: Updated project bibliography: {project}")
             except Exception as e:
                 logger.warning(f"Failed to update bibliography: {e}")
 
@@ -196,9 +193,7 @@ class ScholarPipelineBibTeX:
             logger.warning(f"{self.name}: No papers found in BibTeX text")
             return Papers([], project=project)
 
-        logger.info(
-            f"{self.name}: Loaded {len(papers)} papers from BibTeX text"
-        )
+        logger.info(f"{self.name}: Loaded {len(papers)} papers from BibTeX text")
 
         # Step 2: Process papers in parallel
         papers_collection = Papers(papers, project=project)
@@ -218,9 +213,7 @@ class ScholarPipelineBibTeX:
                 processed_collection,
                 output_path=output_bibtex_path,
             )
-            logger.success(
-                f"{self.name}: Saved enriched BibTeX: {output_bibtex_path}"
-            )
+            logger.success(f"{self.name}: Saved enriched BibTeX: {output_bibtex_path}")
 
         logger.success(
             f"{self.name}: Processed {len(processed_papers)}/{len(papers)} papers"
@@ -261,9 +254,7 @@ def main(args):
         )
     )
 
-    logger.success(
-        f"BibTeX processing complete: {len(papers)} papers processed"
-    )
+    logger.success(f"BibTeX processing complete: {len(papers)} papers processed")
     return 0