scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/browser/utils/wait_redirects.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/browser/utils/wait_redirects.py"
-)
+
+__FILE__ = "./src/scitex/scholar/browser/utils/wait_redirects.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -243,7 +242,7 @@ async def wait_redirects(
     if show_progress:
         await browser_logger.info(
             page,
-            f"{func_name}: Waiting for redirects (max {timeout/1000:.0f}s)...",
+            f"{func_name}: Waiting for redirects (max {timeout / 1000:.0f}s)...",
             duration_ms=timeout,
         )
 
@@ -347,9 +346,7 @@ async def wait_redirects(
         if 300 <= status < 400:
             redirect_count += 1
             if redirect_count >= max_redirects:
-                logger.warning(
-                    f"{func_name}: Max redirects ({max_redirects}) reached"
-                )
+                logger.warning(f"{func_name}: Max redirects ({max_redirects}) reached")
                 navigation_complete.set()
 
         elif 200 <= status < 300:
@@ -425,9 +422,7 @@ async def wait_redirects(
 
                 # Check page load state
                 try:
-                    load_state = await page.evaluate(
-                        "() => document.readyState"
-                    )
+                    load_state = await page.evaluate("() => document.readyState")
                     page_loaded = load_state == "complete"
                 except:
                     page_loaded = False
@@ -457,9 +452,7 @@ async def wait_redirects(
 
                 # Check if URL changed
                 if current_url != last_checked_url:
-                    logger.debug(
-                        f"{func_name}: URL changed: {current_url[:80]}"
-                    )
+                    logger.debug(f"{func_name}: URL changed: {current_url[:80]}")
                     last_checked_url = current_url
                     stable_count = 0
                     dom_stable_count = 0
@@ -470,9 +463,7 @@ async def wait_redirects(
                         logger.info(
                             f"{func_name}: Article URL detected: {current_url[:80]}"
                         )
-                        await asyncio.sleep(
-                            1
-                        )  # Short wait for final resources
+                        await asyncio.sleep(1)  # Short wait for final resources
                         navigation_complete.set()
                         break
                 else:
@@ -484,9 +475,7 @@ async def wait_redirects(
                 # CAPTCHA path: Wait much longer for CAPTCHA solver extension
                 if captcha_detected:
                     captcha_wait_time = (
-                        current_time - captcha_wait_start
-                        if captcha_wait_start
-                        else 0
+                        current_time - captcha_wait_start if captcha_wait_start else 0
                     )
 
                     # Check if CAPTCHA is still present
@@ -509,9 +498,7 @@ async def wait_redirects(
                                     duration_ms=2000,
                                 )
                             )
-                        await asyncio.sleep(
-                            3
-                        )  # Wait for redirect after CAPTCHA
+                        await asyncio.sleep(3)  # Wait for redirect after CAPTCHA
                         captcha_detected = False
                         captcha_wait_start = None
                         stable_count = 0
@@ -520,10 +507,7 @@ async def wait_redirects(
 
                     # Give CAPTCHA solver up to 60 seconds
                     if captcha_wait_time < 60:
-                        if (
-                            int(captcha_wait_time) % 10 == 0
-                            and captcha_wait_time > 0
-                        ):
+                        if int(captcha_wait_time) % 10 == 0 and captcha_wait_time > 0:
                             logger.info(
                                 f"{func_name}: CAPTCHA solver working... ({int(60 - captcha_wait_time)}s remaining)"
                             )
@@ -563,9 +547,7 @@ async def wait_redirects(
                 # Medium path: URL and network stable for longer
                 elif stable_count >= 3 and time_since_activity >= 3:
                     if not is_auth_endpoint(current_url) or found_article:
-                        logger.debug(
-                            f"{func_name}: Complete: URL+network stable (3s)"
-                        )
+                        logger.debug(f"{func_name}: Complete: URL+network stable (3s)")
                         navigation_complete.set()
                         break
 
@@ -608,14 +590,10 @@ async def wait_redirects(
     try:
         # Wait for navigation to complete
         try:
-            await asyncio.wait_for(
-                navigation_complete.wait(), timeout=timeout / 1000
-            )
+            await asyncio.wait_for(navigation_complete.wait(), timeout=timeout / 1000)
         except asyncio.TimeoutError:
             timed_out = True
-            logger.warning(
-                f"{func_name}: Redirect wait timeout after {timeout}ms"
-            )
+            logger.warning(f"{func_name}: Redirect wait timeout after {timeout}ms")
             if show_progress:
                 await browser_logger.info(
                     page,
@@ -631,9 +609,7 @@ async def wait_redirects(
         if wait_for_idle and not timed_out:
             try:
                 idle_timeout = min(5000, timeout // 4)
-                await page.wait_for_load_state(
-                    "networkidle", timeout=idle_timeout
-                )
+                await page.wait_for_load_state("networkidle", timeout=idle_timeout)
             except:
                 logger.debug(f"{func_name}: Network idle wait failed")
 
@@ -656,8 +632,7 @@ async def wait_redirects(
             "total_time_ms": round(total_time_ms, 2),
             "timed_out": timed_out,
             "found_article": found_article,
-            "stopped_at_auth": is_auth_endpoint(final_url)
-            and not found_article,
+            "stopped_at_auth": is_auth_endpoint(final_url) and not found_article,
         }
 
         if track_chain:

scitex/scholar/citation_graph/builder.py

@@ -92,9 +92,7 @@ class CitationGraphBuilder:
 
             return graph
 
-    def _create_paper_node(
-        self, doi: str, similarity_score: float
-    ) -> PaperNode:
+    def _create_paper_node(self, doi: str, similarity_score: float) -> PaperNode:
         """
         Create a PaperNode with metadata from database.
 
@@ -111,8 +109,7 @@ class CitationGraphBuilder:
             # Extract author names
             authors = metadata.get("author", [])
             author_names = [
-                f"{a.get('family', '')} {a.get('given', '')[:1]}"
-                for a in authors[:3]
+                f"{a.get('family', '')} {a.get('given', '')[:1]}" for a in authors[:3]
             ]
 
             # Extract year
@@ -202,8 +199,7 @@ class CitationGraphBuilder:
             return {
                 "doi": doi,
                 "title": metadata.get("title", ["Unknown"])[0],
-                "year": metadata.get("published", {})
-                .get("date-parts", [[0]])[0][0],
+                "year": metadata.get("published", {}).get("date-parts", [[0]])[0][0],
                 "authors": [
                     f"{a.get('family', '')} {a.get('given', '')}"
                     for a in metadata.get("author", [])[:5]

scitex/scholar/citation_graph/database.py

@@ -47,13 +47,10 @@ class CitationDatabase:
             self.conn = sqlite3.connect(
                 f"file:{self.db_path}?mode=ro",
                 uri=True,
-                check_same_thread=False  # Allow multi-threaded access (e.g., Django)
+                check_same_thread=False,  # Allow multi-threaded access (e.g., Django)
             )
         else:
-            self.conn = sqlite3.connect(
-                self.db_path,
-                check_same_thread=False
-            )
+            self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
 
         self.conn.row_factory = sqlite3.Row
 
@@ -117,9 +114,7 @@ class CitationDatabase:
         )
         return [(row[0], row[1]) for row in cursor]
 
-    def get_cocited_papers(
-        self, doi: str, limit: int = 50
-    ) -> List[Tuple[str, int]]:
+    def get_cocited_papers(self, doi: str, limit: int = 50) -> List[Tuple[str, int]]:
         """
         Find papers co-cited with this DOI.
 
@@ -187,9 +182,7 @@ class CitationDatabase:
         Returns:
             Dictionary with paper metadata, or None if not found
         """
-        cursor = self.conn.execute(
-            "SELECT metadata FROM works WHERE doi = ?", (doi,)
-        )
+        cursor = self.conn.execute("SELECT metadata FROM works WHERE doi = ?", (doi,))
         row = cursor.fetchone()
 
         if row:

scitex/scholar/citation_graph/example.py

@@ -24,9 +24,9 @@ def main():
         print("Please update the db_path in this script.")
         return 1
 
-    print("="*70)
+    print("=" * 70)
     print("  Citation Graph Example")
-    print("="*70)
+    print("=" * 70)
     print(f"\nDatabase: {db_path}")
 
     # Initialize builder
@@ -61,20 +61,15 @@ def main():
     # Show top papers by similarity
     print(f"\nTop 10 most similar papers:")
     print(f"{'Rank':<5} {'Score':<7} {'Year':<6} {'Title':<60}")
-    print("-"*85)
+    print("-" * 85)
 
-    sorted_nodes = sorted(
-        graph.nodes,
-        key=lambda n: n.similarity_score,
-        reverse=True
-    )
+    sorted_nodes = sorted(graph.nodes, key=lambda n: n.similarity_score, reverse=True)
 
     for i, node in enumerate(sorted_nodes[:11], 1):
         if node.doi.lower() == seed_doi.lower():
             continue
         print(
-            f"{i:<5} {node.similarity_score:<7.1f} "
-            f"{node.year:<6} {node.title[:60]:<60}"
+            f"{i:<5} {node.similarity_score:<7.1f} {node.year:<6} {node.title[:60]:<60}"
         )
 
     # Export to JSON

scitex/scholar/cli/_url_utils.py

@@ -127,7 +127,7 @@ def get_best_url(
     openurl_resolved: Optional[list] = None,
     url_publisher: Optional[str] = None,
     url_doi: Optional[str] = None,
-    doi: Optional[str] = None
+    doi: Optional[str] = None,
 ) -> Optional[str]:
     """Get the best available URL from multiple sources.
 

scitex/scholar/cli/chrome.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -59,9 +60,10 @@ async def main_async():
         auth_manager=auth_manager,
     )
 
-    browser, context = (
-        await browser_manager.get_authenticated_browser_and_context_async()
-    )
+    (
+        browser,
+        context,
+    ) = await browser_manager.get_authenticated_browser_and_context_async()
     page = await context.new_page()
 
     print(f"args.url: {args.url}")

scitex/scholar/cli/download_pdf.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -71,9 +72,7 @@ python -m scitex.scholar download info""",
     paper_group = paper_parser.add_mutually_exclusive_group(required=True)
     paper_group.add_argument("--doi", help="DOI of the paper")
     paper_group.add_argument("--url", help="URL of the paper")
-    paper_parser.add_argument(
-        "--title", help="Paper title (for DOI-based downloads)"
-    )
+    paper_parser.add_argument("--title", help="Paper title (for DOI-based downloads)")
     paper_parser.add_argument(
         "--project",
         default="default",
@@ -128,7 +127,7 @@ def main():
 
             print(f"\n✅ Downloaded: {results['downloaded']} PDFs")
             print(f"❌ Failed: {results['failed']}")
-            if results.get('errors'):
+            if results.get("errors"):
                 print(f"⚠️  Errors: {results['errors']}")
 
             return 0
@@ -136,6 +135,7 @@ def main():
         except Exception as e:
             print(f"❌ Error: {e}")
             import traceback
+
             traceback.print_exc()
             return 1
 
@@ -171,7 +171,10 @@ def main():
                     auth_manager=ScholarAuthManager(),
                     use_zenrows_proxy=False,
                 )
-                browser, context = await browser_manager.get_authenticated_browser_and_context_async()
+                (
+                    browser,
+                    context,
+                ) = await browser_manager.get_authenticated_browser_and_context_async()
 
                 # Create downloader with authenticated context
                 downloader = ScholarPDFDownloader(context, config=ScholarConfig())
@@ -179,8 +182,7 @@ def main():
                 # Download based on DOI or URL
                 if args.doi:
                     saved_paths = await downloader.download_from_doi(
-                        args.doi,
-                        output_dir=f"/tmp/scholar_downloads/{args.project}/"
+                        args.doi, output_dir=f"/tmp/scholar_downloads/{args.project}/"
                     )
                     success = bool(saved_paths)
                     pdf_path = saved_paths[0] if saved_paths else None
@@ -188,7 +190,7 @@ def main():
                     title = paper.metadata.basic.title or "untitled"
                     pdf_path = await downloader.download_from_url(
                         args.url,
-                        output_path=f"/tmp/scholar_downloads/{args.project}/{title[:30]}.pdf"
+                        output_path=f"/tmp/scholar_downloads/{args.project}/{title[:30]}.pdf",
                     )
                     success = bool(pdf_path)
 
@@ -203,6 +205,7 @@ def main():
             except Exception as e:
                 print(f"❌ Error: {e}")
                 import traceback
+
                 traceback.print_exc()
                 return 1
 
@@ -217,13 +220,9 @@ def main():
         print(f"Extensions: Accept Cookies, Zotero Connector")
         print(f"Zotero Translators: Enabled")
         print(f"=" * 50)
-        print(
-            f"\n💡 This tool specializes in accessing paywalled academic content"
-        )
+        print(f"\n💡 This tool specializes in accessing paywalled academic content")
         print(f"   that requires institutional authentication.")
-        print(
-            f"\n🏆 Competitive Advantage: Access content others can't reach!"
-        )
+        print(f"\n🏆 Competitive Advantage: Access content others can't reach!")
     else:
         parser.print_help()
         return 1

scitex/scholar/cli/handlers/bibtex_handler.py

@@ -26,9 +26,7 @@ async def handle_bibtex_operations(args, scholar):
     # Warn if using both enrich and download together
     if args.enrich and (args.download or args.download_force):
         logger.warning("Using --enrich and --download together")
-        logger.warning(
-            "RECOMMENDED: Run as two separate steps for better reliability:"
-        )
+        logger.warning("RECOMMENDED: Run as two separate steps for better reliability:")
         logger.warning(
             "  Step 1: python -m scitex.scholar --bibtex input.bib --output enriched.bib --project PROJECT --enrich"
         )
@@ -75,9 +73,7 @@ async def handle_bibtex_operations(args, scholar):
             output_path = Path(args.output)
         else:
             # Auto-generate enriched filename
-            output_path = (
-                bibtex_path.parent / f"{bibtex_path.stem}_enriched.bib"
-            )
+            output_path = bibtex_path.parent / f"{bibtex_path.stem}_enriched.bib"
 
         scholar.save_papers_as_bibtex(papers, output_path)
         logger.success(f"Saved enriched BibTeX to: {output_path}")
@@ -86,9 +82,7 @@ async def handle_bibtex_operations(args, scholar):
     if args.project:
         logger.info(f"Saving to project: {args.project}")
         saved_ids = scholar.save_papers_to_library(papers)
-        logger.info(
-            f"Saved {len(saved_ids)} papers to library with symlinks created"
-        )
+        logger.info(f"Saved {len(saved_ids)} papers to library with symlinks created")
 
     # Download PDFs if requested (after library save so symlinks exist)
     if args.download:
@@ -140,9 +134,7 @@ async def handle_bibtex_operations(args, scholar):
         # Create/update merged.bib with all BibTeX files in the project
         from scitex.scholar.storage.BibTeXHandler import BibTeXHandler
 
-        bibtex_handler = BibTeXHandler(
-            project=args.project, config=scholar.config
-        )
+        bibtex_handler = BibTeXHandler(project=args.project, config=scholar.config)
 
         # Get all BibTeX files in the project directory
         bibtex_files = list(project_bibtex_dir.glob("*.bib"))

scitex/scholar/cli/handlers/doi_handler.py

@@ -29,9 +29,7 @@ async def handle_doi_operations(args, scholar):
     # Download PDFs if requested
     if args.download:
         results = await scholar.download_pdfs_from_dois_async(dois)
-        logger.info(
-            f"Downloaded: {results['downloaded']}, Failed: {results['failed']}"
-        )
+        logger.info(f"Downloaded: {results['downloaded']}, Failed: {results['failed']}")
 
     # Enrich if requested (create Papers from DOIs first)
     if args.enrich:

scitex/scholar/cli/handlers/project_handler.py

@@ -34,9 +34,7 @@ async def handle_project_operations(args, scholar):
         if args.has_pdf is False:
             cmd.append("--all")
 
-        logger.info(
-            f"Opening browser with auto-tracking for project: {args.project}"
-        )
+        logger.info(f"Opening browser with auto-tracking for project: {args.project}")
         result = subprocess.run(cmd)
         return result.returncode
 
@@ -52,18 +50,12 @@ async def handle_project_operations(args, scholar):
             # Check if DOI exists (non-empty string)
             if paper.metadata.id.doi and paper.metadata.id.doi.strip():
                 # Check if PDF already exists
-                has_pdf = (
-                    paper.metadata.path.pdfs
-                    and len(paper.metadata.path.pdfs) > 0
-                )
+                has_pdf = paper.metadata.path.pdfs and len(paper.metadata.path.pdfs) > 0
 
                 # Download if: no PDF OR download_force flag is set
                 if not has_pdf or args.download_force:
                     dois_to_download.append(paper.metadata.id.doi)
-            elif (
-                not paper.metadata.path.pdfs
-                or len(paper.metadata.path.pdfs) == 0
-            ):
+            elif not paper.metadata.path.pdfs or len(paper.metadata.path.pdfs) == 0:
                 # Paper has no DOI and no PDF - mark as failed with explanation
                 from scitex.scholar.storage._LibraryManager import (
                     LibraryManager,
@@ -74,9 +66,7 @@ async def handle_project_operations(args, scholar):
                 )
 
                 paper_id = paper.container.scitex_id
-                master_dir = (
-                    scholar.config.path_manager.get_library_master_dir()
-                )
+                master_dir = scholar.config.path_manager.get_library_master_dir()
                 paper_dir = master_dir / paper_id
                 paper_dir.mkdir(parents=True, exist_ok=True)
 
@@ -118,9 +108,7 @@ async def handle_project_operations(args, scholar):
                 logger.info(
                     f"Downloading PDFs for {len(dois_to_download)} papers without PDFs..."
                 )
-            results = await scholar.download_pdfs_from_dois_async(
-                dois_to_download
-            )
+            results = await scholar.download_pdfs_from_dois_async(dois_to_download)
             logger.info(
                 f"Download complete: {results['downloaded']} downloaded, {results['failed']} failed"
             )
@@ -257,9 +245,7 @@ async def handle_project_operations(args, scholar):
             import json
 
             with open(output_path, "w") as f:
-                json.dump(
-                    [p.to_dict() for p in papers], f, indent=2, default=str
-                )
+                json.dump([p.to_dict() for p in papers], f, indent=2, default=str)
         else:
             logger.error(f"Unsupported export format: {extension}")
             logger.info(

scitex/scholar/cli/open_browser.py

@@ -74,15 +74,17 @@ def get_failed_papers(project: str, config: ScholarConfig) -> List[Dict]:
         if not pdf_files and has_screenshots:
             # Failed download
             meta = metadata.get("metadata", {})
-            failed_papers.append({
-                "paper_id": paper_dir.name,
-                "doi": meta.get("id", {}).get("doi"),
-                "title": meta.get("basic", {}).get("title"),
-                "url_doi": meta.get("url", {}).get("doi"),
-                "url_publisher": meta.get("url", {}).get("publisher"),
-                "openurl_resolved": meta.get("url", {}).get("openurl_resolved", []),
-                "pdf_urls": meta.get("url", {}).get("pdfs", []),
-            })
+            failed_papers.append(
+                {
+                    "paper_id": paper_dir.name,
+                    "doi": meta.get("id", {}).get("doi"),
+                    "title": meta.get("basic", {}).get("title"),
+                    "url_doi": meta.get("url", {}).get("doi"),
+                    "url_publisher": meta.get("url", {}).get("publisher"),
+                    "openurl_resolved": meta.get("url", {}).get("openurl_resolved", []),
+                    "pdf_urls": meta.get("url", {}).get("pdfs", []),
+                }
+            )
 
     return failed_papers
 
@@ -125,23 +127,23 @@ def get_pending_papers(project: str, config: ScholarConfig) -> List[Dict]:
         if not pdf_files and not has_screenshots:
             # Pending download
             meta = metadata.get("metadata", {})
-            pending_papers.append({
-                "paper_id": paper_dir.name,
-                "doi": meta.get("id", {}).get("doi"),
-                "title": meta.get("basic", {}).get("title"),
-                "url_doi": meta.get("url", {}).get("doi"),
-                "url_publisher": meta.get("url", {}).get("publisher"),
-                "openurl_resolved": meta.get("url", {}).get("openurl_resolved", []),
-                "pdf_urls": meta.get("url", {}).get("pdfs", []),
-            })
+            pending_papers.append(
+                {
+                    "paper_id": paper_dir.name,
+                    "doi": meta.get("id", {}).get("doi"),
+                    "title": meta.get("basic", {}).get("title"),
+                    "url_doi": meta.get("url", {}).get("doi"),
+                    "url_publisher": meta.get("url", {}).get("publisher"),
+                    "openurl_resolved": meta.get("url", {}).get("openurl_resolved", []),
+                    "pdf_urls": meta.get("url", {}).get("pdfs", []),
+                }
+            )
 
     return pending_papers
 
 
 def open_browser_with_urls(
-    papers: List[Dict],
-    profile: str = None,
-    headless: bool = False
+    papers: List[Dict], profile: str = None, headless: bool = False
 ) -> None:
     """Open browser with URL tabs for papers.
 
@@ -163,13 +165,15 @@ def open_browser_with_urls(
             openurl_resolved=paper.get("openurl_resolved"),
             url_publisher=paper.get("url_publisher"),
             url_doi=paper.get("url_doi"),
-            doi=paper.get("doi")
+            doi=paper.get("doi"),
         )
 
         if url:
             urls_to_open.append(url)
         else:
-            logger.warning(f"No valid URL found for paper: {paper.get('title', 'Unknown')[:50]}...")
+            logger.warning(
+                f"No valid URL found for paper: {paper.get('title', 'Unknown')[:50]}..."
+            )
 
     if not urls_to_open:
         logger.warning("No URLs to open")
@@ -179,6 +183,7 @@ def open_browser_with_urls(
 
     # Get cache directory for browser profile
     from scitex.scholar.config import ScholarConfig
+
     config = ScholarConfig()
 
     if profile:
@@ -194,9 +199,11 @@ def open_browser_with_urls(
             str(profile_dir),
             headless=headless,
             args=[
-                '--disable-blink-features=AutomationControlled',
-                '--disable-features=UserAgentClientHint',
-            ] if not headless else []
+                "--disable-blink-features=AutomationControlled",
+                "--disable-features=UserAgentClientHint",
+            ]
+            if not headless
+            else [],
         )
 
         # Open first URL in first tab
@@ -233,28 +240,21 @@ def main():
         description="Open browser with failed/pending PDF URLs for manual download"
     )
     parser.add_argument(
-        "--project",
-        required=True,
-        help="Project name (e.g., neurovista, pac)"
+        "--project", required=True, help="Project name (e.g., neurovista, pac)"
     )
     parser.add_argument(
-        "--all",
-        action="store_true",
-        help="Open both failed and pending PDFs"
+        "--all", action="store_true", help="Open both failed and pending PDFs"
     )
     parser.add_argument(
         "--pending",
         action="store_true",
-        help="Open only pending (not attempted) PDFs instead of failed"
+        help="Open only pending (not attempted) PDFs instead of failed",
     )
     parser.add_argument(
-        "--profile",
-        help="Browser profile name to use (default: system)"
+        "--profile", help="Browser profile name to use (default: system)"
     )
     parser.add_argument(
-        "--headless",
-        action="store_true",
-        help="Run in headless mode (for testing)"
+        "--headless", action="store_true", help="Run in headless mode (for testing)"
     )
 
     args = parser.parse_args()
@@ -276,7 +276,9 @@ def main():
         papers.extend(failed)
 
     if not papers:
-        logger.warning(f"No {'pending' if args.pending else 'failed'} papers found for project: {args.project}")
+        logger.warning(
+            f"No {'pending' if args.pending else 'failed'} papers found for project: {args.project}"
+        )
         return
 
     # Show summary