scitex 2.5.0__py3-none-any.whl → 2.7.3__py3-none-any.whl

scitex/scholar/examples/04_01-url.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -43,9 +44,9 @@ import scitex as stx
 """Parameters"""
 
 """Functions & Classes"""
-async def demonstrate_url_finding(
-    doi: str = None, use_cache: bool = False
-) -> dict:
+
+
+async def demonstrate_url_finding(doi: str = None, use_cache: bool = False) -> dict:
     """Demonstrate URL finding capabilities.
 
     Parameters
@@ -75,9 +76,10 @@ async def demonstrate_url_finding(
         browser_mode="interactive",
         chrome_profile_name="system",
     )
-    browser, context = (
-        await browser_manager.get_authenticated_browser_and_context_async()
-    )
+    (
+        browser,
+        context,
+    ) = await browser_manager.get_authenticated_browser_and_context_async()
 
     print(f"🔍 Creating URL finder (cache: {use_cache})...")
     url_finder = ScholarURLFinder(context, use_cache=use_cache)
@@ -108,9 +110,7 @@ async def main_async(args) -> dict:
     print("🔗 Scholar URL Finder Demonstration")
     print("=" * 40)
 
-    results = await demonstrate_url_finding(
-        doi=args.doi, use_cache=not args.no_cache
-    )
+    results = await demonstrate_url_finding(doi=args.doi, use_cache=not args.no_cache)
 
     print("✅ URL finding demonstration completed")
     return results

scitex/scholar/examples/04_02-url-for-bibtex.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -46,6 +47,8 @@ import scitex as stx
 """Parameters"""
 
 """Functions & Classes"""
+
+
 async def process_bibtex_urls(
     bibtex_path: str,
     use_cache_engine: bool = True,
@@ -103,9 +106,10 @@ async def process_bibtex_urls(
         auth_manager=ScholarAuthManager(config=config),
         config=config,
     )
-    browser, context = (
-        await browser_manager.get_authenticated_browser_and_context_async()
-    )
+    (
+        browser,
+        context,
+    ) = await browser_manager.get_authenticated_browser_and_context_async()
 
     print("🔧 Initializing components...")
     engine = ScholarEngine(config=config, use_cache=use_cache_engine)

scitex/scholar/examples/04_02-url-for-dois.py

@@ -30,52 +30,50 @@ logger = logging.getLogger(__name__)
 
 
 async def test_doi_url_finding(
-    doi: str,
-    url_finder: ScholarURLFinder,
-    use_cache: bool = True
+    doi: str, url_finder: ScholarURLFinder, use_cache: bool = True
 ) -> Dict:
     """
     Test URL finding for a single DOI.
-    
+
     Returns:
         dict with URL results and timing
     """
     start_time = datetime.now()
-    
+
     try:
         logger.info(f"Testing DOI: {doi}")
-        
+
         # Find URLs
         result = await url_finder.find_urls(doi=doi)
-        
+
         # Add timing
         end_time = datetime.now()
         result["time_seconds"] = (end_time - start_time).total_seconds()
         result["success"] = bool(result.get("url_openurl_resolved"))
-        
+
         # Check if reached final article
         resolved = result.get("url_openurl_resolved", "")
         result["reached_article"] = (
-            "/article/" in resolved or
-            "/content/" in resolved or
-            "/full/" in resolved or
-            ".pdf" in resolved
+            "/article/" in resolved
+            or "/content/" in resolved
+            or "/full/" in resolved
+            or ".pdf" in resolved
         )
         result["stopped_at_auth"] = (
-            "auth." in resolved.lower() or
-            "login" in resolved.lower() or
-            "shibboleth" in resolved.lower()
+            "auth." in resolved.lower()
+            or "login" in resolved.lower()
+            or "shibboleth" in resolved.lower()
         )
-        
+
         return result
-        
+
     except Exception as e:
         logger.error(f"Error processing DOI {doi}: {e}")
         return {
             "doi": doi,
             "error": str(e),
             "success": False,
-            "time_seconds": (datetime.now() - start_time).total_seconds()
+            "time_seconds": (datetime.now() - start_time).total_seconds(),
         }
 
 
@@ -83,81 +81,81 @@ async def process_dois_batch(
     dois: List[str],
     url_finder: ScholarURLFinder,
     batch_size: int = 5,
-    use_cache: bool = True
+    use_cache: bool = True,
 ) -> List[Dict]:
     """
     Process DOIs in batches to avoid overwhelming the browser.
     """
     results = []
-    
+
     for i in range(0, len(dois), batch_size):
-        batch = dois[i:i+batch_size]
-        logger.info(f"Processing batch {i//batch_size + 1}: {len(batch)} DOIs")
-        
+        batch = dois[i : i + batch_size]
+        logger.info(f"Processing batch {i // batch_size + 1}: {len(batch)} DOIs")
+
         # Process batch sequentially (to avoid multiple tabs)
         for doi in batch:
             if not doi or pd.isna(doi):
                 logger.warning(f"Skipping empty DOI")
                 continue
-                
+
             result = await test_doi_url_finding(doi, url_finder, use_cache)
             results.append(result)
-            
+
             # Small delay between requests
             await asyncio.sleep(2)
-        
+
         logger.info(f"Batch complete. Processed {len(results)}/{len(dois)} DOIs")
-    
+
     return results
 
 
 def load_dois_from_csv(csv_path: Path, limit: Optional[int] = None) -> List[str]:
     """Load DOIs from CSV file."""
     dois = []
-    
+
     df = pd.read_csv(csv_path)
-    
+
     # Find DOI column (could be 'doi', 'DOI', 'doi_url', etc.)
-    doi_columns = [col for col in df.columns if 'doi' in col.lower()]
-    
+    doi_columns = [col for col in df.columns if "doi" in col.lower()]
+
     if not doi_columns:
         raise ValueError(f"No DOI column found in {csv_path}")
-    
+
     doi_column = doi_columns[0]
     logger.info(f"Using column '{doi_column}' for DOIs")
-    
+
     # Extract DOIs
     for value in df[doi_column].dropna():
         # Extract DOI from URL if needed
-        if 'doi.org/' in str(value):
-            doi = value.split('doi.org/')[-1]
+        if "doi.org/" in str(value):
+            doi = value.split("doi.org/")[-1]
         else:
             doi = str(value)
-        
-        if doi and doi != 'nan':
+
+        if doi and doi != "nan":
             dois.append(doi)
-    
+
     if limit:
         dois = dois[:limit]
-    
+
     logger.info(f"Loaded {len(dois)} DOIs from {csv_path}")
     return dois
 
 
 def generate_report(results: List[Dict], output_dir: Path):
     """Generate summary report of URL finding results."""
-    
+
     # Create summary
     total = len(results)
     successful = sum(1 for r in results if r.get("success"))
     reached_article = sum(1 for r in results if r.get("reached_article"))
     stopped_at_auth = sum(1 for r in results if r.get("stopped_at_auth"))
     errors = sum(1 for r in results if r.get("error"))
-    
+
     # Average time
     times = [r.get("time_seconds", 0) for r in results if "time_seconds" in r]
     avg_time = sum(times) / len(times) if times else 0
-    
+
     report = {
         "summary": {
             "total_dois": total,
@@ -165,37 +163,41 @@ def generate_report(results: List[Dict], output_dir: Path):
             "reached_article": reached_article,
             "stopped_at_auth": stopped_at_auth,
             "errors": errors,
-            "success_rate": f"{(successful/total)*100:.1f}%" if total > 0 else "0%",
-            "article_reach_rate": f"{(reached_article/total)*100:.1f}%" if total > 0 else "0%",
-            "avg_time_seconds": round(avg_time, 2)
+            "success_rate": f"{(successful / total) * 100:.1f}%" if total > 0 else "0%",
+            "article_reach_rate": f"{(reached_article / total) * 100:.1f}%"
+            if total > 0
+            else "0%",
+            "avg_time_seconds": round(avg_time, 2),
         },
-        "results": results
+        "results": results,
     }
-    
+
     # Save report
     report_path = output_dir / "url_finding_report.json"
     with open(report_path, "w") as f:
         json.dump(report, f, indent=2, default=str)
-    
+
     # Save CSV for easy analysis
     df = pd.DataFrame(results)
     csv_path = output_dir / "url_finding_results.csv"
     df.to_csv(csv_path, index=False)
-    
+
     # Print summary
-    print("\n" + "="*60)
+    print("\n" + "=" * 60)
     print("📊 URL Finding Summary")
-    print("="*60)
+    print("=" * 60)
     print(f"Total DOIs tested: {total}")
     print(f"Successful: {successful} ({report['summary']['success_rate']})")
-    print(f"Reached article: {reached_article} ({report['summary']['article_reach_rate']})")
+    print(
+        f"Reached article: {reached_article} ({report['summary']['article_reach_rate']})"
+    )
     print(f"Stopped at auth: {stopped_at_auth}")
     print(f"Errors: {errors}")
     print(f"Average time: {avg_time:.2f} seconds")
     print(f"\nReports saved to:")
     print(f"  - {report_path}")
     print(f"  - {csv_path}")
-    
+
     return report
 
 
@@ -207,62 +209,53 @@ async def main():
         "--csv",
         type=str,
         default="/home/ywatanabe/proj/SciTeX-Code/src/scitex/scholar/examples/99_maintenance_out/data/scholar/pac_title_and_doi_url.csv",
-        help="Path to CSV file with DOIs"
+        help="Path to CSV file with DOIs",
     )
     parser.add_argument(
         "--limit",
         type=int,
         default=10,
-        help="Limit number of DOIs to test (default: 10)"
+        help="Limit number of DOIs to test (default: 10)",
     )
     parser.add_argument(
         "--batch-size",
         type=int,
         default=5,
-        help="Batch size for processing (default: 5)"
+        help="Batch size for processing (default: 5)",
     )
-    parser.add_argument(
-        "--no-cache",
-        "-nc",
-        action="store_true",
-        help="Disable cache"
-    )
-    parser.add_argument(
-        "--output-dir",
-        type=str,
-        help="Output directory for reports"
-    )
-    
+    parser.add_argument("--no-cache", "-nc", action="store_true", help="Disable cache")
+    parser.add_argument("--output-dir", type=str, help="Output directory for reports")
+
     args = parser.parse_args()
-    
+
     print("🔗 Multi-DOI URL Finding Test")
-    print("="*60)
-    
+    print("=" * 60)
+
     # Setup output directory
     if args.output_dir:
         output_dir = Path(args.output_dir)
     else:
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
         output_dir = Path(f"./url_test_results_{timestamp}")
-    
+
     output_dir.mkdir(parents=True, exist_ok=True)
     print(f"Output directory: {output_dir}")
-    
+
     # Load DOIs
     csv_path = Path(args.csv)
     if not csv_path.exists():
         logger.error(f"CSV file not found: {csv_path}")
         return
-    
+
     dois = load_dois_from_csv(csv_path, limit=args.limit)
-    
+
     if not dois:
         logger.error("No DOIs found in CSV")
         return
-    
+
     print(f"Testing {len(dois)} DOIs from {csv_path.name}")
-    print("-"*60)
-    
+    print("-" * 60)
+
     # Initialize browser
     print("🌐 Initializing authenticated browser...")
     auth_manager = ScholarAuthManager()
@@ -271,45 +264,42 @@ async def main():
         browser_mode="stealth",
         chrome_profile_name="extension",
     )
-    
-    browser, context = await browser_manager.get_authenticated_browser_and_context_async()
-    
+
+    (
+        browser,
+        context,
+    ) = await browser_manager.get_authenticated_browser_and_context_async()
+
     # Create URL finder
     print("🔍 Creating URL finder...")
-    url_finder = ScholarURLFinder(
-        context=context,
-        use_cache=not args.no_cache
-    )
-    
+    url_finder = ScholarURLFinder(context=context, use_cache=not args.no_cache)
+
     # Process DOIs
     print(f"\n📋 Processing {len(dois)} DOIs in batches of {args.batch_size}...")
-    print("-"*60)
-    
+    print("-" * 60)
+
     results = await process_dois_batch(
-        dois,
-        url_finder,
-        batch_size=args.batch_size,
-        use_cache=not args.no_cache
+        dois, url_finder, batch_size=args.batch_size, use_cache=not args.no_cache
     )
-    
+
     # Generate report
     report = generate_report(results, output_dir)
-    
+
     # Show problematic DOIs
     print("\n⚠️ DOIs that stopped at authentication:")
     for r in results:
         if r.get("stopped_at_auth"):
             print(f"  - {r.get('doi')}: {r.get('url_openurl_resolved', 'N/A')[:60]}...")
-    
+
     print("\n❌ DOIs with errors:")
     for r in results:
         if r.get("error"):
             print(f"  - {r.get('doi')}: {r.get('error')[:60]}...")
-    
+
     # Cleanup
     await browser.close()
     print("\n✅ Testing complete!")
 
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    asyncio.run(main())

scitex/scholar/examples/05_download_pdf.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -42,6 +43,8 @@ import scitex as stx
 """Parameters"""
 
 """Functions & Classes"""
+
+
 async def demonstrate_pdf_download(
     pdf_url: str = None, output_path: str = None, browser_mode: str = "stealth"
 ) -> str:
@@ -68,7 +71,9 @@ async def demonstrate_pdf_download(
     )
 
     # Default parameters
-    default_url = "https://www.science.org/cms/asset/b9925b7f-c841-48d1-a90c-1631b7cff596/pap.pdf"
+    default_url = (
+        "https://www.science.org/cms/asset/b9925b7f-c841-48d1-a90c-1631b7cff596/pap.pdf"
+    )
     default_output = "/tmp/hippocampal_ripples-downloaded.pdf"
 
     download_url = pdf_url or default_url
@@ -80,9 +85,10 @@ async def demonstrate_pdf_download(
         browser_mode=browser_mode,
         auth_manager=ScholarAuthManager(),
     )
-    browser, context = (
-        await browser_manager.get_authenticated_browser_and_context_async()
-    )
+    (
+        browser,
+        context,
+    ) = await browser_manager.get_authenticated_browser_and_context_async()
 
     print("📥 Initializing PDF downloader...")
     pdf_downloader = ScholarPDFDownloader(context)

scitex/scholar/examples/06_find_and_download.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/examples/06_find_and_download.py"
-)
+
+__FILE__ = "./src/scitex/scholar/examples/06_find_and_download.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -60,9 +59,10 @@ async def find_and_download_pdf(
         browser_mode=browser_mode,
         chrome_profile_name="system",
     )
-    browser, context = (
-        await browser_manager.get_authenticated_browser_and_context_async()
-    )
+    (
+        browser,
+        context,
+    ) = await browser_manager.get_authenticated_browser_and_context_async()
 
     # Find PDF URLs
     logger.info(f"{func_name}: Finding PDF URLs...")

scitex/scholar/examples/06_parse_bibtex.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -42,13 +43,15 @@ import scitex as stx
 """Parameters"""
 
 """Functions & Classes"""
+
+
 def demonstrate_bibtex_parsing(
     openaccess_path: str = "./data/scholar/openaccess.bib",
     paywalled_path: str = "./data/scholar/paywalled.bib",
-    n_samples: int = 3
+    n_samples: int = 3,
 ) -> dict:
     """Demonstrate BibTeX parsing capabilities.
-    
+
     Parameters
     ----------
     openaccess_path : str, default="./data/scholar/openaccess.bib"
@@ -57,7 +60,7 @@ def demonstrate_bibtex_parsing(
         Path to paywalled BibTeX file
     n_samples : int, default=3
         Number of sample entries to display
-        
+
     Returns
     -------
     dict
@@ -66,28 +69,25 @@ def demonstrate_bibtex_parsing(
     from scitex.scholar.utils import parse_bibtex
 
     results = {}
-    
+
     print(f"📚 Parsing OpenAccess BibTeX file: {openaccess_path}")
     openaccess_parsed = parse_bibtex(openaccess_path)
     print(f"📊 Found {len(openaccess_parsed)} OpenAccess papers")
-    
+
     print(f"\n📋 Sample OpenAccess entries (first {n_samples}):")
     print("=" * 50)
     pprint(openaccess_parsed[:n_samples])
-    
+
     print(f"\n📚 Parsing Paywalled BibTeX file: {paywalled_path}")
     paywalled_parsed = parse_bibtex(paywalled_path)
     print(f"📊 Found {len(paywalled_parsed)} Paywalled papers")
-    
+
     print(f"\n📋 Sample Paywalled entries (first {n_samples}):")
     print("=" * 50)
     pprint(paywalled_parsed[:n_samples])
-    
-    results = {
-        "openaccess": openaccess_parsed,
-        "paywalled": paywalled_parsed
-    }
-    
+
+    results = {"openaccess": openaccess_parsed, "paywalled": paywalled_parsed}
+
     return results
 
 
@@ -106,18 +106,18 @@ def main(args) -> int:
     """
     print("📚 Scholar BibTeX Parser Demonstration")
     print("=" * 40)
-    
+
     try:
         results = demonstrate_bibtex_parsing(
             openaccess_path=args.openaccess_path,
             paywalled_path=args.paywalled_path,
-            n_samples=args.n_samples
+            n_samples=args.n_samples,
         )
-        
+
         total_entries = len(results["openaccess"]) + len(results["paywalled"])
         print(f"\n✅ Successfully parsed {total_entries} total BibTeX entries")
         return 0
-        
+
     except Exception as e:
         print(f"❌ Error: {e}")
         return 1

scitex/scholar/examples/07_storage_integration.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/examples/07_storage_integration.py"
-)
+
+__FILE__ = "./src/scitex/scholar/examples/07_storage_integration.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -46,6 +45,8 @@ import scitex as stx
 """Parameters"""
 
 """Functions & Classes"""
+
+
 async def demonstrate_paper_storage() -> None:
     """Demonstrate individual Paper storage capabilities."""
     from pathlib import Path
@@ -66,9 +67,7 @@ async def demonstrate_paper_storage() -> None:
     paper.metadata.publication.journal = "Nature AI Research"
     paper.metadata.publication.impact_factor = 42.778
     paper.metadata.citation_count.total = 1234
-    paper.metadata.set_doi(
-        "10.1038/nature.ai.2025.001"
-    )  # Auto-syncs DOI and URL
+    paper.metadata.set_doi("10.1038/nature.ai.2025.001")  # Auto-syncs DOI and URL
     paper.container.projects = ["storage_demo"]
 
     print(f"Created paper with title: {paper.metadata.basic.title}")
@@ -110,9 +109,7 @@ async def demonstrate_papers_collection() -> None:
 
     # Paper 1
     p1 = Paper()
-    p1.metadata.basic.title = (
-        "Deep Learning for Scientific Literature Analysis"
-    )
+    p1.metadata.basic.title = "Deep Learning for Scientific Literature Analysis"
     p1.metadata.basic.authors = ["Alice Researcher", "Bob Scholar"]
     p1.metadata.basic.year = 2024
     p1.metadata.publication.journal = "AI Journal"