scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/.legacy/metadata/doi/sources/_UnifiedSource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -230,16 +231,12 @@ class UnifiedSource:
 
         return True
 
-    def _validate_against_query(
-        self, metadata: Dict, query_title: str
-    ) -> bool:
+    def _validate_against_query(self, metadata: Dict, query_title: str) -> bool:
         """Validate metadata matches the original query with strict title matching."""
         if not query_title or not metadata:
             return True
 
-        paper_title = (
-            metadata.get("basic", {}).get("title", "").lower().strip()
-        )
+        paper_title = metadata.get("basic", {}).get("title", "").lower().strip()
         query_title = query_title.lower().strip()
 
         if not paper_title:
@@ -288,9 +285,7 @@ class UnifiedSource:
         query_title = getattr(self, "_last_query_title", None)
         valid_sources = {}
         for source_name, metadata in source_results.items():
-            if metadata and self._validate_against_query(
-                metadata, query_title
-            ):
+            if metadata and self._validate_against_query(metadata, query_title):
                 valid_sources[source_name] = metadata
 
         if not valid_sources:
@@ -301,9 +296,7 @@ class UnifiedSource:
 
         # Merge all other valid sources
         for source_name, metadata in list(valid_sources.items())[1:]:
-            base_metadata = self.merge_metadata_structures(
-                base_metadata, metadata
-            )
+            base_metadata = self.merge_metadata_structures(base_metadata, metadata)
 
         # Track all attempted searches
         if "system" not in base_metadata:
@@ -345,9 +338,7 @@ class UnifiedSource:
 
                 # Initialize source lists if needed
                 if not isinstance(current_sources, list):
-                    current_sources = (
-                        [current_sources] if current_sources else []
-                    )
+                    current_sources = [current_sources] if current_sources else []
                     merged[section][f"{key}_sources"] = current_sources
 
                 # Convert single source to list
@@ -357,15 +348,11 @@ class UnifiedSource:
                 should_replace = False
                 if current_value is None:
                     should_replace = True
-                elif source_priority.get(
-                    new_sources[0], 0
-                ) > source_priority.get(
+                elif source_priority.get(new_sources[0], 0) > source_priority.get(
                     current_sources[0] if current_sources else "", 0
                 ):
                     should_replace = True
-                elif isinstance(value, list) and isinstance(
-                    current_value, list
-                ):
+                elif isinstance(value, list) and isinstance(current_value, list):
                     if len(value) > len(current_value):
                         should_replace = True
                 elif isinstance(value, str) and isinstance(current_value, str):
@@ -388,7 +375,6 @@ if __name__ == "__main__":
     from pprint import pprint
 
     async def main_async():
-
         TITLE = "Attention is All You Need"
         DOI = "10.1038/nature14539"
         # DOI = "https://doi.org/10.48550/arXiv.1706.03762"

scitex/scholar/.legacy/metadata/doi/sources/__init__.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------

scitex/scholar/.legacy/metadata/doi/utils/_PubMedConverter.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -74,8 +75,7 @@ class PubMedConverter:
 
         # Compile patterns
         self.compiled_patterns = [
-            re.compile(pattern, re.IGNORECASE)
-            for pattern in self.PMID_PATTERNS
+            re.compile(pattern, re.IGNORECASE) for pattern in self.PMID_PATTERNS
         ]
 
     @retry(
@@ -333,9 +333,7 @@ class PubMedConverter:
 
         return results
 
-    async def bibtex_entries2dois_async(
-        self, entries: List[Dict]
-    ) -> Dict[str, str]:
+    async def bibtex_entries2dois_async(self, entries: List[Dict]) -> Dict[str, str]:
         """
         Async version of batch PMID to DOI conversion.
 
@@ -386,9 +384,7 @@ class PubMedConverter:
         return converted_dois
 
 
-def pmid2doi(
-    pmid: Union[str, int], pubmed_email=None, api_key=None, config=None
-):
+def pmid2doi(pmid: Union[str, int], pubmed_email=None, api_key=None, config=None):
     config = config or ScholarConfig()
     pubmed_email = config.resolve("pubmed_email", pubmed_email)
     pubmed_api_key = config.resolve("pubmed_api_key", api_key)

scitex/scholar/.legacy/metadata/doi/utils/_RateLimitHandler.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -151,9 +152,7 @@ class RateLimitHandler:
         self.rate_limit_history: List[RateLimitInfo] = []
 
         # Adaptive rate limiting tracking
-        self.recent_rates = deque(
-            maxlen=50
-        )  # Track recent success/failure rates
+        self.recent_rates = deque(maxlen=50)  # Track recent success/failure rates
         self.global_adaptive_delay = 0.1  # Global adaptive delay
 
         # Load existing state if available
@@ -198,9 +197,7 @@ class RateLimitHandler:
             self.state_file.parent.mkdir(parents=True, exist_ok=True)
 
             data = {
-                "sources": [
-                    asdict(state) for state in self.source_states.values()
-                ],
+                "sources": [asdict(state) for state in self.source_states.values()],
                 "global_backoff": self.global_backoff,
                 "global_adaptive_delay": self.global_adaptive_delay,
                 "last_updated": datetime.now().isoformat(),
@@ -252,9 +249,7 @@ class RateLimitHandler:
                 )
 
             # Check for rate limit indicators in response text
-            response_text = (
-                response.text.lower() if hasattr(response, "text") else ""
-            )
+            response_text = response.text.lower() if hasattr(response, "text") else ""
             for error_msg in patterns.get("error_messages", []):
                 if error_msg in response_text:
                     wait_time = self._calculate_backoff_time(source)
@@ -274,9 +269,7 @@ class RateLimitHandler:
                 keyword in error_str
                 for keyword in ["timeout", "connection", "read timeout"]
             ):
-                wait_time = self._calculate_backoff_time(
-                    source, base_multiplier=2.0
-                )
+                wait_time = self._calculate_backoff_time(source, base_multiplier=2.0)
                 return RateLimitInfo(
                     source=source,
                     limit_type=RateLimitType.CONNECTION_TIMEOUT,
@@ -297,9 +290,7 @@ class RateLimitHandler:
 
         return None
 
-    def _extract_retry_after(
-        self, response: requests.Response
-    ) -> Optional[int]:
+    def _extract_retry_after(self, response: requests.Response) -> Optional[int]:
         """Extract retry-after header value."""
         retry_after_header = response.headers.get(
             "Retry-After"
@@ -322,9 +313,7 @@ class RateLimitHandler:
         base_delay = state.base_delay * base_multiplier
 
         # Apply exponential backoff based on consecutive failures
-        backoff_multiplier = min(
-            2**state.consecutive_failures, 64
-        )  # Cap at 64x
+        backoff_multiplier = min(2**state.consecutive_failures, 64)  # Cap at 64x
         calculated_delay = base_delay * backoff_multiplier
 
         # Add jitter to prevent thundering herd
@@ -385,9 +374,7 @@ class RateLimitHandler:
         # Reset failure tracking on success
         if state.consecutive_failures > 0:
             state.consecutive_failures = max(0, state.consecutive_failures - 1)
-            state.current_delay = max(
-                state.base_delay, state.current_delay * 0.8
-            )
+            state.current_delay = max(state.base_delay, state.current_delay * 0.8)
 
         # Clear rate limit if expired
         if state.is_rate_limited and time.time() >= state.rate_limit_until:
@@ -397,9 +384,7 @@ class RateLimitHandler:
 
         self._save_state()
 
-    def record_failure(
-        self, source: str, exception: Optional[Exception] = None
-    ):
+    def record_failure(self, source: str, exception: Optional[Exception] = None):
         """Record a failed request for a source."""
         state = self.get_source_state(source)
         state.total_requests += 1
@@ -471,9 +456,7 @@ class RateLimitHandler:
 
         return max(0.0, state.rate_limit_until - time.time())
 
-    async def wait_with_countdown_async(
-        self, wait_time: float, source: str = "API"
-    ):
+    async def wait_with_countdown_async(self, wait_time: float, source: str = "API"):
         """Wait for specified time with countdown display."""
         if wait_time <= 0:
             return
@@ -492,9 +475,9 @@ class RateLimitHandler:
 
                 # Format remaining time
                 if remaining >= 3600:
-                    time_str = f"{remaining/3600:.1f}h"
+                    time_str = f"{remaining / 3600:.1f}h"
                 elif remaining >= 60:
-                    time_str = f"{remaining/60:.1f}m"
+                    time_str = f"{remaining / 60:.1f}m"
                 else:
                     time_str = f"{remaining:.0f}s"
 
@@ -585,9 +568,7 @@ class RateLimitHandler:
             elif state.total_requests > 10 and state.success_rate > 0.8:
                 recent_success_rate = self.calculate_recent_success_rate()
                 if recent_success_rate > 0.8:
-                    state.adaptive_delay = max(
-                        0.05, state.adaptive_delay * 0.9
-                    )
+                    state.adaptive_delay = max(0.05, state.adaptive_delay * 0.9)
                     logger.debug(
                         f"Decreased adaptive delay for {source}: {state.adaptive_delay:.3f}s"
                     )
@@ -597,24 +578,19 @@ class RateLimitHandler:
             # Global adaptive delay logic
             recent_success_rate = self.calculate_recent_success_rate()
             total_rate_limited = sum(
-                state.rate_limited_count
-                for state in self.source_states.values()
+                state.rate_limited_count for state in self.source_states.values()
             )
 
             # Increase global delay if getting rate limited frequently
             if total_rate_limited > 5:
-                self.global_adaptive_delay = min(
-                    2.0, self.global_adaptive_delay * 1.5
-                )
+                self.global_adaptive_delay = min(2.0, self.global_adaptive_delay * 1.5)
                 logger.debug(
                     f"Increased global adaptive delay: {self.global_adaptive_delay:.3f}s"
                 )
 
             # Decrease global delay if success rate is high
             elif recent_success_rate > 0.8 and len(self.recent_rates) > 10:
-                self.global_adaptive_delay = max(
-                    0.05, self.global_adaptive_delay * 0.9
-                )
+                self.global_adaptive_delay = max(0.05, self.global_adaptive_delay * 0.9)
                 logger.debug(
                     f"Decreased global adaptive delay: {self.global_adaptive_delay:.3f}s"
                 )
@@ -665,9 +641,7 @@ if __name__ == "__main__":
                 self.text = text
 
         response_429 = MockResponse(429, {"Retry-After": "60"})
-        rate_limit = handler.detect_rate_limit(
-            "crossref", response=response_429
-        )
+        rate_limit = handler.detect_rate_limit("crossref", response=response_429)
 
         if rate_limit:
             print(

scitex/scholar/.legacy/metadata/doi/utils/_TextNormalizer.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -131,8 +132,7 @@ class TextNormalizer:
 
         # Compile regex patterns for efficiency
         self.compiled_patterns = {
-            pattern: re.compile(pattern)
-            for pattern in self.LATEX_TO_UNICODE.keys()
+            pattern: re.compile(pattern) for pattern in self.LATEX_TO_UNICODE.keys()
         }
 
     def normalize_latex(self, text: str) -> str:
@@ -188,9 +188,7 @@ class TextNormalizer:
         if self.ascii_fallback:
             # Remove combining characters (accents)
             normalized = "".join(
-                char
-                for char in normalized
-                if unicodedata.category(char) != "Mn"
+                char for char in normalized if unicodedata.category(char) != "Mn"
             )
 
         return normalized
@@ -345,13 +343,9 @@ class TextNormalizer:
         if not entries:
             return entries
 
-        normalized_entries = [
-            self.normalize_bibtex_entry(entry) for entry in entries
-        ]
+        normalized_entries = [self.normalize_bibtex_entry(entry) for entry in entries]
 
-        logger.info(
-            f"TextNormalizer: Normalized {len(normalized_entries)} entries"
-        )
+        logger.info(f"TextNormalizer: Normalized {len(normalized_entries)} entries")
 
         return normalized_entries
 
@@ -384,9 +378,7 @@ class TextNormalizer:
         # ASCII fallback (if enabled)
         if self.ascii_fallback:
             ascii_normalizer = TextNormalizer(ascii_fallback=True)
-            ascii_normalized = ascii_normalizer.normalize_unicode(
-                unicode_normalized
-            )
+            ascii_normalized = ascii_normalizer.normalize_unicode(unicode_normalized)
             variants.add(ascii_normalized)
 
         # Remove duplicates and empty strings
@@ -394,9 +386,7 @@ class TextNormalizer:
 
         return sorted(list(set(variants)))
 
-    def is_title_match(
-        self, title1: str, title2: str, threshold: float = 0.6
-    ) -> bool:
+    def is_title_match(self, title1: str, title2: str, threshold: float = 0.6) -> bool:
         """
         Check if two titles match using advanced normalization and similarity.
 
@@ -575,7 +565,7 @@ def main():
         variants = normalizer.create_search_variants(name)
         print(f"   {name}:")
         for i, variant in enumerate(variants):
-            print(f"     {i+1}. {variant}")
+            print(f"     {i + 1}. {variant}")
 
     print("\n" + "=" * 60)
     print("✅ TextNormalizer test completed!")

scitex/scholar/.legacy/metadata/doi/utils/_URLDOIExtractor.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -53,8 +54,7 @@ class URLDOIExtractor:
     def __init__(self):
         """Initialize the URL DOI extractor."""
         self.compiled_patterns = [
-            re.compile(pattern, re.IGNORECASE)
-            for pattern in self.DOI_URL_PATTERNS
+            re.compile(pattern, re.IGNORECASE) for pattern in self.DOI_URL_PATTERNS
         ]
 
     def extract_doi_from_url(self, url: str) -> Optional[str]:
@@ -85,9 +85,7 @@ class URLDOIExtractor:
 
                 # Validate DOI format
                 if self._is_valid_doi(potential_doi):
-                    logger.debug(
-                        f"Extracted DOI '{potential_doi}' from URL: {url}"
-                    )
+                    logger.debug(f"Extracted DOI '{potential_doi}' from URL: {url}")
                     return potential_doi
 
         return None
@@ -158,9 +156,7 @@ class URLDOIExtractor:
                 results[str(i)] = extracted_doi
 
         if results:
-            logger.info(
-                f"URLDOIExtractor: Extracted DOIs from {len(results)} entries"
-            )
+            logger.info(f"URLDOIExtractor: Extracted DOIs from {len(results)} entries")
 
         return results
 

scitex/scholar/.legacy/metadata/doi/utils/__init__.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------

scitex/scholar/.legacy/metadata/doi/utils/_to_complete_metadata_structure.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------

scitex/scholar/.legacy/metadata/enrichment/_LibraryEnricher.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -89,9 +90,7 @@ if __name__ == "__main__":
         # 2. Enrich entire project library
         library_enricher = LibraryEnricher()
         results = await library_enricher.enrich_project_async("hippocampus")
-        pprint(
-            f"Enriched {results['enriched']} of {results['processed']} papers"
-        )
+        pprint(f"Enriched {results['enriched']} of {results['processed']} papers")
 
     asyncio.run(main_async())
 

scitex/scholar/.legacy/metadata/enrichment/enrichers/_ImpactFactorEnricher.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -23,15 +24,11 @@ logger = logging.getLogger(__name__)
 
 
 class ImpactFactorEnricher:
-    def __init__(
-        self, config: Optional[ScholarConfig] = None, cache_size: int = 1000
-    ):
+    def __init__(self, config: Optional[ScholarConfig] = None, cache_size: int = 1000):
         self.config = config or ScholarConfig()
         self._factor_instance = None
         self._init_package()
-        self._get_metrics = lru_cache(maxsize=cache_size)(
-            self._get_metrics_uncached
-        )
+        self._get_metrics = lru_cache(maxsize=cache_size)(self._get_metrics_uncached)
 
     def _init_package(self) -> None:
         """Initialize impact_factor package."""
@@ -64,15 +61,11 @@ class ImpactFactorEnricher:
             if "enriched_at" not in metadata:
                 metadata["enriched_at"] = datetime.now().isoformat()
 
-            logger.info(
-                f"Found impact factor {metrics['impact_factor']} for {journal}"
-            )
+            logger.info(f"Found impact factor {metrics['impact_factor']} for {journal}")
 
         return metadata
 
-    def _get_metrics_uncached(
-        self, journal_name: str
-    ) -> Optional[Dict[str, Any]]:
+    def _get_metrics_uncached(self, journal_name: str) -> Optional[Dict[str, Any]]:
         """Get journal metrics."""
         try:
             with suppress_db_logs():
@@ -110,4 +103,5 @@ def suppress_db_logs():
     finally:
         logging.getLogger("sqlalchemy.engine").setLevel(old_level)
 
+
 # EOF

scitex/scholar/.legacy/metadata/enrichment/enrichers/_SmartEnricher.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -100,9 +101,7 @@ class SmartEnricher:
         needs_api = any(field in missing_fields for field in api_fields)
 
         if needs_api:
-            comprehensive_data = (
-                self.unified_source.get_comprehensive_metadata(doi)
-            )
+            comprehensive_data = self.unified_source.get_comprehensive_metadata(doi)
             if comprehensive_data:
                 logger.info(
                     f"Got comprehensive data with keys: {list(comprehensive_data.keys())}"
@@ -110,14 +109,9 @@ class SmartEnricher:
 
                 # Only update missing fields with non-null values
                 for field in missing_fields:
-                    if (
-                        field in comprehensive_data
-                        and comprehensive_data[field]
-                    ):
+                    if field in comprehensive_data and comprehensive_data[field]:
                         metadata[field] = comprehensive_data[field]
-                        logger.debug(
-                            f"Updated {field}: {comprehensive_data[field]}"
-                        )
+                        logger.debug(f"Updated {field}: {comprehensive_data[field]}")
 
         # Add impact factor only if journal is present and impact_factor is missing
         if "impact_factor" in missing_fields and metadata.get("journal"):
@@ -166,4 +160,5 @@ class SmartEnricher:
             return True
         return False
 
+
 # EOF

scitex/scholar/.legacy/metadata/enrichment/sources/_UnifiedMetadataSource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -75,9 +76,7 @@ class UnifiedMetadataSource:
                 if hasattr(source, "get_metadata_by_doi"):
                     metadata = source.get_metadata_by_doi(doi)
                     if metadata:
-                        return self._standardize_metadata(
-                            metadata, source.name
-                        )
+                        return self._standardize_metadata(metadata, source.name)
 
                 # Fallback: use abstract getter for specific data
                 if hasattr(source, "get_abstract"):
@@ -102,8 +101,7 @@ class UnifiedMetadataSource:
             "title": metadata.get("title"),
             "abstract": metadata.get("abstract"),
             "citation_count": metadata.get("citation_count"),
-            "keywords": metadata.get("keywords")
-            or metadata.get("fieldsOfStudy"),
+            "keywords": metadata.get("keywords") or metadata.get("fieldsOfStudy"),
             "journal": metadata.get("journal") or metadata.get("venue"),
             "issn": metadata.get("issn"),
             "volume": metadata.get("volume"),
@@ -124,4 +122,5 @@ class UnifiedMetadataSource:
 
         return result
 
+
 # EOF

scitex/scholar/.legacy/metadata/query_to_full_meta_json.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -112,9 +113,7 @@ async def query_to_full_metadata_json(
     enricher = SmartEnricher()
     # The enricher modifies the dictionary in place
     enriched_metadata = enricher.enrich_metadata_json(metadata)
-    print(
-        "✅ Metadata enriched with abstract, keywords, citation counts, etc."
-    )
+    print("✅ Metadata enriched with abstract, keywords, citation counts, etc.")
 
     # 3. Enriched Metadata -> URLs
     # ============================
@@ -128,9 +127,10 @@ async def query_to_full_metadata_json(
 
     try:
         # Initialize an authenticated browser context
-        browser, context = (
-            await browser_manager.get_authenticated_browser_and_context_async()
-        )
+        (
+            browser,
+            context,
+        ) = await browser_manager.get_authenticated_browser_and_context_async()
         url_handler = URLHandler(context)
 
         # Pass the DOI to get all related URLs
@@ -138,9 +138,7 @@ async def query_to_full_metadata_json(
 
         # Add the found URLs to the main metadata dictionary
         enriched_metadata["urls"] = urls
-        print(
-            f"✅ Found {len(urls.get('url_pdf', []))} PDF URL(s) and other links."
-        )
+        print(f"✅ Found {len(urls.get('url_pdf', []))} PDF URL(s) and other links.")
 
     except Exception as e:
         print(f"⚠️ Could not find URLs due to a browser error: {e}")
@@ -149,9 +147,7 @@ async def query_to_full_metadata_json(
         await browser_manager.close()
 
     # 4. Initialize complete structure with null values
-    complete_metadata = initialize_complete_metadata_structure(
-        enriched_metadata
-    )
+    complete_metadata = initialize_complete_metadata_structure(enriched_metadata)
 
     return complete_metadata
 

scitex/scholar/.legacy/metadata/urls/_URLMetadataHandler.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -138,9 +139,7 @@ class URLMetadataHandler:
                 with open(metadata_path, "w") as f:
                     json.dump(metadata, f, indent=2, default=str)
 
-                logger.success(
-                    f"Updated URLs in metadata: {metadata_path.name}"
-                )
+                logger.success(f"Updated URLs in metadata: {metadata_path.name}")
                 return True
             else:
                 logger.info("No new URLs to add")
@@ -380,4 +379,5 @@ def get_papers_for_download(project: str = None) -> List[Dict]:
     handler = URLMetadataHandler()
     return handler.get_undownloaded_papers(project)
 
+
 # EOF