scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/.legacy/metadata/doi/sources/_PubMedSource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -85,9 +86,7 @@ class PubMedSource(BaseDOISource):
             query_parts.append(f"{year}[pdat]")
         query = " AND ".join(query_parts)
 
-        search_url = (
-            "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
-        )
+        search_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
         search_params = {
             "db": "pubmed",
             "term": query,
@@ -95,9 +94,7 @@ class PubMedSource(BaseDOISource):
             "retmax": 5,
             "email": self.email,
         }
-        response = self.session.get(
-            search_url, params=search_params, timeout=30
-        )
+        response = self.session.get(search_url, params=search_params, timeout=30)
         response.raise_for_status()
         data = response.json()
         pmids = data.get("esearchresult", {}).get("idlist", [])
@@ -109,9 +106,7 @@ class PubMedSource(BaseDOISource):
                 metadata
                 and metadata.get("basic")
                 and metadata.get("basic").get("title")
-                and self._is_title_match(
-                    title, metadata.get("basic").get("title")
-                )
+                and self._is_title_match(title, metadata.get("basic").get("title"))
             ):
                 if return_as == "dict":
                     return metadata
@@ -124,13 +119,9 @@ class PubMedSource(BaseDOISource):
         return_as: Optional[str] = "dict",
     ) -> Optional[Dict[str, Any]]:
         """Search by DOI using PubMed database"""
-        doi = doi.replace("https://doi.org/", "").replace(
-            "http://doi.org/", ""
-        )
+        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
 
-        search_url = (
-            "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
-        )
+        search_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
         search_params = {
             "db": "pubmed",
             "term": f'"{doi}"[doi]',
@@ -139,9 +130,7 @@ class PubMedSource(BaseDOISource):
             "email": self.email,
         }
 
-        response = self.session.get(
-            search_url, params=search_params, timeout=30
-        )
+        response = self.session.get(search_url, params=search_params, timeout=30)
         response.raise_for_status()
         data = response.json()
         pmids = data.get("esearchresult", {}).get("idlist", [])
@@ -252,9 +241,7 @@ class PubMedSource(BaseDOISource):
                 "journal": journal,
                 "journal_sources": [self.name] if journal else None,
                 "short_journal": short_journal,
-                "short_journal_sources": (
-                    [self.name] if short_journal else None
-                ),
+                "short_journal_sources": ([self.name] if short_journal else None),
                 "issn": issn,
                 "issn_sources": [self.name] if issn else None,
                 "volume": volume,
@@ -293,9 +280,7 @@ if __name__ == "__main__":
 
     # Search by title
     outputs["metadata_by_title_dict"] = source.search(title=TITLE)
-    outputs["metadata_by_title_json"] = source.search(
-        title=TITLE, return_as="json"
-    )
+    outputs["metadata_by_title_json"] = source.search(title=TITLE, return_as="json")
 
     # Search by DOI
     outputs["metadata_by_doi_dict"] = source.search(doi=DOI)
@@ -303,9 +288,7 @@ if __name__ == "__main__":
 
     # Search by PubMed ID
     outputs["metadata_by_pmid_dict"] = source.search(pmid=PMID)
-    outputs["metadata_by_pmid_json"] = source.search(
-        pmid=PMID, return_as="json"
-    )
+    outputs["metadata_by_pmid_json"] = source.search(pmid=PMID, return_as="json")
 
     for k, v in outputs.items():
         print("----------------------------------------")

scitex/scholar/.legacy/metadata/doi/sources/_SemanticScholarSource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -34,9 +35,7 @@ logger = logging.getLogger(__name__)
 class SemanticScholarSource(BaseDOISource):
     """Combined Semantic Scholar source with enhanced features."""
 
-    def __init__(
-        self, email: str = "research@example.com", api_key: str = None
-    ):
+    def __init__(self, email: str = "research@example.com", api_key: str = None):
         super().__init__(email)
         self.api_key = api_key or os.getenv("SEMANTIC_SCHOLAR_API_KEY")
         self.base_url = "https://api.semanticscholar.org/graph/v1"
@@ -105,13 +104,9 @@ class SemanticScholarSource(BaseDOISource):
         """Search by DOI directly"""
         self._handle_rate_limit()
 
-        doi = doi.replace("https://doi.org/", "").replace(
-            "http://doi.org/", ""
-        )
+        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
         url = f"{self.base_url}/paper/{doi}"
-        params = {
-            "fields": "title,year,authors,externalIds,url,venue,abstract"
-        }
+        params = {"fields": "title,year,authors,externalIds,url,venue,abstract"}
 
         try:
             assert return_as in [
@@ -179,8 +174,7 @@ class SemanticScholarSource(BaseDOISource):
                 paper_title = paper.get("title", "")
                 paper_year = paper.get("year")
                 paper_authors = [
-                    author.get("name", "")
-                    for author in paper.get("authors", [])
+                    author.get("name", "") for author in paper.get("authors", [])
                 ]
 
                 # Check title match
@@ -219,9 +213,7 @@ class SemanticScholarSource(BaseDOISource):
         wait=wait_exponential(multiplier=2, min=4, max=20),
         retry=retry_if_exception_type((requests.ConnectionError,)),
     )
-    def _search_by_corpus_id(
-        self, corpus_id: str, return_as: str
-    ) -> Optional[Dict]:
+    def _search_by_corpus_id(self, corpus_id: str, return_as: str) -> Optional[Dict]:
         """Search by Corpus ID directly"""
         if not corpus_id.isdigit():
             corpus_id = corpus_id.replace("CorpusId:", "")
@@ -229,9 +221,7 @@ class SemanticScholarSource(BaseDOISource):
         self._handle_rate_limit()
 
         url = f"{self.base_url}/paper/CorpusId:{corpus_id}"
-        params = {
-            "fields": "title,year,authors,externalIds,url,venue,abstract"
-        }
+        params = {"fields": "title,year,authors,externalIds,url,venue,abstract"}
 
         try:
             assert return_as in [
@@ -244,9 +234,7 @@ class SemanticScholarSource(BaseDOISource):
                 raise requests.ConnectionError("Rate limit exceeded")
 
             if response.status_code == 404:
-                logger.warn(
-                    f"Semantic Scholar Corpus ID not found: {corpus_id}"
-                )
+                logger.warn(f"Semantic Scholar Corpus ID not found: {corpus_id}")
                 return None
 
             response.raise_for_status()
@@ -285,12 +273,8 @@ class SemanticScholarSource(BaseDOISource):
                 "title_sources": [self.name] if paper_title else None,
                 "year": paper_year if paper_year else None,
                 "year_sources": [self.name] if paper_year else None,
-                "abstract": (
-                    paper.get("abstract") if paper.get("abstract") else None
-                ),
-                "abstract_sources": (
-                    [self.name] if paper.get("abstract") else None
-                ),
+                "abstract": (paper.get("abstract") if paper.get("abstract") else None),
+                "abstract_sources": ([self.name] if paper.get("abstract") else None),
                 "authors": extracted_authors if extracted_authors else None,
                 "authors_sources": [self.name] if extracted_authors else None,
             },
@@ -332,9 +316,7 @@ if __name__ == "__main__":
 
     # Search by title
     outputs["metadata_by_title_dict"] = source.search(title=TITLE)
-    outputs["metadata_by_title_json"] = source.search(
-        title=TITLE, return_as="json"
-    )
+    outputs["metadata_by_title_json"] = source.search(title=TITLE, return_as="json")
 
     # # Search by DOI
     # outputs["metadata_by_doi_dict"] = source.search(doi=DOI)

scitex/scholar/.legacy/metadata/doi/sources/_SourceManager.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -69,13 +70,9 @@ class SourceManager:
         self.config = config or ScholarConfig()
         self.sources = self.config.resolve("sources", sources)
         # Emails
-        self.crossref_email = self.config.resolve(
-            "crossref_email", email_crossref
-        )
+        self.crossref_email = self.config.resolve("crossref_email", email_crossref)
         self.pubmed_email = self.config.resolve("pubmed_email", email_pubmed)
-        self.openalex_email = self.config.resolve(
-            "openalex_email", email_openalex
-        )
+        self.openalex_email = self.config.resolve("openalex_email", email_openalex)
         self.semantic_scholar_email = self.config.resolve(
             "semantic_scholar_email",
             email_semantic_scholar,
@@ -105,9 +102,7 @@ class SourceManager:
 
         return self._source_instances.get(name)
 
-    def _create_source_instance(
-        self, source_name: str
-    ) -> Optional[BaseDOISource]:
+    def _create_source_instance(self, source_name: str) -> Optional[BaseDOISource]:
         """Create a new source instance with proper configuration.
 
         Args:
@@ -134,9 +129,7 @@ class SourceManager:
                 source_instance.set_rate_limit_handler(self.rate_limit_handler)
 
             # Configure source-specific rate limiting parameters
-            self._configure_source_specific_settings(
-                source_instance, source_name
-            )
+            self._configure_source_specific_settings(source_instance, source_name)
 
             logger.debug(f"Created source instance: {source_name}")
             return source_instance
@@ -179,14 +172,10 @@ class SourceManager:
         # Configure source-specific rate limiting parameters
         if source_name.lower() == "pubmed":
             # NCBI requires max 3 requests per second (0.35s delay)
-            state = self.rate_limit_handler.get_source_state(
-                source_name.lower()
-            )
+            state = self.rate_limit_handler.get_source_state(source_name.lower())
             state.base_delay = 0.35
             state.adaptive_delay = 0.35
-            logger.debug(
-                f"Configured PubMed-specific rate limiting: 0.35s delay"
-            )
+            logger.debug(f"Configured PubMed-specific rate limiting: 0.35s delay")
 
     def get_all_sources(self) -> List[BaseDOISource]:
         """Get all configured source instances.
@@ -279,9 +268,7 @@ class SourceManager:
                 if source:
                     validation["source_status"][name] = "available"
                 else:
-                    validation["warnings"].append(
-                        f"Could not create source: {name}"
-                    )
+                    validation["warnings"].append(f"Could not create source: {name}")
                     validation["source_status"][name] = "creation_failed"
             except Exception as e:
                 validation["errors"].append(f"Error with source {name}: {e}")

scitex/scholar/.legacy/metadata/doi/sources/_SourceResolutionStrategy.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -75,13 +76,9 @@ class SourceResolutionStrategy:
         self.sources = self.config.resolve("sources", sources)
 
         # Emails
-        self.crossref_email = self.config.resolve(
-            "crossref_email", email_crossref
-        )
+        self.crossref_email = self.config.resolve("crossref_email", email_crossref)
         self.pubmed_email = self.config.resolve("pubmed_email", email_pubmed)
-        self.openalex_email = self.config.resolve(
-            "openalex_email", email_openalex
-        )
+        self.openalex_email = self.config.resolve("openalex_email", email_openalex)
         self.semantic_scholar_email = self.config.resolve(
             "semantic_scholar_email",
             email_semantic_scholar,
@@ -89,14 +86,10 @@ class SourceResolutionStrategy:
         self.arxiv_email = self.config.resolve("arxiv_email", email_arxiv)
 
         # Rate Limit Handler
-        self.rate_limit_handler = rate_limit_handler or RateLimitHandler(
-            config
-        )
+        self.rate_limit_handler = rate_limit_handler or RateLimitHandler(config)
 
         # Source Rotation Manager
-        self.source_rotation_manager = SourceRotationManager(
-            self.rate_limit_handler
-        )
+        self.source_rotation_manager = SourceRotationManager(self.rate_limit_handler)
 
         # Initialize source instances cache
         self._source_instances: Dict[str, BaseDOISource] = {}
@@ -132,9 +125,7 @@ class SourceResolutionStrategy:
         """
 
         # Try CorpusID resolution if URL contains CorpusID
-        corpus_result = await self._corpusid2metadata_async(
-            url, title, year, authors
-        )
+        corpus_result = await self._corpusid2metadata_async(url, title, year, authors)
         if corpus_result:
             return corpus_result
 
@@ -146,14 +137,12 @@ class SourceResolutionStrategy:
 
         # Get available sources (not rate limited)
         if self.rate_limit_handler:
-            available_sources = self.rate_limit_handler.get_available_sources(
-                sources
-            )
+            available_sources = self.rate_limit_handler.get_available_sources(sources)
 
             if not available_sources:
                 # All sources are rate limited - get wait time for earliest available
-                next_available_time = (
-                    self.rate_limit_handler.get_next_available_time(sources)
+                next_available_time = self.rate_limit_handler.get_next_available_time(
+                    sources
                 )
                 wait_time = max(0, next_available_time - time.time())
 
@@ -165,24 +154,20 @@ class SourceResolutionStrategy:
                         wait_time, "any source"
                     )
                     # Retry with updated availability
-                    available_sources = (
-                        self.rate_limit_handler.get_available_sources(sources)
+                    available_sources = self.rate_limit_handler.get_available_sources(
+                        sources
                     )
 
                 if not available_sources:
-                    logger.error(
-                        "No sources available after waiting for rate limits"
-                    )
+                    logger.error("No sources available after waiting for rate limits")
                     return None
         else:
             available_sources = sources
 
         # Get optimal source order using intelligent selection
         if self.source_rotation_manager:
-            optimal_sources = (
-                self.source_rotation_manager.get_optimal_source_order(
-                    paper_info, available_sources, max_sources=3
-                )
+            optimal_sources = self.source_rotation_manager.get_optimal_source_order(
+                paper_info, available_sources, max_sources=3
             )
         else:
             # Fallback to first 3 sources
@@ -207,10 +192,8 @@ class SourceResolutionStrategy:
         # Try fallback sources if primary sources failed
         if self.source_rotation_manager:
             tried_sources = optimal_sources
-            fallback_sources = (
-                self.source_rotation_manager.get_fallback_sources(
-                    tried_sources, sources
-                )
+            fallback_sources = self.source_rotation_manager.get_fallback_sources(
+                tried_sources, sources
             )
 
             if fallback_sources:
@@ -259,15 +242,11 @@ class SourceResolutionStrategy:
 
         # Add rate limiting stats if available
         if self.rate_limit_handler:
-            stats["rate_limit_stats"] = (
-                self.rate_limit_handler.get_statistics()
-            )
+            stats["rate_limit_stats"] = self.rate_limit_handler.get_statistics()
 
         # Add source rotation stats if available
         if self.source_rotation_manager:
-            stats["rotation_stats"] = (
-                self.source_rotation_manager.get_statistics()
-            )
+            stats["rotation_stats"] = self.source_rotation_manager.get_statistics()
 
         # Add individual source stats
         source_details = {}
@@ -300,9 +279,7 @@ class SourceResolutionStrategy:
 
                 # Inject rate limit handler into source
                 if self.rate_limit_handler:
-                    source_instance.set_rate_limit_handler(
-                        self.rate_limit_handler
-                    )
+                    source_instance.set_rate_limit_handler(self.rate_limit_handler)
 
                 self._source_instances[name] = source_instance
         return self._source_instances.get(name)
@@ -358,9 +335,7 @@ class SourceResolutionStrategy:
                         },
                     }
             except Exception as e:
-                logger.debug(
-                    f"CorpusID resolution failed for {corpus_id}: {e}"
-                )
+                logger.debug(f"CorpusID resolution failed for {corpus_id}: {e}")
         else:
             logger.warning(
                 "Semantic Scholar source not available for CorpusID resolution"
@@ -453,15 +428,11 @@ class SourceResolutionStrategy:
 
                 # Check if this was a rate limit error
                 if self.rate_limit_handler:
-                    rate_limit_info = (
-                        self.rate_limit_handler.detect_rate_limit(
-                            source_name, exception=e
-                        )
+                    rate_limit_info = self.rate_limit_handler.detect_rate_limit(
+                        source_name, exception=e
                     )
                     if rate_limit_info:
-                        self.rate_limit_handler.record_rate_limit(
-                            rate_limit_info
-                        )
+                        self.rate_limit_handler.record_rate_limit(rate_limit_info)
                         logger.warning(
                             f"Rate limit detected for {source_name}, skipping to next source"
                         )
@@ -529,9 +500,7 @@ class SourceResolutionStrategy:
                     logger.debug(f"  Journal: {metadata['journal']}")
                 return metadata
             else:
-                logger.debug(
-                    f"No comprehensive metadata found via {source.name}"
-                )
+                logger.debug(f"No comprehensive metadata found via {source.name}")
                 return None
 
         except Exception as e:

scitex/scholar/.legacy/metadata/doi/sources/_SourceRotationManager.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -145,9 +146,7 @@ class SourceRotationManager:
                     perf.specialty_scores[paper_type] * 0.9 + 1.0 * 0.1
                 )
             else:
-                perf.specialty_scores[paper_type] = (
-                    0.8  # Start with decent score
-                )
+                perf.specialty_scores[paper_type] = 0.8  # Start with decent score
         else:
             # Reduce specialty scores on failure
             paper_type = self._classify_paper(paper_info)
@@ -294,9 +293,7 @@ class SourceRotationManager:
         # Calculate scores for each available source
         source_scores = []
         for source in available_sources:
-            score = self._calculate_source_score(
-                source, paper_type, paper_info
-            )
+            score = self._calculate_source_score(source, paper_type, paper_info)
             source_scores.append((source, score))
 
         # Sort by score (descending) and take top max_sources
@@ -321,9 +318,7 @@ class SourceRotationManager:
         rate_limit_state = self.rate_limit_handler.get_source_state(source)
 
         # Base score from source specialties
-        base_score = self.source_specialties.get(source, {}).get(
-            paper_type, 0.5
-        )
+        base_score = self.source_specialties.get(source, {}).get(paper_type, 0.5)
 
         # Adjust for learned performance
         performance_factor = 1.0
@@ -363,9 +358,7 @@ class SourceRotationManager:
         current_time = time.time()
         time_since_last = current_time - rate_limit_state.last_success
         if time_since_last > 300:  # 5 minutes
-            recency_factor = min(
-                1.2, 1.0 + (time_since_last / 3600)
-            )  # Bonus up to 20%
+            recency_factor = min(1.2, 1.0 + (time_since_last / 3600))  # Bonus up to 20%
 
         # Calculate final score
         final_score = (
@@ -379,9 +372,7 @@ class SourceRotationManager:
 
         return final_score
 
-    def should_rotate_sources(
-        self, current_failures: int, total_attempts: int
-    ) -> bool:
+    def should_rotate_sources(self, current_failures: int, total_attempts: int) -> bool:
         """Determine if we should rotate to different sources.
 
         Args:
@@ -437,9 +428,7 @@ class SourceRotationManager:
         for source in available_fallbacks:
             perf = self.get_source_performance(source)
             # Use overall success rate for fallbacks
-            score = (
-                perf.overall_success_rate if perf.total_attempts > 0 else 0.5
-            )
+            score = perf.overall_success_rate if perf.total_attempts > 0 else 0.5
             fallback_scores.append((source, score))
 
         fallback_scores.sort(key=lambda x: x[1], reverse=True)
@@ -567,9 +556,7 @@ if __name__ == "__main__":
             "arxiv",
         ]
         failed_sources = ["crossref", "semantic_scholar"]
-        fallbacks = rotation_manager.get_fallback_sources(
-            failed_sources, all_sources
-        )
+        fallbacks = rotation_manager.get_fallback_sources(failed_sources, all_sources)
         print(f"   Fallback sources after {failed_sources}: {fallbacks}")
 
         # Show statistics

scitex/scholar/.legacy/metadata/doi/sources/_URLDOISource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -78,9 +79,7 @@ class URLDOISource(BaseDOISource):
 
     def _search_by_doi(self, doi: str, return_as: str) -> Optional[Dict]:
         """Search by DOI directly"""
-        doi = doi.replace("https://doi.org/", "").replace(
-            "http://doi.org/", ""
-        )
+        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
 
         try:
             assert return_as in [
@@ -301,22 +300,20 @@ class URLDOISource(BaseDOISource):
                 if semantic_id.isdigit():
                     url = f"https://api.semanticscholar.org/graph/v1/paper/CorpusId:{semantic_id}"
                 else:
-                    url = f"https://api.semanticscholar.org/graph/v1/paper/{semantic_id}"
+                    url = (
+                        f"https://api.semanticscholar.org/graph/v1/paper/{semantic_id}"
+                    )
 
                 params = {"fields": "externalIds,title,authors"}
                 headers = {"User-Agent": f"SciTeX/1.0 (mailto:{self.email})"}
                 if self.api_key:
                     headers["x-api-key"] = self.api_key
 
-                response = requests.get(
-                    url, params=params, headers=headers, timeout=15
-                )
+                response = requests.get(url, params=params, headers=headers, timeout=15)
 
                 if response.status_code == 429:
                     if attempt < max_retries - 1:
-                        delay = (base_delay * (2**attempt)) + random.uniform(
-                            0.5, 1.5
-                        )
+                        delay = (base_delay * (2**attempt)) + random.uniform(0.5, 1.5)
                         time.sleep(delay)
                         continue
                     return None
@@ -337,14 +334,10 @@ class URLDOISource(BaseDOISource):
             except requests.HTTPError as exc:
                 if exc.response and exc.response.status_code == 429:
                     continue
-                logger.debug(
-                    f"Semantic Scholar HTTP error for {semantic_id}: {exc}"
-                )
+                logger.debug(f"Semantic Scholar HTTP error for {semantic_id}: {exc}")
                 return None
             except Exception as exc:
-                logger.debug(
-                    f"Semantic Scholar lookup failed for {semantic_id}: {exc}"
-                )
+                logger.debug(f"Semantic Scholar lookup failed for {semantic_id}: {exc}")
                 return None
         return None