scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/.legacy/metadata/doi/sources/.combined-SemanticScholarSource/_SemanticScholarSourceEnhanced.py

@@ -46,10 +46,12 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
         """Lazy load session with proper headers."""
         if self._session is None:
             self._session = requests.Session()
-            self._session.headers.update({
-                "User-Agent": f"SciTeX/1.0 (mailto:{self.email})",
-                "Accept": "application/json"
-            })
+            self._session.headers.update(
+                {
+                    "User-Agent": f"SciTeX/1.0 (mailto:{self.email})",
+                    "Accept": "application/json",
+                }
+            )
         return self._session
 
     @property
@@ -60,16 +62,20 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
     def rate_limit_delay(self) -> float:
         return 1.2  # Slightly more conservative than original
 
-    def _is_title_match(self, query_title: str, paper_title: str, threshold: float = 0.8) -> bool:
+    def _is_title_match(
+        self, query_title: str, paper_title: str, threshold: float = 0.8
+    ) -> bool:
         """Enhanced title matching using TextNormalizer utility."""
         # Use the advanced TextNormalizer from utils
-        is_match = self.text_normalizer.is_title_match(query_title, paper_title, threshold)
-        
+        is_match = self.text_normalizer.is_title_match(
+            query_title, paper_title, threshold
+        )
+
         if logger.isEnabledFor(logging.DEBUG):
             logger.debug(f"Title match result: {is_match} (threshold: {threshold})")
             logger.debug(f"Query: {query_title}")
             logger.debug(f"Paper: {paper_title}")
-        
+
         return is_match
 
     @retry(
@@ -89,9 +95,9 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
         """Enhanced search with better DOI extraction."""
         if not title:
             return None
-            
+
         logger.debug(f"Searching Semantic Scholar for: {title}")
-        
+
         url = f"{self.base_url}/paper/search"
         params = {
             "query": title,
@@ -101,81 +107,87 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
 
         try:
             response = self.session.get(url, params=params, timeout=30)
-            
+
             if response.status_code == 429:
                 raise requests.HTTPError("Rate limited", response=response)
-            
+
             response.raise_for_status()
             data = response.json()
-            
-            logger.debug(f"Semantic Scholar returned {len(data.get('data', []))} results")
-            
+
+            logger.debug(
+                f"Semantic Scholar returned {len(data.get('data', []))} results"
+            )
+
             papers = data.get("data", [])
-            
+
             # Try multiple matching strategies
             for paper in papers:
                 doi = self._extract_doi_from_paper(paper, title, year, authors)
                 if doi:
                     logger.info(f"Found DOI via enhanced Semantic Scholar: {doi}")
                     return doi
-            
+
             logger.debug("No DOI found in Semantic Scholar results")
             return None
-            
+
         except requests.RequestException as e:
             logger.debug(f"Semantic Scholar API error: {e}")
             return None
 
     def _extract_doi_from_paper(
-        self, 
-        paper: dict, 
-        query_title: str, 
-        query_year: Optional[int], 
-        query_authors: Optional[List[str]]
+        self,
+        paper: dict,
+        query_title: str,
+        query_year: Optional[int],
+        query_authors: Optional[List[str]],
     ) -> Optional[str]:
         """Extract DOI from paper with multiple validation strategies."""
-        
+
         paper_title = paper.get("title", "")
         paper_year = paper.get("year")
-        
+
         # Title matching
         if not self._is_title_match(query_title, paper_title):
             logger.debug(f"Title mismatch: '{paper_title}' vs '{query_title}'")
             return None
-        
+
         # Year validation (if provided)
         if query_year and paper_year:
             try:
-                paper_year_int = int(paper_year) if isinstance(paper_year, str) else paper_year
-                query_year_int = int(query_year) if isinstance(query_year, str) else query_year
-                
+                paper_year_int = (
+                    int(paper_year) if isinstance(paper_year, str) else paper_year
+                )
+                query_year_int = (
+                    int(query_year) if isinstance(query_year, str) else query_year
+                )
+
                 if abs(paper_year_int - query_year_int) > 2:  # Allow 2 year difference
                     logger.debug(f"Year mismatch: {paper_year_int} vs {query_year_int}")
                     return None
             except (ValueError, TypeError):
                 pass  # Skip year validation if conversion fails
-        
+
         # Extract DOI from multiple possible fields
         external_ids = paper.get("externalIds", {})
-        
+
         # Primary DOI field
         if external_ids and "DOI" in external_ids:
             doi = external_ids["DOI"]
             if doi:
                 return self._clean_doi(doi)
-        
+
         # Alternative DOI sources
         for field in ["doi", "DOI"]:
             if field in paper and paper[field]:
                 return self._clean_doi(paper[field])
-        
+
         # Extract from URL field if present using utility
         paper_url = paper.get("url", "")
         if paper_url:
             doi = self.url_doi_extractor.extract_doi_from_url(paper_url)
             if doi:
                 return doi
-        
+
         logger.debug(f"No DOI found in paper: {paper_title}")
         return None
 
@@ -194,10 +206,10 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
         try:
             response = self.session.get(url, params=params, timeout=30)
             response.raise_for_status()
-            
+
             data = response.json()
             return data.get("abstract")
-            
+
         except requests.RequestException as e:
             logger.debug(f"Error fetching abstract for DOI {doi}: {e}")
             return None
@@ -214,4 +226,4 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
 # Export
 __all__ = ["SemanticScholarSourceEnhanced"]
 
-# EOF
+# EOF

scitex/scholar/.legacy/metadata/doi/sources/_ArXivSource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -72,15 +73,11 @@ class ArXivSource(BaseDOISource):
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=2, max=10),
-        retry=retry_if_exception_type(
-            (requests.HTTPError, requests.ConnectionError)
-        ),
+        retry=retry_if_exception_type((requests.HTTPError, requests.ConnectionError)),
     )
     def _search_by_doi(self, doi: str, return_as: str) -> Optional[Dict]:
         """Search solely on doi"""
-        doi = doi.replace("https://doi.org/", "").replace(
-            "http://doi.org/", ""
-        )
+        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
         arxiv_id = doi.split("arXiv.")[-1]
 
         params = {
@@ -96,9 +93,7 @@ class ArXivSource(BaseDOISource):
                 "dict",
                 "json",
             ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(
-                self.base_url, params=params, timeout=30
-            )
+            response = self.session.get(self.base_url, params=params, timeout=30)
             response.raise_for_status()
 
             feed = feedparser.parse(response.text)
@@ -112,9 +107,7 @@ class ArXivSource(BaseDOISource):
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=2, max=10),
-        retry=retry_if_exception_type(
-            (requests.HTTPError, requests.ConnectionError)
-        ),
+        retry=retry_if_exception_type((requests.HTTPError, requests.ConnectionError)),
     )
     def _search_by_metadata(
         self,
@@ -141,9 +134,7 @@ class ArXivSource(BaseDOISource):
                 "dict",
                 "json",
             ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(
-                self.base_url, params=params, timeout=30
-            )
+            response = self.session.get(self.base_url, params=params, timeout=30)
             response.raise_for_status()
 
             feed = feedparser.parse(response.text)
@@ -159,17 +150,13 @@ class ArXivSource(BaseDOISource):
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=2, max=10),
-        retry=retry_if_exception_type(
-            (requests.HTTPError, requests.ConnectionError)
-        ),
+        retry=retry_if_exception_type((requests.HTTPError, requests.ConnectionError)),
     )
     def _scrape_doi(self, url_publisher):
         try:
             response = requests.get(url_publisher)
             soup = BeautifulSoup(response.content, "html.parser")
-            doi_link = soup.find(
-                "a", href=lambda href: href and "doi.org" in href
-            )
+            doi_link = soup.find("a", href=lambda href: href and "doi.org" in href)
             doi = doi_link.get("href").split("doi.org/")[-1]
             return doi
         except Exception as e:
@@ -178,13 +165,9 @@ class ArXivSource(BaseDOISource):
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=2, max=10),
-        retry=retry_if_exception_type(
-            (requests.HTTPError, requests.ConnectionError)
-        ),
+        retry=retry_if_exception_type((requests.HTTPError, requests.ConnectionError)),
     )
-    def _extract_metadata_from_entry(
-        self, entry, return_as: str
-    ) -> Optional[Dict]:
+    def _extract_metadata_from_entry(self, entry, return_as: str) -> Optional[Dict]:
         """Extract metadata from ArXiv entry"""
         arxiv_id = entry.id.split("/abs/")[-1].split("v")[0]
         year = entry.get("published", "")[:4]
@@ -306,9 +289,7 @@ if __name__ == "__main__":
 
     # Search by title
     outputs["metadata_by_title_dict"] = source.search(title=TITLE)
-    outputs["metadata_by_title_json"] = source.search(
-        title=TITLE, return_as="json"
-    )
+    outputs["metadata_by_title_json"] = source.search(title=TITLE, return_as="json")
 
     # Search by DOI
     outputs["metadata_by_doi_dict"] = source.search(doi=DOI)

scitex/scholar/.legacy/metadata/doi/sources/_BaseDOISource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -96,9 +97,7 @@ class BaseDOISource(ABC):
     def session(self):
         if self._session is None:
             self._session = requests.Session()
-            self._session.headers.update(
-                {"User-Agent": self._get_user_agent()}
-            )
+            self._session.headers.update({"User-Agent": self._get_user_agent()})
         return self._session
 
     def _get_user_agent(self) -> str:
@@ -142,16 +141,12 @@ class BaseDOISource(ABC):
 
                 # Check for rate limits
                 if self.rate_limit_handler:
-                    rate_limit_info = (
-                        self.rate_limit_handler.detect_rate_limit(
-                            source=self.name.lower(), response=response
-                        )
+                    rate_limit_info = self.rate_limit_handler.detect_rate_limit(
+                        source=self.name.lower(), response=response
                     )
 
                     if rate_limit_info:
-                        self.rate_limit_handler.record_rate_limit(
-                            rate_limit_info
-                        )
+                        self.rate_limit_handler.record_rate_limit(rate_limit_info)
 
                         # If this isn't the last attempt, wait and retry
                         if attempt < max_retries:
@@ -170,9 +165,7 @@ class BaseDOISource(ABC):
                 # Check for successful response
                 if response.status_code == 200:
                     if self.rate_limit_handler:
-                        self.rate_limit_handler.record_success(
-                            self.name.lower()
-                        )
+                        self.rate_limit_handler.record_success(self.name.lower())
                         # Record success for adaptive rate limiting
                         self.rate_limit_handler.record_request_outcome(
                             self.name.lower(), success=True
@@ -181,9 +174,7 @@ class BaseDOISource(ABC):
                 elif response.status_code in [429, 503, 502, 504]:
                     # Server errors that might be temporary
                     if attempt < max_retries:
-                        wait_time = min(
-                            2**attempt, 30
-                        )  # Exponential backoff, max 30s
+                        wait_time = min(2**attempt, 30)  # Exponential backoff, max 30s
                         logger.info(
                             f"Server error {response.status_code} on attempt {attempt + 1}, "
                             f"waiting {wait_time}s before retry"
@@ -197,22 +188,16 @@ class BaseDOISource(ABC):
                         return response  # Return for caller to handle
                 else:
                     # Other HTTP errors
-                    logger.debug(
-                        f"HTTP {response.status_code} from {self.name}: {url}"
-                    )
+                    logger.debug(f"HTTP {response.status_code} from {self.name}: {url}")
                     return response
 
             except requests.exceptions.Timeout as e:
                 if self.rate_limit_handler:
-                    rate_limit_info = (
-                        self.rate_limit_handler.detect_rate_limit(
-                            source=self.name.lower(), exception=e
-                        )
+                    rate_limit_info = self.rate_limit_handler.detect_rate_limit(
+                        source=self.name.lower(), exception=e
                     )
                     if rate_limit_info and attempt < max_retries:
-                        self.rate_limit_handler.record_rate_limit(
-                            rate_limit_info
-                        )
+                        self.rate_limit_handler.record_rate_limit(rate_limit_info)
                         logger.info(
                             f"Timeout on attempt {attempt + 1}, waiting {rate_limit_info.wait_time:.1f}s"
                         )
@@ -231,9 +216,7 @@ class BaseDOISource(ABC):
                         f"Timeout after {max_retries + 1} attempts for {self.name}"
                     )
                     if self.rate_limit_handler:
-                        self.rate_limit_handler.record_failure(
-                            self.name.lower(), e
-                        )
+                        self.rate_limit_handler.record_failure(self.name.lower(), e)
                         # Record failure for adaptive rate limiting
                         self.rate_limit_handler.record_request_outcome(
                             self.name.lower(), success=False
@@ -253,9 +236,7 @@ class BaseDOISource(ABC):
                         f"Request failed after {max_retries + 1} attempts: {e}"
                     )
                     if self.rate_limit_handler:
-                        self.rate_limit_handler.record_failure(
-                            self.name.lower(), e
-                        )
+                        self.rate_limit_handler.record_failure(self.name.lower(), e)
                         # Record failure for adaptive rate limiting
                         self.rate_limit_handler.record_request_outcome(
                             self.name.lower(), success=False
@@ -265,9 +246,7 @@ class BaseDOISource(ABC):
             except Exception as e:
                 logger.error(f"Unexpected error in {self.name}: {e}")
                 if self.rate_limit_handler:
-                    self.rate_limit_handler.record_failure(
-                        self.name.lower(), e
-                    )
+                    self.rate_limit_handler.record_failure(self.name.lower(), e)
                     # Record failure for adaptive rate limiting
                     self.rate_limit_handler.record_request_outcome(
                         self.name.lower(), success=False
@@ -285,13 +264,9 @@ class BaseDOISource(ABC):
             return
 
         # Use advanced rate limiting with adaptive delays
-        wait_time = self.rate_limit_handler.get_wait_time_for_source(
-            self.name.lower()
-        )
+        wait_time = self.rate_limit_handler.get_wait_time_for_source(self.name.lower())
         if wait_time > 0:
-            logger.debug(
-                f"Rate limiting {self.name}: waiting {wait_time:.1f}s"
-            )
+            logger.debug(f"Rate limiting {self.name}: waiting {wait_time:.1f}s")
             time.sleep(wait_time)
 
     async def _apply_rate_limiting_async(self):
@@ -303,9 +278,7 @@ class BaseDOISource(ABC):
             return
 
         # Use advanced rate limiting with countdown and adaptive delays
-        wait_time = self.rate_limit_handler.get_wait_time_for_source(
-            self.name.lower()
-        )
+        wait_time = self.rate_limit_handler.get_wait_time_for_source(self.name.lower())
         if wait_time > 0:
             await self.rate_limit_handler.wait_with_countdown_async(
                 wait_time, self.name
@@ -324,9 +297,7 @@ class BaseDOISource(ABC):
         # Run sync search in executor
         loop = asyncio.get_event_loop()
         try:
-            result = await loop.run_in_executor(
-                None, self.search, title, year, authors
-            )
+            result = await loop.run_in_executor(None, self.search, title, year, authors)
             return result
         except Exception as e:
             logger.error(f"Error in async search for {self.name}: {e}")
@@ -372,4 +343,5 @@ class BaseDOISource(ABC):
         """
         return self.text_normalizer.is_title_match(title1, title2, threshold)
 
+
 # EOF

scitex/scholar/.legacy/metadata/doi/sources/_CrossRefLocalSource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------

scitex/scholar/.legacy/metadata/doi/sources/_CrossRefSource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -56,9 +57,7 @@ class CrossRefSource(BaseDOISource):
 
     def _search_by_doi(self, doi: str, return_as: str) -> Optional[Dict]:
         """Search by DOI directly"""
-        doi = doi.replace("https://doi.org/", "").replace(
-            "http://doi.org/", ""
-        )
+        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
         url = f"{self.base_url}/{doi}"
 
         try:
@@ -103,9 +102,7 @@ class CrossRefSource(BaseDOISource):
                 "dict",
                 "json",
             ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(
-                self.base_url, params=params, timeout=30
-            )
+            response = self.session.get(self.base_url, params=params, timeout=30)
             response.raise_for_status()
             data = response.json()
 
@@ -121,9 +118,7 @@ class CrossRefSource(BaseDOISource):
             logger.warn(f"CrossRef metadata error: {exc}")
             return None
 
-    def _extract_metadata_from_item(
-        self, item, return_as: str
-    ) -> Optional[Dict]:
+    def _extract_metadata_from_item(self, item, return_as: str) -> Optional[Dict]:
         """Extract metadata from CrossRef item"""
         item_title = " ".join(item.get("title", []))
         if item_title.endswith("."):
@@ -147,9 +142,7 @@ class CrossRefSource(BaseDOISource):
         container_titles = item.get("container-title", [])
         short_container_titles = item.get("short-container-title", [])
         journal = container_titles[0] if container_titles else None
-        short_journal = (
-            short_container_titles[0] if short_container_titles else None
-        )
+        short_journal = short_container_titles[0] if short_container_titles else None
         issn_list = item.get("ISSN", [])
         issn = issn_list[0] if issn_list else None
 
@@ -163,12 +156,8 @@ class CrossRefSource(BaseDOISource):
                 "title_sources": [self.name] if item_title else None,
                 "year": pub_year if pub_year else None,
                 "year_sources": [self.name] if pub_year else None,
-                "abstract": (
-                    item.get("abstract") if item.get("abstract") else None
-                ),
-                "abstract_sources": (
-                    [self.name] if item.get("abstract") else None
-                ),
+                "abstract": (item.get("abstract") if item.get("abstract") else None),
+                "abstract_sources": ([self.name] if item.get("abstract") else None),
                 "authors": extracted_authors if extracted_authors else None,
                 "authors_sources": [self.name] if extracted_authors else None,
             },
@@ -176,15 +165,9 @@ class CrossRefSource(BaseDOISource):
                 "journal": journal if journal else None,
                 "journal_sources": [self.name] if journal else None,
                 "short_journal": short_journal if short_journal else None,
-                "short_journal_sources": (
-                    [self.name] if short_journal else None
-                ),
-                "publisher": (
-                    item.get("publisher") if item.get("publisher") else None
-                ),
-                "publisher_sources": (
-                    [self.name] if item.get("publisher") else None
-                ),
+                "short_journal_sources": ([self.name] if short_journal else None),
+                "publisher": (item.get("publisher") if item.get("publisher") else None),
+                "publisher_sources": ([self.name] if item.get("publisher") else None),
                 "volume": item.get("volume") if item.get("volume") else None,
                 "volume_sources": [self.name] if item.get("volume") else None,
                 "issue": item.get("issue") if item.get("issue") else None,
@@ -194,9 +177,7 @@ class CrossRefSource(BaseDOISource):
             },
             "url": {
                 "doi": (
-                    "https://doi.org/" + item.get("DOI")
-                    if item.get("DOI")
-                    else None
+                    "https://doi.org/" + item.get("DOI") if item.get("DOI") else None
                 ),
                 "doi_sources": [self.name] if item.get("DOI") else None,
             },
@@ -226,9 +207,7 @@ if __name__ == "__main__":
 
     # Search by title
     outputs["metadata_by_title_dict"] = source.search(title=TITLE)
-    outputs["metadata_by_title_json"] = source.search(
-        title=TITLE, return_as="json"
-    )
+    outputs["metadata_by_title_json"] = source.search(title=TITLE, return_as="json")
 
     # Search by DOI
     outputs["metadata_by_doi_dict"] = source.search(doi=DOI)

scitex/scholar/.legacy/metadata/doi/sources/_OpenAlexSource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -56,9 +57,7 @@ class OpenAlexSource(BaseDOISource):
 
     def _search_by_doi(self, doi: str, return_as: str) -> Optional[Dict]:
         """Search by DOI directly"""
-        doi = doi.replace("https://doi.org/", "").replace(
-            "http://doi.org/", ""
-        )
+        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
 
         params = {
             "filter": f"doi:{doi}",
@@ -70,9 +69,7 @@ class OpenAlexSource(BaseDOISource):
                 "dict",
                 "json",
             ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(
-                self.base_url, params=params, timeout=30
-            )
+            response = self.session.get(self.base_url, params=params, timeout=30)
             response.raise_for_status()
 
             data = response.json()
@@ -109,9 +106,7 @@ class OpenAlexSource(BaseDOISource):
                 "dict",
                 "json",
             ], "return_as must be either of 'dict' or 'json'"
-            response = self.session.get(
-                self.base_url, params=params, timeout=30
-            )
+            response = self.session.get(self.base_url, params=params, timeout=30)
             response.raise_for_status()
 
             data = response.json()
@@ -128,9 +123,7 @@ class OpenAlexSource(BaseDOISource):
             logger.warn(f"OpenAlex metadata error: {exc}")
             return None
 
-    def _extract_metadata_from_work(
-        self, work, return_as: str
-    ) -> Optional[Dict]:
+    def _extract_metadata_from_work(self, work, return_as: str) -> Optional[Dict]:
         """Extract metadata from OpenAlex work"""
         work_title = work.get("title", "")
         if work_title and work_title.endswith("."):
@@ -199,8 +192,7 @@ class OpenAlexSource(BaseDOISource):
         citation_count_by_year = work.get("counts_by_year")
         if citation_count_by_year:
             citation_count_by_year = {
-                str(dd["year"]): dd["cited_by_count"]
-                for dd in citation_count_by_year
+                str(dd["year"]): dd["cited_by_count"] for dd in citation_count_by_year
             }
             citation_counts = {
                 "total": citation_count,
@@ -238,9 +230,7 @@ class OpenAlexSource(BaseDOISource):
                     if work.get("publication_year")
                     else None
                 ),
-                "year_sources": (
-                    [self.name] if work.get("publication_year") else None
-                ),
+                "year_sources": ([self.name] if work.get("publication_year") else None),
                 "authors": extracted_authors if extracted_authors else None,
                 "authors_sources": [self.name] if extracted_authors else None,
                 "keywords": keywords if keywords else None,
@@ -298,9 +288,7 @@ if __name__ == "__main__":
 
     # Search by title
     outputs["metadata_by_title_dict"] = source.search(title=TITLE)
-    outputs["metadata_by_title_json"] = source.search(
-        title=TITLE, return_as="json"
-    )
+    outputs["metadata_by_title_json"] = source.search(title=TITLE, return_as="json")
 
     # Search by DOI
     outputs["metadata_by_doi_dict"] = source.search(doi=DOI)