scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/metadata_engines/individual/SemanticScholarEngine.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -34,9 +35,7 @@ logger = logging.getLogger(__name__)
 class SemanticScholarEngine(BaseDOIEngine):
     """Combined Semantic Scholar engine with enhanced features."""
 
-    def __init__(
-        self, email: str = "research@example.com", api_key: str = None
-    ):
+    def __init__(self, email: str = "research@example.com", api_key: str = None):
         super().__init__(email)
         self.api_key = api_key or os.getenv("SEMANTIC_SCHOLAR_API_KEY")
         self.base_url = "https://api.semanticscholar.org/graph/v1"
@@ -105,13 +104,9 @@ class SemanticScholarEngine(BaseDOIEngine):
         """Search by DOI directly"""
         self._handle_rate_limit()
 
-        doi = doi.replace("https://doi.org/", "").replace(
-            "http://doi.org/", ""
-        )
+        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
         url = f"{self.base_url}/paper/{doi}"
-        params = {
-            "fields": "title,year,authors,externalIds,url,venue,abstract"
-        }
+        params = {"fields": "title,year,authors,externalIds,url,venue,abstract"}
 
         try:
             assert return_as in [
@@ -181,8 +176,7 @@ class SemanticScholarEngine(BaseDOIEngine):
                 paper_title = paper.get("title", "")
                 paper_year = paper.get("year")
                 paper_authors = [
-                    author.get("name", "")
-                    for author in paper.get("authors", [])
+                    author.get("name", "") for author in paper.get("authors", [])
                 ]
 
                 # Check title match
@@ -226,9 +220,7 @@ class SemanticScholarEngine(BaseDOIEngine):
         wait=wait_exponential(multiplier=2, min=4, max=20),
         retry=retry_if_exception_type((requests.ConnectionError,)),
     )
-    def _search_by_corpus_id(
-        self, corpus_id: str, return_as: str
-    ) -> Optional[Dict]:
+    def _search_by_corpus_id(self, corpus_id: str, return_as: str) -> Optional[Dict]:
         """Search by Corpus ID directly"""
         if not corpus_id.isdigit():
             corpus_id = corpus_id.replace("CorpusId:", "")
@@ -236,9 +228,7 @@ class SemanticScholarEngine(BaseDOIEngine):
         self._handle_rate_limit()
 
         url = f"{self.base_url}/paper/CorpusId:{corpus_id}"
-        params = {
-            "fields": "title,year,authors,externalIds,url,venue,abstract"
-        }
+        params = {"fields": "title,year,authors,externalIds,url,venue,abstract"}
 
         try:
             assert return_as in [
@@ -251,9 +241,7 @@ class SemanticScholarEngine(BaseDOIEngine):
                 raise requests.ConnectionError("Rate limit exceeded")
 
             if response.status_code == 404:
-                logger.warning(
-                    f"Semantic Scholar Corpus ID not found: {corpus_id}"
-                )
+                logger.warning(f"Semantic Scholar Corpus ID not found: {corpus_id}")
                 return None
 
             response.raise_for_status()
@@ -303,12 +291,8 @@ class SemanticScholarEngine(BaseDOIEngine):
                 "title_engines": [self.name] if paper_title else None,
                 "year": paper_year if paper_year else None,
                 "year_engines": [self.name] if paper_year else None,
-                "abstract": (
-                    paper.get("abstract") if paper.get("abstract") else None
-                ),
-                "abstract_engines": (
-                    [self.name] if paper.get("abstract") else None
-                ),
+                "abstract": (paper.get("abstract") if paper.get("abstract") else None),
+                "abstract_engines": ([self.name] if paper.get("abstract") else None),
                 "authors": extracted_authors if extracted_authors else None,
                 "authors_engines": [self.name] if extracted_authors else None,
             },
@@ -323,7 +307,9 @@ class SemanticScholarEngine(BaseDOIEngine):
                 "publisher_engines": [self.name] if paper.get("url") else None,
                 "arxiv": f"https://arxiv.org/abs/{arxiv_id}" if arxiv_id else None,
                 "arxiv_engines": [self.name] if arxiv_id else None,
-                "corpus_id": f"https://www.semanticscholar.org/paper/{corpus_id}" if corpus_id else None,
+                "corpus_id": f"https://www.semanticscholar.org/paper/{corpus_id}"
+                if corpus_id
+                else None,
                 "corpus_id_engines": [self.name] if corpus_id else None,
             },
             "system": {
@@ -341,7 +327,9 @@ class SemanticScholarEngine(BaseDOIEngine):
     def _clean_doi(self, doi: str) -> str:
         return doi.strip() if doi else doi
 
-    async def convert_corpus_id_to_doi_async(self, corpus_id: str, page) -> Optional[str]:
+    async def convert_corpus_id_to_doi_async(
+        self, corpus_id: str, page
+    ) -> Optional[str]:
         """Convert Semantic Scholar Corpus ID to DOI by navigating to page and extracting.
 
         Args:
@@ -373,7 +361,9 @@ class SemanticScholarEngine(BaseDOIEngine):
             if doi_meta:
                 doi = await doi_meta.get_attribute("content")
                 if doi:
-                    logger.info(f"Found DOI in meta tag for Corpus ID {corpus_id}: {doi}")
+                    logger.info(
+                        f"Found DOI in meta tag for Corpus ID {corpus_id}: {doi}"
+                    )
                     return doi
 
             # Method 2: Look for DOI link in page
@@ -388,14 +378,19 @@ class SemanticScholarEngine(BaseDOIEngine):
             # Method 3: Search for DOI pattern in page text
             content = await page.content()
             import re
-            doi_pattern = r'10\.\d{4,}/[-._;()/:\w]+'
+
+            doi_pattern = r"10\.\d{4,}/[-._;()/:\w]+"
             matches = re.findall(doi_pattern, content)
             if matches:
                 doi = matches[0]  # Take first match
-                logger.info(f"Found DOI pattern in content for Corpus ID {corpus_id}: {doi}")
+                logger.info(
+                    f"Found DOI pattern in content for Corpus ID {corpus_id}: {doi}"
+                )
                 return doi
 
-            logger.warning(f"No DOI found on Semantic Scholar page for Corpus ID {corpus_id}")
+            logger.warning(
+                f"No DOI found on Semantic Scholar page for Corpus ID {corpus_id}"
+            )
             return None
 
         except Exception as e:
@@ -417,9 +412,7 @@ if __name__ == "__main__":
 
     # Search by title
     outputs["metadata_by_title_dict"] = engine.search(title=TITLE)
-    outputs["metadata_by_title_json"] = engine.search(
-        title=TITLE, return_as="json"
-    )
+    outputs["metadata_by_title_json"] = engine.search(title=TITLE, return_as="json")
 
     # Emptry Result
     outputs["empty_dict"] = engine._create_minimal_metadata(return_as="dict")

scitex/scholar/metadata_engines/individual/URLDOIEngine.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -78,9 +79,7 @@ class URLDOIEngine(BaseDOIEngine):
 
     def _search_by_doi(self, doi: str, return_as: str) -> Optional[Dict]:
         """Search by DOI directly"""
-        doi = doi.replace("https://doi.org/", "").replace(
-            "http://doi.org/", ""
-        )
+        doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "")
 
         try:
             assert return_as in [
@@ -184,9 +183,7 @@ class URLDOIEngine(BaseDOIEngine):
                         return metadata
                     if return_as == "json":
                         return json.dumps(metadata, indent=2)
-                return self._create_minimal_metadata(
-                    doi=doi, return_as=return_as
-                )
+                return self._create_minimal_metadata(doi=doi, return_as=return_as)
 
             # Continue with other extractions (IEEE, Semantic Scholar)
             ieee_id = self._extract_ieee_id(url)
@@ -219,9 +216,7 @@ class URLDOIEngine(BaseDOIEngine):
                         return metadata
                     if return_as == "json":
                         return json.dumps(metadata, indent=2)
-                return self._create_minimal_metadata(
-                    doi=doi, return_as=return_as
-                )
+                return self._create_minimal_metadata(doi=doi, return_as=return_as)
 
             semantic_id = self._extract_semantic_corpus_id(url)
             if semantic_id:
@@ -323,22 +318,20 @@ class URLDOIEngine(BaseDOIEngine):
                 if semantic_id.isdigit():
                     url = f"https://api.semanticscholar.org/graph/v1/paper/CorpusId:{semantic_id}"
                 else:
-                    url = f"https://api.semanticscholar.org/graph/v1/paper/{semantic_id}"
+                    url = (
+                        f"https://api.semanticscholar.org/graph/v1/paper/{semantic_id}"
+                    )
 
                 params = {"fields": "externalIds,title,authors"}
                 headers = {"User-Agent": f"SciTeX/1.0 (mailto:{self.email})"}
                 if self.api_key:
                     headers["x-api-key"] = self.api_key
 
-                response = requests.get(
-                    url, params=params, headers=headers, timeout=15
-                )
+                response = requests.get(url, params=params, headers=headers, timeout=15)
 
                 if response.status_code == 429:
                     if attempt < max_retries - 1:
-                        delay = (base_delay * (2**attempt)) + random.uniform(
-                            0.5, 1.5
-                        )
+                        delay = (base_delay * (2**attempt)) + random.uniform(0.5, 1.5)
                         time.sleep(delay)
                         continue
                     return None
@@ -359,14 +352,10 @@ class URLDOIEngine(BaseDOIEngine):
             except requests.HTTPError as exc:
                 if exc.response and exc.response.status_code == 429:
                     continue
-                logger.debug(
-                    f"Semantic Scholar HTTP error for {semantic_id}: {exc}"
-                )
+                logger.debug(f"Semantic Scholar HTTP error for {semantic_id}: {exc}")
                 return None
             except Exception as exc:
-                logger.debug(
-                    f"Semantic Scholar lookup failed for {semantic_id}: {exc}"
-                )
+                logger.debug(f"Semantic Scholar lookup failed for {semantic_id}: {exc}")
                 return None
         return None
 

scitex/scholar/metadata_engines/individual/_BaseDOIEngine.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -101,9 +102,7 @@ class BaseDOIEngine(ABC):
     def session(self):
         if self._session is None:
             self._session = requests.Session()
-            self._session.headers.update(
-                {"User-Agent": self._get_user_agent()}
-            )
+            self._session.headers.update({"User-Agent": self._get_user_agent()})
         return self._session
 
     def _get_user_agent(self) -> str:
@@ -132,10 +131,10 @@ class BaseDOIEngine(ABC):
         # Remove common meta characters but keep alphanumeric, spaces, and basic punctuation
         # Keep: letters, numbers, spaces, hyphens, periods, commas
         # Remove: ()[]{}!@#$%^&*+=<>?/\|~`"':;
-        cleaned = re.sub(r'[()[\]{}!@#$%^&*+=<>?/\\|~`"\':;]', ' ', query)
+        cleaned = re.sub(r'[()[\]{}!@#$%^&*+=<>?/\\|~`"\':;]', " ", query)
 
         # Collapse multiple spaces into one
-        cleaned = re.sub(r'\s+', ' ', cleaned)
+        cleaned = re.sub(r"\s+", " ", cleaned)
 
         return cleaned.strip()
 
@@ -176,16 +175,12 @@ class BaseDOIEngine(ABC):
 
                 # Check for rate limits
                 if self.rate_limit_handler:
-                    rate_limit_info = (
-                        self.rate_limit_handler.detect_rate_limit(
-                            engine=self.name.lower(), response=response
-                        )
+                    rate_limit_info = self.rate_limit_handler.detect_rate_limit(
+                        engine=self.name.lower(), response=response
                     )
 
                     if rate_limit_info:
-                        self.rate_limit_handler.record_rate_limit(
-                            rate_limit_info
-                        )
+                        self.rate_limit_handler.record_rate_limit(rate_limit_info)
 
                         # If this isn't the last attempt, wait and retry
                         if attempt < max_retries:
@@ -204,9 +199,7 @@ class BaseDOIEngine(ABC):
                 # Check for successful response
                 if response.status_code == 200:
                     if self.rate_limit_handler:
-                        self.rate_limit_handler.record_success(
-                            self.name.lower()
-                        )
+                        self.rate_limit_handler.record_success(self.name.lower())
                         # Record success for adaptive rate limiting
                         self.rate_limit_handler.record_request_outcome(
                             self.name.lower(), success=True
@@ -215,9 +208,7 @@ class BaseDOIEngine(ABC):
                 elif response.status_code in [429, 503, 502, 504]:
                     # Server errors that might be temporary
                     if attempt < max_retries:
-                        wait_time = min(
-                            2**attempt, 30
-                        )  # Exponential backoff, max 30s
+                        wait_time = min(2**attempt, 30)  # Exponential backoff, max 30s
                         logger.info(
                             f"Server error {response.status_code} on attempt {attempt + 1}, "
                             f"waiting {wait_time}s before retry"
@@ -231,22 +222,16 @@ class BaseDOIEngine(ABC):
                         return response  # Return for caller to handle
                 else:
                     # Other HTTP errors
-                    logger.debug(
-                        f"HTTP {response.status_code} from {self.name}: {url}"
-                    )
+                    logger.debug(f"HTTP {response.status_code} from {self.name}: {url}")
                     return response
 
             except requests.exceptions.Timeout as e:
                 if self.rate_limit_handler:
-                    rate_limit_info = (
-                        self.rate_limit_handler.detect_rate_limit(
-                            engine=self.name.lower(), exception=e
-                        )
+                    rate_limit_info = self.rate_limit_handler.detect_rate_limit(
+                        engine=self.name.lower(), exception=e
                     )
                     if rate_limit_info and attempt < max_retries:
-                        self.rate_limit_handler.record_rate_limit(
-                            rate_limit_info
-                        )
+                        self.rate_limit_handler.record_rate_limit(rate_limit_info)
                         logger.info(
                             f"Timeout on attempt {attempt + 1}, waiting {rate_limit_info.wait_time:.1f}s"
                         )
@@ -265,9 +250,7 @@ class BaseDOIEngine(ABC):
                         f"Timeout after {max_retries + 1} attempts for {self.name}"
                     )
                     if self.rate_limit_handler:
-                        self.rate_limit_handler.record_failure(
-                            self.name.lower(), e
-                        )
+                        self.rate_limit_handler.record_failure(self.name.lower(), e)
                         # Record failure for adaptive rate limiting
                         self.rate_limit_handler.record_request_outcome(
                             self.name.lower(), success=False
@@ -287,9 +270,7 @@ class BaseDOIEngine(ABC):
                         f"Request failed after {max_retries + 1} attempts: {e}"
                     )
                     if self.rate_limit_handler:
-                        self.rate_limit_handler.record_failure(
-                            self.name.lower(), e
-                        )
+                        self.rate_limit_handler.record_failure(self.name.lower(), e)
                         # Record failure for adaptive rate limiting
                         self.rate_limit_handler.record_request_outcome(
                             self.name.lower(), success=False
@@ -299,9 +280,7 @@ class BaseDOIEngine(ABC):
             except Exception as e:
                 logger.error(f"Unexpected error in {self.name}: {e}")
                 if self.rate_limit_handler:
-                    self.rate_limit_handler.record_failure(
-                        self.name.lower(), e
-                    )
+                    self.rate_limit_handler.record_failure(self.name.lower(), e)
                     # Record failure for adaptive rate limiting
                     self.rate_limit_handler.record_request_outcome(
                         self.name.lower(), success=False
@@ -319,13 +298,9 @@ class BaseDOIEngine(ABC):
             return
 
         # Use advanced rate limiting with adaptive delays
-        wait_time = self.rate_limit_handler.get_wait_time_for_engine(
-            self.name.lower()
-        )
+        wait_time = self.rate_limit_handler.get_wait_time_for_engine(self.name.lower())
         if wait_time > 0:
-            logger.debug(
-                f"Rate limiting {self.name}: waiting {wait_time:.1f}s"
-            )
+            logger.debug(f"Rate limiting {self.name}: waiting {wait_time:.1f}s")
             time.sleep(wait_time)
 
     async def _apply_rate_limiting_async(self):
@@ -337,9 +312,7 @@ class BaseDOIEngine(ABC):
             return
 
         # Use advanced rate limiting with countdown and adaptive delays
-        wait_time = self.rate_limit_handler.get_wait_time_for_engine(
-            self.name.lower()
-        )
+        wait_time = self.rate_limit_handler.get_wait_time_for_engine(self.name.lower())
         if wait_time > 0:
             await self.rate_limit_handler.wait_with_countdown_async(
                 wait_time, self.name
@@ -358,9 +331,7 @@ class BaseDOIEngine(ABC):
         # Run sync search in executor
         loop = asyncio.get_event_loop()
         try:
-            result = await loop.run_in_executor(
-                None, self.search, title, year, authors
-            )
+            result = await loop.run_in_executor(None, self.search, title, year, authors)
             return result
         except Exception as e:
             logger.error(f"Error in async search for {self.name}: {e}")

scitex/scholar/metadata_engines/utils/_PubMedConverter.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -74,8 +75,7 @@ class PubMedConverter:
 
         # Compile patterns
         self.compiled_patterns = [
-            re.compile(pattern, re.IGNORECASE)
-            for pattern in self.PMID_PATTERNS
+            re.compile(pattern, re.IGNORECASE) for pattern in self.PMID_PATTERNS
         ]
 
     @retry(
@@ -333,9 +333,7 @@ class PubMedConverter:
 
         return results
 
-    async def bibtex_entries2dois_async(
-        self, entries: List[Dict]
-    ) -> Dict[str, str]:
+    async def bibtex_entries2dois_async(self, entries: List[Dict]) -> Dict[str, str]:
         """
         Async version of batch PMID to DOI conversion.
 
@@ -386,9 +384,7 @@ class PubMedConverter:
         return converted_dois
 
 
-def pmid2doi(
-    pmid: Union[str, int], pubmed_email=None, api_key=None, config=None
-):
+def pmid2doi(pmid: Union[str, int], pubmed_email=None, api_key=None, config=None):
     config = config or ScholarConfig()
     pubmed_email = config.resolve("pubmed_email", pubmed_email)
     pubmed_api_key = config.resolve("pubmed_api_key", api_key)

scitex/scholar/metadata_engines/utils/_URLDOIExtractor.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -49,8 +50,7 @@ class URLDOIExtractor:
 
     # ScienceDirect PII pattern
     SCIENCEDIRECT_PII_PATTERN = re.compile(
-        r"sciencedirect\.com/science/article/pii/([A-Z0-9]+)",
-        re.IGNORECASE
+        r"sciencedirect\.com/science/article/pii/([A-Z0-9]+)", re.IGNORECASE
     )
 
     # Valid DOI pattern for validation
@@ -60,8 +60,7 @@ class URLDOIExtractor:
         """Initialize the URL DOI extractor."""
         self.name = self.__class__.__name__
         self.compiled_patterns = [
-            re.compile(pattern, re.IGNORECASE)
-            for pattern in self.DOI_URL_PATTERNS
+            re.compile(pattern, re.IGNORECASE) for pattern in self.DOI_URL_PATTERNS
         ]
 
     def extract_doi_from_url(self, url: str) -> Optional[str]:
@@ -104,9 +103,7 @@ class URLDOIExtractor:
 
                 # Validate DOI format
                 if self._is_valid_doi(potential_doi):
-                    logger.debug(
-                        f"Extracted DOI '{potential_doi}' from URL: {url}"
-                    )
+                    logger.debug(f"Extracted DOI '{potential_doi}' from URL: {url}")
                     return potential_doi
 
         return None
@@ -177,9 +174,7 @@ class URLDOIExtractor:
                 results[str(i)] = extracted_doi
 
         if results:
-            logger.info(
-                f"URLDOIExtractor: Extracted DOIs from {len(results)} entries"
-            )
+            logger.info(f"URLDOIExtractor: Extracted DOIs from {len(results)} entries")
 
         return results
 

scitex/scholar/metadata_engines/utils/__init__.py

@@ -5,11 +5,13 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
 from ._PubMedConverter import PubMedConverter, pmid2doi
+
 # Import TextNormalizer from central location
 from scitex.scholar.utils.text import TextNormalizer
 from ._URLDOIExtractor import URLDOIExtractor

scitex/scholar/metadata_engines/utils/_metadata2bibtex.py

@@ -5,10 +5,12 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
+
 def metadata2bibtex(metadata, key=None):
     """Convert complete metadata structure to BibTeX entry."""
     if not key:
@@ -100,4 +102,5 @@ def _format_pages(publication_data):
         return first_page
     return None
 
+
 # EOF

scitex/scholar/metadata_engines/utils/_standardize_metadata.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/engines/utils/_standardize_metadata.py"
-)
+
+__FILE__ = "./src/scitex/scholar/engines/utils/_standardize_metadata.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------