scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/integration/zotero/mapper.py

@@ -110,7 +110,9 @@ class ZoteroMapper:
             paper.metadata.id.doi_engines = ["zotero"]
 
         # Publication details
-        journal = data.get("publicationTitle", "") or data.get("journalAbbreviation", "")
+        journal = data.get("publicationTitle", "") or data.get(
+            "journalAbbreviation", ""
+        )
         if journal:
             paper.metadata.publication.journal = journal
             paper.metadata.publication.journal_engines = ["zotero"]
@@ -194,17 +196,18 @@ class ZoteroMapper:
                 if len(parts) == 2:
                     last_name = parts[0].strip()
                     first_name = parts[1].strip()
-                    creators.append({
-                        "creatorType": "author",
-                        "firstName": first_name,
-                        "lastName": last_name
-                    })
+                    creators.append(
+                        {
+                            "creatorType": "author",
+                            "firstName": first_name,
+                            "lastName": last_name,
+                        }
+                    )
                 else:
                     # Single name - treat as lastName
-                    creators.append({
-                        "creatorType": "author",
-                        "lastName": author.strip()
-                    })
+                    creators.append(
+                        {"creatorType": "author", "lastName": author.strip()}
+                    )
 
         # Build tags list
         tags = []
@@ -252,7 +255,9 @@ class ZoteroMapper:
         if paper.metadata.citation_count.total:
             extra_lines.append(f"Citation-Count: {paper.metadata.citation_count.total}")
         if paper.metadata.publication.impact_factor:
-            extra_lines.append(f"Impact-Factor: {paper.metadata.publication.impact_factor}")
+            extra_lines.append(
+                f"Impact-Factor: {paper.metadata.publication.impact_factor}"
+            )
 
         if extra_lines:
             zotero_item["extra"] = "\n".join(extra_lines)
@@ -273,7 +278,7 @@ class ZoteroMapper:
         import re
 
         # Try to find 4-digit year
-        match = re.search(r'\b(19|20)\d{2}\b', date_str)
+        match = re.search(r"\b(19|20)\d{2}\b", date_str)
         if match:
             try:
                 return int(match.group(0))

scitex/scholar/metadata_engines/.combined-SemanticScholarSource/_SemanticScholarSource.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -40,10 +41,10 @@ class SemanticScholarSource(BaseDOISource):
     def __init__(self, email: str = "research@example.com", api_key: str = None):
         super().__init__()  # Initialize base class to access utilities
         self.email = email
-        self.api_key = api_key or os.getenv('SEMANTIC_SCHOLAR_API_KEY')
+        self.api_key = api_key or os.getenv("SEMANTIC_SCHOLAR_API_KEY")
         self._session = None
         self.base_url = "https://api.semanticscholar.org/graph/v1"
-        
+
         # Enhanced rate limiting based on API key availability
         self._rate_limit_delay = 0.5 if self.api_key else 2.0  # Faster with API key
 
@@ -55,12 +56,12 @@ class SemanticScholarSource(BaseDOISource):
 
             self._session = requests.Session()
             headers = {"User-Agent": f"SciTeX/1.0 (mailto:{self.email})"}
-            
+
             # Add API key if available
             if self.api_key:
                 headers["x-api-key"] = self.api_key
                 logger.info("Using Semantic Scholar API key for enhanced rate limits")
-            
+
             self._session.headers.update(headers)
         return self._session
 
@@ -109,48 +110,54 @@ class SemanticScholarSource(BaseDOISource):
                 return doi
 
         return None
-        
-    def _extract_doi_from_paper(self, paper: dict, query_title: str, query_year: Optional[int]) -> Optional[str]:
+
+    def _extract_doi_from_paper(
+        self, paper: dict, query_title: str, query_year: Optional[int]
+    ) -> Optional[str]:
         """Enhanced DOI extraction from paper with multiple validation strategies."""
         paper_title = paper.get("title", "")
         paper_year = paper.get("year")
-        
+
         # Title matching
         if not paper_title or not self._is_title_match(query_title, paper_title):
             return None
-            
+
         # Year validation (if provided) - allow 2 year difference for robustness
         if query_year and paper_year:
             try:
-                paper_year_int = int(paper_year) if isinstance(paper_year, str) else paper_year
-                query_year_int = int(query_year) if isinstance(query_year, str) else query_year
-                
+                paper_year_int = (
+                    int(paper_year) if isinstance(paper_year, str) else paper_year
+                )
+                query_year_int = (
+                    int(query_year) if isinstance(query_year, str) else query_year
+                )
+
                 if abs(paper_year_int - query_year_int) > 2:
                     return None
             except (ValueError, TypeError):
                 pass  # Skip year validation if conversion fails
-        
+
         # Extract DOI from multiple possible fields
         external_ids = paper.get("externalIds", {})
-        
+
         # Primary DOI field
         if external_ids and "DOI" in external_ids:
             doi = external_ids["DOI"]
             if doi:
                 return doi.strip()
-        
+
         # Alternative DOI sources
         for field in ["doi", "DOI"]:
             if field in paper and paper[field]:
                 return paper[field].strip()
-        
+
         # Extract from URL field if present using utility
         paper_url = paper.get("url", "")
         if paper_url:
             doi = self.url_doi_extractor.extract_doi_from_url(paper_url)
             if doi:
                 return doi
-        
+
         return None
 
     def get_abstract(self, doi: str) -> Optional[str]:
@@ -164,9 +171,7 @@ class SemanticScholarSource(BaseDOISource):
                 data = response.json()
                 abstract = data.get("abstract")
                 if abstract:
-                    logger.debug(
-                        f"Found abstract from Semantic Scholar for DOI: {doi}"
-                    )
+                    logger.debug(f"Found abstract from Semantic Scholar for DOI: {doi}")
                     return abstract
             elif response.status_code == 404:
                 logger.debug(f"Paper not found in Semantic Scholar: {doi}")
@@ -213,7 +218,7 @@ class SemanticScholarSource(BaseDOISource):
             if doi:
                 paper_title = paper.get("title", "")
                 paper_year = paper.get("year")
-                
+
                 extracted_authors = []
                 for author in paper.get("authors", []):
                     if author.get("name"):
@@ -225,14 +230,10 @@ class SemanticScholarSource(BaseDOISource):
                     "journal": paper.get("venue"),
                     "journal_source": "semantic_scholar",
                     "year": (
-                        paper_year
-                        if isinstance(paper_year, int)
-                        else paper.get("year")
+                        paper_year if isinstance(paper_year, int) else paper.get("year")
                     ),
                     "abstract": paper.get("abstract"),
-                    "authors": (
-                        extracted_authors if extracted_authors else None
-                    ),
+                    "authors": (extracted_authors if extracted_authors else None),
                 }
 
         return None
@@ -249,32 +250,32 @@ class SemanticScholarSource(BaseDOISource):
         """Resolve DOI from CorpusID using Semantic Scholar API."""
         if not corpus_id or not corpus_id.isdigit():
             return None
-            
+
         try:
             url = f"{self.base_url}/paper/CorpusId:{corpus_id}"
             params = {"fields": "externalIds,title"}
-            
+
             response = self.session.get(url, params=params, timeout=15)
-            
+
             if response.status_code == 429:
                 raise requests.HTTPError("Rate limited", response=response)
             elif response.status_code == 404:
                 logger.debug(f"CorpusID {corpus_id} not found in Semantic Scholar")
                 return None
-            
+
             response.raise_for_status()
-            
+
             data = response.json()
-            external_ids = data.get('externalIds', {})
-            doi = external_ids.get('DOI')
-            
+            external_ids = data.get("externalIds", {})
+            doi = external_ids.get("DOI")
+
             if doi:
                 logger.info(f"Resolved CorpusID {corpus_id} → DOI: {doi}")
                 return doi
             else:
                 logger.debug(f"CorpusID {corpus_id} found but no DOI available")
                 return None
-                
+
         except requests.HTTPError as e:
             if e.response and e.response.status_code == 429:
                 raise  # Re-raise for retry logic
@@ -284,4 +285,5 @@ class SemanticScholarSource(BaseDOISource):
             logger.debug(f"CorpusID resolution error for {corpus_id}: {e}")
             return None
 
+
 # EOF

scitex/scholar/metadata_engines/.combined-SemanticScholarSource/_SemanticScholarSourceEnhanced.py

@@ -46,10 +46,12 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
         """Lazy load session with proper headers."""
         if self._session is None:
             self._session = requests.Session()
-            self._session.headers.update({
-                "User-Agent": f"SciTeX/1.0 (mailto:{self.email})",
-                "Accept": "application/json"
-            })
+            self._session.headers.update(
+                {
+                    "User-Agent": f"SciTeX/1.0 (mailto:{self.email})",
+                    "Accept": "application/json",
+                }
+            )
         return self._session
 
     @property
@@ -60,10 +62,14 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
     def rate_limit_delay(self) -> float:
         return 1.2  # Slightly more conservative than original
 
-    def _is_title_match(self, query_title: str, paper_title: str, threshold: float = 0.8) -> bool:
+    def _is_title_match(
+        self, query_title: str, paper_title: str, threshold: float = 0.8
+    ) -> bool:
         """Enhanced title matching using TextNormalizer utility."""
         # Use the advanced TextNormalizer from utils
-        is_match = self.text_normalizer.is_likely_same_title(query_title, paper_title, threshold)
+        is_match = self.text_normalizer.is_likely_same_title(
+            query_title, paper_title, threshold
+        )
 
         if logger.isEnabledFor(logging.DEBUG):
             logger.debug(f"Title match result: {is_match} (threshold: {threshold})")
@@ -89,9 +95,9 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
         """Enhanced search with better DOI extraction."""
         if not title:
             return None
-            
+
         logger.debug(f"Searching Semantic Scholar for: {title}")
-        
+
         url = f"{self.base_url}/paper/search"
         params = {
             "query": title,
@@ -101,81 +107,87 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
 
         try:
             response = self.session.get(url, params=params, timeout=30)
-            
+
             if response.status_code == 429:
                 raise requests.HTTPError("Rate limited", response=response)
-            
+
             response.raise_for_status()
             data = response.json()
-            
-            logger.debug(f"Semantic Scholar returned {len(data.get('data', []))} results")
-            
+
+            logger.debug(
+                f"Semantic Scholar returned {len(data.get('data', []))} results"
+            )
+
             papers = data.get("data", [])
-            
+
             # Try multiple matching strategies
             for paper in papers:
                 doi = self._extract_doi_from_paper(paper, title, year, authors)
                 if doi:
                     logger.info(f"Found DOI via enhanced Semantic Scholar: {doi}")
                     return doi
-            
+
             logger.debug("No DOI found in Semantic Scholar results")
             return None
-            
+
         except requests.RequestException as e:
             logger.debug(f"Semantic Scholar API error: {e}")
             return None
 
     def _extract_doi_from_paper(
-        self, 
-        paper: dict, 
-        query_title: str, 
-        query_year: Optional[int], 
-        query_authors: Optional[List[str]]
+        self,
+        paper: dict,
+        query_title: str,
+        query_year: Optional[int],
+        query_authors: Optional[List[str]],
     ) -> Optional[str]:
         """Extract DOI from paper with multiple validation strategies."""
-        
+
         paper_title = paper.get("title", "")
         paper_year = paper.get("year")
-        
+
         # Title matching
         if not self._is_title_match(query_title, paper_title):
             logger.debug(f"Title mismatch: '{paper_title}' vs '{query_title}'")
             return None
-        
+
         # Year validation (if provided)
         if query_year and paper_year:
             try:
-                paper_year_int = int(paper_year) if isinstance(paper_year, str) else paper_year
-                query_year_int = int(query_year) if isinstance(query_year, str) else query_year
-                
+                paper_year_int = (
+                    int(paper_year) if isinstance(paper_year, str) else paper_year
+                )
+                query_year_int = (
+                    int(query_year) if isinstance(query_year, str) else query_year
+                )
+
                 if abs(paper_year_int - query_year_int) > 2:  # Allow 2 year difference
                     logger.debug(f"Year mismatch: {paper_year_int} vs {query_year_int}")
                     return None
             except (ValueError, TypeError):
                 pass  # Skip year validation if conversion fails
-        
+
         # Extract DOI from multiple possible fields
         external_ids = paper.get("externalIds", {})
-        
+
         # Primary DOI field
         if external_ids and "DOI" in external_ids:
             doi = external_ids["DOI"]
             if doi:
                 return self._clean_doi(doi)
-        
+
         # Alternative DOI sources
         for field in ["doi", "DOI"]:
             if field in paper and paper[field]:
                 return self._clean_doi(paper[field])
-        
+
         # Extract from URL field if present using utility
         paper_url = paper.get("url", "")
         if paper_url:
             doi = self.url_doi_extractor.extract_doi_from_url(paper_url)
             if doi:
                 return doi
-        
+
         logger.debug(f"No DOI found in paper: {paper_title}")
         return None
 
@@ -194,10 +206,10 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
         try:
             response = self.session.get(url, params=params, timeout=30)
             response.raise_for_status()
-            
+
             data = response.json()
             return data.get("abstract")
-            
+
         except requests.RequestException as e:
             logger.debug(f"Error fetching abstract for DOI {doi}: {e}")
             return None
@@ -214,4 +226,4 @@ class SemanticScholarSourceEnhanced(BaseDOISource):
 # Export
 __all__ = ["SemanticScholarSourceEnhanced"]
 
-# EOF
+# EOF

scitex/scholar/metadata_engines/ScholarEngine.py

@@ -5,9 +5,8 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
-__FILE__ = (
-    "./src/scitex/scholar/metadata_engines/ScholarEngine.py"
-)
+
+__FILE__ = "./src/scitex/scholar/metadata_engines/ScholarEngine.py"
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 import json
@@ -114,7 +113,9 @@ class ScholarEngine:
             query_str = (
                 f"title: {title}"
                 if title
-                else f"doi: {doi}" if doi else "unknown query"
+                else f"doi: {doi}"
+                if doi
+                else "unknown query"
             )
             N_PRINT = 50
             if len(query_str) < N_PRINT:
@@ -211,9 +212,7 @@ class ScholarEngine:
                     logger.fail(f"Failed query '{query}': {error}")
 
         if dois:
-            batched_metadata = [
-                await self.search_async(doi=doi) for doi in tqdm(dois)
-            ]
+            batched_metadata = [await self.search_async(doi=doi) for doi in tqdm(dois)]
             _print_stats(dois, batched_metadata)
             return batched_metadata
 
@@ -242,10 +241,11 @@ class ScholarEngine:
                     self._engine_instances[name] = engine_classes[name]()
                 elif name == "CrossRefLocal":
                     # Get API URL from config (supports SCITEX_SCHOLAR_CROSSREF_API_URL env var)
-                    api_url = self.config.resolve("crossref_api_url", "http://127.0.0.1:3333")
+                    api_url = self.config.resolve(
+                        "crossref_api_url", "http://127.0.0.1:3333"
+                    )
                     self._engine_instances[name] = engine_classes[name](
-                        "research@example.com",
-                        api_url=api_url
+                        "research@example.com", api_url=api_url
                     )
                 else:
                     self._engine_instances[name] = engine_classes[name](
@@ -350,9 +350,7 @@ class ScholarEngine:
             return True
 
         first = metadata_list[0]
-        first_title = (
-            (first.get("basic", {}).get("title") or "").lower().strip()
-        )
+        first_title = (first.get("basic", {}).get("title") or "").lower().strip()
         first_year = first.get("basic", {}).get("year")
         first_authors = first.get("basic", {}).get("authors", [])
         first_author_surname = (
@@ -360,16 +358,12 @@ class ScholarEngine:
         )
 
         for metadata in metadata_list[1:]:
-            title = (
-                (metadata.get("basic", {}).get("title") or "").lower().strip()
-            )
+            title = (metadata.get("basic", {}).get("title") or "").lower().strip()
             year = metadata.get("basic", {}).get("year")
             authors = metadata.get("basic", {}).get("authors", [])
             first_author = authors[0]
             if first_author:
-                author_surname = (
-                    authors[0].split()[-1].lower() if authors else ""
-                )
+                author_surname = authors[0].split()[-1].lower() if authors else ""
             else:
                 author_surname = ""
 
@@ -393,16 +387,12 @@ class ScholarEngine:
 
         return True
 
-    def _validate_against_query(
-        self, metadata: Dict, query_title: str
-    ) -> bool:
+    def _validate_against_query(self, metadata: Dict, query_title: str) -> bool:
         """Validate metadata matches the original query with strict title matching."""
         if not query_title or not metadata:
             return True
 
-        paper_title = (
-            (metadata.get("basic", {}).get("title") or "").lower().strip()
-        )
+        paper_title = (metadata.get("basic", {}).get("title") or "").lower().strip()
         if not paper_title:
             return False
 
@@ -449,17 +439,13 @@ class ScholarEngine:
         query_title = getattr(self, "_last_query_title", None)
         valid_engines = {}
         for engine_name, metadata in engine_results.items():
-            if metadata and self._validate_against_query(
-                metadata, query_title
-            ):
+            if metadata and self._validate_against_query(metadata, query_title):
                 valid_engines[engine_name] = metadata
 
         if not valid_engines:
             # Return all engine results without validation if nothing matches
             # This allows partial enrichment even if title validation fails
-            logger.warning(
-                "No engines returned matching metadata, using all results"
-            )
+            logger.warning("No engines returned matching metadata, using all results")
             valid_engines = {k: v for k, v in engine_results.items() if v}
 
         # If still no valid engines, return empty structure
@@ -472,9 +458,7 @@ class ScholarEngine:
 
         # Merge all other valid engines
         for engine_name, metadata in list(valid_engines.items())[1:]:
-            base_metadata = self._merge_metadata_structures(
-                base_metadata, metadata
-            )
+            base_metadata = self._merge_metadata_structures(base_metadata, metadata)
 
         # Track all attempted searches
         if "system" not in base_metadata:
@@ -517,9 +501,7 @@ class ScholarEngine:
 
                 # Initialize engine lists if needed
                 if not isinstance(current_engines, list):
-                    current_engines = (
-                        [current_engines] if current_engines else []
-                    )
+                    current_engines = [current_engines] if current_engines else []
                     merged[section][f"{key}_engines"] = current_engines
 
                 # Convert single engine to list
@@ -529,15 +511,11 @@ class ScholarEngine:
                 should_replace = False
                 if current_value is None:
                     should_replace = True
-                elif engine_priority.get(
-                    new_engines[0], 0
-                ) > engine_priority.get(
+                elif engine_priority.get(new_engines[0], 0) > engine_priority.get(
                     current_engines[0] if current_engines else "", 0
                 ):
                     should_replace = True
-                elif isinstance(value, list) and isinstance(
-                    current_value, list
-                ):
+                elif isinstance(value, list) and isinstance(current_value, list):
                     if len(value) > len(current_value):
                         should_replace = True
                 elif isinstance(value, str) and isinstance(current_value, str):

scitex/scholar/metadata_engines/__init__.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------