scitex 2.4.3__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/.legacy/_tmp/search_engine/web/_CrossRefSearchEngine.py

@@ -27,180 +27,178 @@ logger = logging.getLogger(__name__)
 
 class CrossRefSearchEngine(BaseSearchEngine):
     """CrossRef search engine for academic papers."""
-    
-    def __init__(self, config: Optional[ScholarConfig] = None, api_key: Optional[str] = None, email: Optional[str] = None):
+
+    def __init__(
+        self,
+        config: Optional[ScholarConfig] = None,
+        api_key: Optional[str] = None,
+        email: Optional[str] = None,
+    ):
         super().__init__(name="crossref", rate_limit=0.5)
-        
+
         self.config = config or ScholarConfig()
-        
+
         # Use sophisticated config resolution: direct → config → env → default
         self.api_key = self.config.resolve(
-            key="crossref_api_key",
-            direct_val=api_key,
-            default=None,
-            type=str
+            key="crossref_api_key", direct_val=api_key, default=None, type=str
         )
-        
+
         self.email = self.config.resolve(
-            key="crossref_email", 
+            key="crossref_email",
             direct_val=email,
             default="research@example.com",
-            type=str
+            type=str,
         )
-        
+
         self.base_url = "https://api.crossref.org/works"
-        
+
     async def search_async(self, query: str, limit: int = 20, **kwargs) -> List[Paper]:
         """Search CrossRef for papers."""
         await self._rate_limit_async()
-        
+
         # Build query parameters
         params = {
-            'query': query,
-            'rows': min(limit, 1000),
-            'sort': 'relevance',
-            'order': 'desc'
+            "query": query,
+            "rows": min(limit, 1000),
+            "sort": "relevance",
+            "order": "desc",
         }
-        
+
         # Add filters for year if provided
         filters = []
-        if 'year_min' in kwargs and kwargs['year_min'] is not None:
+        if "year_min" in kwargs and kwargs["year_min"] is not None:
             filters.append(f"from-pub-date:{kwargs['year_min']}")
-        if 'year_max' in kwargs and kwargs['year_max'] is not None:
+        if "year_max" in kwargs and kwargs["year_max"] is not None:
             filters.append(f"until-pub-date:{kwargs['year_max']}")
-        
+
         if filters:
-            params['filter'] = ','.join(filters)
-        
+            params["filter"] = ",".join(filters)
+
         # Add API key if available
         if self.api_key:
-            params['key'] = self.api_key
-            
+            params["key"] = self.api_key
+
         # Headers with user agent
-        headers = {
-            'User-Agent': f'SciTeX/1.0 (mailto:{self.email})'
-        }
-        
+        headers = {"User-Agent": f"SciTeX/1.0 (mailto:{self.email})"}
+
         papers = []
-        
+
         try:
             async with aiohttp.ClientSession() as session:
-                async with session.get(self.base_url, params=params, headers=headers) as response:
+                async with session.get(
+                    self.base_url, params=params, headers=headers
+                ) as response:
                     if response.status == 200:
                         data = await response.json()
                         papers = self._parse_crossref_response(data)
                     else:
                         error_text = await response.text()
-                        logger.error(f"CrossRef search failed: {response.status} - {error_text}")
+                        logger.error(
+                            f"CrossRef search failed: {response.status} - {error_text}"
+                        )
                         raise SearchError(
                             query=query,
                             source="crossref",
-                            reason=f"API returned status {response.status}"
+                            reason=f"API returned status {response.status}",
                         )
-                        
+
         except asyncio.TimeoutError:
             logger.error("CrossRef search timed out")
-            raise SearchError(
-                query=query,
-                source="crossref", 
-                reason="Search timed out"
-            )
+            raise SearchError(query=query, source="crossref", reason="Search timed out")
         except Exception as e:
             logger.error(f"CrossRef search error: {e}")
-            raise SearchError(
-                query=query,
-                source="crossref",
-                reason=str(e)
-            )
-        
+            raise SearchError(query=query, source="crossref", reason=str(e))
+
         return papers
-    
+
     def _parse_crossref_response(self, data: Dict[str, Any]) -> List[Paper]:
         """Parse CrossRef API response into Paper objects."""
         papers = []
-        
-        items = data.get('message', {}).get('items', [])
-        
+
+        items = data.get("message", {}).get("items", [])
+
         for item in items:
             try:
                 # Extract basic metadata
-                title = ' '.join(item.get('title', ['No title']))
-                
+                title = " ".join(item.get("title", ["No title"]))
+
                 # Authors
                 authors = []
-                for author in item.get('author', []):
-                    given = author.get('given', '')
-                    family = author.get('family', '')
+                for author in item.get("author", []):
+                    given = author.get("given", "")
+                    family = author.get("family", "")
                     if given and family:
                         authors.append(f"{given} {family}")
                     elif family:
                         authors.append(family)
-                
+
                 # Abstract - CrossRef doesn't always have abstracts
-                abstract = item.get('abstract', '')
-                
+                abstract = item.get("abstract", "")
+
                 # Year from published-print or published-online
                 year = None
-                published = item.get('published-print') or item.get('published-online')
-                if published and 'date-parts' in published:
-                    date_parts = published['date-parts']
+                published = item.get("published-print") or item.get("published-online")
+                if published and "date-parts" in published:
+                    date_parts = published["date-parts"]
                     if date_parts and date_parts[0]:
                         year = str(date_parts[0][0])
-                
-                # Extract comprehensive journal information  
-                container_titles = item.get('container-title', [])
-                short_container_titles = item.get('short-container-title', [])
-                
+
+                # Extract comprehensive journal information
+                container_titles = item.get("container-title", [])
+                short_container_titles = item.get("short-container-title", [])
+
                 journal = container_titles[0] if container_titles else None
-                short_journal = short_container_titles[0] if short_container_titles else None
-                
+                short_journal = (
+                    short_container_titles[0] if short_container_titles else None
+                )
+
                 # Get ISSN (can be a list)
-                issn_list = item.get('ISSN', [])
+                issn_list = item.get("ISSN", [])
                 issn = issn_list[0] if issn_list else None
-                
+
                 # DOI
-                doi = item.get('DOI')
-                
+                doi = item.get("DOI")
+
                 # Citation count
-                citation_count = item.get('is-referenced-by-count', 0)
-                
+                citation_count = item.get("is-referenced-by-count", 0)
+
                 # Keywords/subjects
-                keywords = item.get('subject', [])
-                
+                keywords = item.get("subject", [])
+
                 # URL
-                url = item.get('URL')
-                
+                url = item.get("URL")
+
                 paper = Paper(
                     title=title,
                     authors=authors,
                     abstract=abstract,
-                    source='crossref',
+                    source="crossref",
                     year=year,
                     doi=doi,
                     journal=journal,
                     keywords=keywords,
                     citation_count=citation_count,
                     metadata={
-                        'citation_count_source': 'CrossRef',
-                        'url': url,
-                        'publisher': item.get('publisher'),
-                        'issn': issn,
-                        'issn_list': issn_list,
-                        'short_journal': short_journal,
-                        'volume': item.get('volume'),
-                        'issue': item.get('issue'),
-                        'type': item.get('type'),
-                        'score': item.get('score'),
-                        'journal_source': 'crossref'
-                    }
+                        "citation_count_source": "CrossRef",
+                        "url": url,
+                        "publisher": item.get("publisher"),
+                        "issn": issn,
+                        "issn_list": issn_list,
+                        "short_journal": short_journal,
+                        "volume": item.get("volume"),
+                        "issue": item.get("issue"),
+                        "type": item.get("type"),
+                        "score": item.get("score"),
+                        "journal_source": "crossref",
+                    },
                 )
-                
+
                 papers.append(paper)
-                
+
             except Exception as e:
                 logger.warning(f"Failed to parse CrossRef item: {e}")
                 continue
-        
+
         return papers
 
     async def fetch_by_id_async(self, identifier: str) -> Optional[Paper]:
@@ -208,29 +206,31 @@ class CrossRefSearchEngine(BaseSearchEngine):
         try:
             # Rate limiting
             await self._rate_limit_async()
-            
+
             # CrossRef API URL for fetching by DOI
             url = f"https://api.crossref.org/works/{identifier}"
-            
-            headers = {
-                'User-Agent': f'SciTeX/1.0 (mailto:{self.email})'
-            }
-            
+
+            headers = {"User-Agent": f"SciTeX/1.0 (mailto:{self.email})"}
+
             async with aiohttp.ClientSession() as session:
                 async with session.get(url, headers=headers) as response:
                     if response.status == 200:
                         data = await response.json()
                         # Parse the single work item
-                        items = [data.get('message', {})]
-                        papers = self._parse_crossref_response({'message': {'items': items}})
+                        items = [data.get("message", {})]
+                        papers = self._parse_crossref_response(
+                            {"message": {"items": items}}
+                        )
                         return papers[0] if papers else None
                     elif response.status == 404:
                         logger.debug(f"Paper not found for identifier: {identifier}")
                         return None
                     else:
-                        logger.error(f"CrossRef fetch_by_id_async failed: {response.status}")
+                        logger.error(
+                            f"CrossRef fetch_by_id_async failed: {response.status}"
+                        )
                         return None
-                        
+
         except Exception as e:
             logger.error(f"Error fetching paper by ID {identifier}: {e}")
             return None
@@ -241,40 +241,45 @@ class CrossRefSearchEngine(BaseSearchEngine):
             # Fetch the paper by DOI
             paper = await self.fetch_by_id_async(doi)
             return paper.citation_count if paper else None
-            
+
         except Exception as e:
             logger.error(f"Error getting citation count for {doi}: {e}")
             return None
 
-    async def resolve_doi_async(self, title: str, year: Optional[int] = None) -> Optional[str]:
+    async def resolve_doi_async(
+        self, title: str, year: Optional[int] = None
+    ) -> Optional[str]:
         """Resolve title to DOI using CrossRef search."""
         try:
             # Search for the paper by title
             search_query = title
             if year:
                 search_query += f" {year}"
-                
+
             papers = await self.search_async(search_query, limit=5)
-            
+
             # Look for exact or close title matches
             title_lower = title.lower().strip()
             for paper in papers:
                 if paper.doi and paper.title:
                     paper_title_lower = paper.title.lower().strip()
                     # Simple matching - could be improved with fuzzy matching
-                    if title_lower in paper_title_lower or paper_title_lower in title_lower:
+                    if (
+                        title_lower in paper_title_lower
+                        or paper_title_lower in title_lower
+                    ):
                         # If year is specified, prefer papers from that year
                         if year and paper.year and abs(paper.year - year) <= 1:
                             return paper.doi
                         elif not year:
                             return paper.doi
-            
+
             # If no exact match but we have results, return the first DOI
             if papers and papers[0].doi:
                 return papers[0].doi
-                
+
             return None
-            
+
         except Exception as e:
             logger.error(f"Error resolving DOI for title '{title}': {e}")
             return None
@@ -283,14 +288,14 @@ class CrossRefSearchEngine(BaseSearchEngine):
 async def main():
     """Test function for CrossRefSearchEngine."""
     from scitex.scholar.config import ScholarConfig
-    
+
     config = ScholarConfig()
     engine = CrossRefSearchEngine(config=config)
-    
+
     print("Testing CrossRef search engine...")
     print(f"Email: {engine.email}")
     print(f"API Key: {'Set' if engine.api_key else 'Not set'}")
-    
+
     try:
         papers = await engine.search_async("deep learning", limit=5)
         print(f"Found {len(papers)} papers:")
@@ -307,7 +312,8 @@ async def main():
 
 if __name__ == "__main__":
     import asyncio
+
     asyncio.run(main())
 
 
-# EOF
+# EOF

scitex/scholar/.legacy/_tmp/search_engine/web/_GoogleScholarSearchEngine.py

@@ -25,17 +25,18 @@ logger = logging.getLogger(__name__)
 
 class GoogleScholarSearchEngine(BaseSearchEngine):
     """Search engine for Google Scholar using scholarly package."""
-    
+
     def __init__(self, timeout: int = 10):
         super().__init__(name="google_scholar", rate_limit=2.0)
         self._scholarly = None
         self.timeout = timeout
-        
+
     def _init_scholarly(self):
         """Lazy load scholarly package."""
         if self._scholarly is None:
             try:
                 from scholarly import scholarly
+
                 self._scholarly = scholarly
             except ImportError:
                 raise ImportError(
@@ -43,15 +44,15 @@ class GoogleScholarSearchEngine(BaseSearchEngine):
                     "Install with: pip install scholarly"
                 )
         return self._scholarly
-    
+
     async def search_async(self, query: str, limit: int = 20, **kwargs) -> List[Paper]:
         """Search Google Scholar for papers."""
         papers = []
-        
+
         try:
             # Initialize scholarly
             scholarly = self._init_scholarly()
-            
+
             # Quick test to see if Google Scholar is accessible
             try:
                 test_search = scholarly.search_pubs("test")
@@ -62,27 +63,27 @@ class GoogleScholarSearchEngine(BaseSearchEngine):
                     raise SearchError(
                         query=query,
                         source="google_scholar",
-                        reason="Google Scholar is blocking automated access. Use PubMed or Semantic Scholar instead."
+                        reason="Google Scholar is blocking automated access. Use PubMed or Semantic Scholar instead.",
                     )
-            
+
             # Apply year filters if provided
-            year_min = kwargs.get('year_min')
-            year_max = kwargs.get('year_max')
-            
+            year_min = kwargs.get("year_min")
+            year_max = kwargs.get("year_max")
+
             # Build query with year filters
             search_query = query
             if year_min and year_max:
-                search_query += f" after:{year_min-1} before:{year_max+1}"
+                search_query += f" after:{year_min - 1} before:{year_max + 1}"
             elif year_min:
-                search_query += f" after:{year_min-1}"
+                search_query += f" after:{year_min - 1}"
             elif year_max:
-                search_query += f" before:{year_max+1}"
-            
+                search_query += f" before:{year_max + 1}"
+
             logger.info(f"Searching Google Scholar: {search_query}")
-            
+
             # Run search in executor with timeout to avoid blocking
             loop = asyncio.get_event_loop()
-            
+
             def search_with_limit():
                 """Search and limit results to avoid hanging."""
                 results = []
@@ -97,107 +98,112 @@ class GoogleScholarSearchEngine(BaseSearchEngine):
                     if "Cannot Fetch" in str(e) or "403" in str(e):
                         logger.info("Google Scholar is blocking automated requests.")
                 return results
-            
+
             # Apply timeout to prevent hanging
             try:
                 search_results = await asyncio.wait_for(
-                    loop.run_in_executor(None, search_with_limit),
-                    timeout=self.timeout
+                    loop.run_in_executor(None, search_with_limit), timeout=self.timeout
                 )
             except asyncio.TimeoutError:
-                logger.warning(f"Google Scholar search timed out after {self.timeout} seconds")
+                logger.warning(
+                    f"Google Scholar search timed out after {self.timeout} seconds"
+                )
                 raise SearchError(
                     query=search_query,
                     source="google_scholar",
-                    reason=f"Search timed out after {self.timeout} seconds. Google Scholar may be blocking requests."
+                    reason=f"Search timed out after {self.timeout} seconds. Google Scholar may be blocking requests.",
                 )
-            
+
             # Process results
             for result in search_results:
                 try:
                     # Extract basic info from search result
-                    bib = result.get('bib', {})
-                    
-                    title = bib.get('title', '')
+                    bib = result.get("bib", {})
+
+                    title = bib.get("title", "")
                     if not title:
                         continue
-                    
+
                     # Authors
-                    authors = bib.get('author', '').split(' and ')
-                    if not authors or authors == ['']:
+                    authors = bib.get("author", "").split(" and ")
+                    if not authors or authors == [""]:
                         authors = []
-                    
+
                     # Abstract (often not available in search results)
-                    abstract = bib.get('abstract', '')
-                    
+                    abstract = bib.get("abstract", "")
+
                     # Year
                     year = None
-                    pub_year = bib.get('pub_year')
+                    pub_year = bib.get("pub_year")
                     if pub_year:
                         try:
                             year = int(pub_year)
                         except:
                             pass
-                    
+
                     # Journal/Venue
-                    journal = bib.get('venue', '')
-                    
+                    journal = bib.get("venue", "")
+
                     # Citation count
-                    citation_count = result.get('num_citations', 0)
-                    
+                    citation_count = result.get("num_citations", 0)
+
                     # URL
-                    url = result.get('pub_url', '')
-                    
+                    url = result.get("pub_url", "")
+
                     # Try to extract DOI from URL or other fields
                     doi = None
-                    if 'doi.org/' in url:
-                        doi = url.split('doi.org/')[-1]
-                    
+                    if "doi.org/" in url:
+                        doi = url.split("doi.org/")[-1]
+
                     # Create Paper object
                     paper = Paper(
                         title=title,
                         authors=authors,
-                        abstract=abstract or "Abstract not available from Google Scholar search",
-                        source='google_scholar',
+                        abstract=abstract
+                        or "Abstract not available from Google Scholar search",
+                        source="google_scholar",
                         year=year,
                         journal=journal,
                         doi=doi,
                         citation_count=citation_count,
                         metadata={
-                            'google_scholar_url': url,
-                            'google_scholar_id': result.get('author_id', ''),
-                            'eprint_url': result.get('eprint_url', '')
-                        }
+                            "google_scholar_url": url,
+                            "google_scholar_id": result.get("author_id", ""),
+                            "eprint_url": result.get("eprint_url", ""),
+                        },
                     )
-                    
+
                     papers.append(paper)
-                    
+
                 except Exception as e:
                     logger.warning(f"Failed to parse Google Scholar result: {e}")
                     continue
-                    
+
         except ImportError as e:
             logger.error(f"Google Scholar search unavailable: {e}")
             raise SearchError(
                 query=query,
                 source="google_scholar",
-                reason=f"Google Scholar search unavailable: {e}"
+                reason=f"Google Scholar search unavailable: {e}",
             )
         except Exception as e:
             logger.error(f"Google Scholar search failed: {e}")
-            if "robot" in str(e).lower() or "captcha" in str(e).lower() or "Cannot Fetch" in str(e):
+            if (
+                "robot" in str(e).lower()
+                or "captcha" in str(e).lower()
+                or "Cannot Fetch" in str(e)
+            ):
                 raise SearchError(
                     query=query,
                     source="google_scholar",
-                    reason="Google Scholar is blocking automated access. Consider using PubMed or Semantic Scholar instead."
+                    reason="Google Scholar is blocking automated access. Consider using PubMed or Semantic Scholar instead.",
                 )
             raise SearchError(
-                query=query,
-                source="google_scholar",
-                reason=f"Search failed: {e}"
+                query=query, source="google_scholar", reason=f"Search failed: {e}"
             )
-        
+
         logger.info(f"Found {len(papers)} papers from Google Scholar")
         return papers
 
-# EOF
+
+# EOF