scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/.legacy/_tmp/search_engine/local/_VectorSearchEngine.py

@@ -28,8 +28,10 @@ logger = logging.getLogger(__name__)
 
 class VectorSearchEngine(BaseSearchEngine):
     """Vector similarity search using sentence embeddings."""
-    
-    def __init__(self, index_path: Optional[Path] = None, model_name: str = "all-MiniLM-L6-v2"):
+
+    def __init__(
+        self, index_path: Optional[Path] = None, model_name: str = "all-MiniLM-L6-v2"
+    ):
         super().__init__(name="vector", rate_limit=0)
         self.index_path = index_path or get_scholar_dir() / "vector_index.pkl"
         self.model_name = model_name
@@ -37,51 +39,51 @@ class VectorSearchEngine(BaseSearchEngine):
         self._papers = []
         self._embeddings = None
         self._load_index()
-    
+
     async def search_async(self, query: str, limit: int = 20, **kwargs) -> List[Paper]:
         """Search using vector similarity."""
         # Vector search is CPU-bound, use thread
         return await asyncio.to_thread(self._search_sync, query, limit)
-    
+
     def _search_sync(self, query: str, limit: int) -> List[Paper]:
         """Synchronous vector search implementation."""
         if not self._embeddings or not self._papers:
             return []
-        
+
         # Lazy load model
         if self._model is None:
             self._load_model()
-        
+
         # Encode query
         query_embedding = self._model.encode([query])[0]
-        
+
         # Calculate similarities
         similarities = np.dot(self._embeddings, query_embedding)
-        
+
         # Get top results
         top_indices = np.argsort(similarities)[-limit:][::-1]
-        
+
         results = []
         for idx in top_indices:
             if idx < len(self._papers):
                 results.append(self._papers[idx])
-        
+
         return results
-    
+
     def add_papers(self, papers: List[Paper]) -> None:
         """Add papers to vector index."""
         if self._model is None:
             self._load_model()
-        
+
         # Create searchable text for each paper
         texts = []
         for paper in papers:
             text = f"{paper.title} {paper.abstract}"
             texts.append(text)
-        
+
         # Encode papers
         new_embeddings = self._model.encode(texts)
-        
+
         # Add to index
         if self._embeddings is None:
             self._embeddings = new_embeddings
@@ -89,38 +91,39 @@ class VectorSearchEngine(BaseSearchEngine):
         else:
             self._embeddings = np.vstack([self._embeddings, new_embeddings])
             self._papers.extend(papers)
-        
+
         # Save index
         self._save_index()
-    
+
     def _load_model(self) -> None:
         """Load sentence transformer model."""
         try:
             from sentence_transformers import SentenceTransformer
+
             self._model = SentenceTransformer(self.model_name)
         except ImportError:
-            logger.warning("sentence-transformers not installed. Vector search disabled.")
+            logger.warning(
+                "sentence-transformers not installed. Vector search disabled."
+            )
             self._model = None
-    
+
     def _load_index(self) -> None:
         """Load vector index from disk."""
         if self.index_path.exists():
             try:
-                with open(self.index_path, 'rb') as f:
+                with open(self.index_path, "rb") as f:
                     data = pickle.load(f)
-                    self._papers = data.get('papers', [])
-                    self._embeddings = data.get('embeddings')
+                    self._papers = data.get("papers", [])
+                    self._embeddings = data.get("embeddings")
             except Exception as e:
                 logger.warning(f"Failed to load vector index: {e}")
-    
+
     def _save_index(self) -> None:
         """Save vector index to disk."""
         self.index_path.parent.mkdir(parents=True, exist_ok=True)
-        data = {
-            'papers': self._papers,
-            'embeddings': self._embeddings
-        }
-        with open(self.index_path, 'wb') as f:
+        data = {"papers": self._papers, "embeddings": self._embeddings}
+        with open(self.index_path, "wb") as f:
             pickle.dump(data, f)
 
-# EOF
+
+# EOF

scitex/scholar/.legacy/_tmp/search_engine/web/_ArxivSearchEngine.py

@@ -27,10 +27,10 @@ logger = logging.getLogger(__name__)
 
 class ArxivSearchEngine(BaseSearchEngine):
     """arXiv search engine using the arXiv API."""
-    
+
     def __init__(self, config: Optional[ScholarConfig] = None):
         """Initialize arXiv search engine.
-        
+
         Parameters
         ----------
         config : ScholarConfig, optional
@@ -39,10 +39,10 @@ class ArxivSearchEngine(BaseSearchEngine):
         super().__init__(name="arxiv", rate_limit=0.5)
         self.config = config or ScholarConfig()
         self.base_url = "http://export.arxiv.org/api/query"
-    
+
     async def search_async(self, query: str, limit: int = 20, **kwargs) -> List[Paper]:
         """Search arXiv for papers.
-        
+
         Parameters
         ----------
         query : str
@@ -51,24 +51,24 @@ class ArxivSearchEngine(BaseSearchEngine):
             Maximum number of results
         **kwargs : dict
             Additional parameters (currently unused for arXiv)
-            
+
         Returns
         -------
         List[Paper]
             List of papers from arXiv
         """
         await self._rate_limit_async()
-        
+
         params = {
-            'search_query': f'all:{query}',
-            'start': 0,
-            'max_results': limit,
-            'sortBy': 'relevance',
-            'sortOrder': 'descending'
+            "search_query": f"all:{query}",
+            "start": 0,
+            "max_results": limit,
+            "sortBy": "relevance",
+            "sortOrder": "descending",
         }
-        
+
         papers = []
-        
+
         try:
             async with aiohttp.ClientSession() as session:
                 async with session.get(self.base_url, params=params) as response:
@@ -78,98 +78,102 @@ class ArxivSearchEngine(BaseSearchEngine):
                     else:
                         logger.error(f"arXiv search failed: {response.status}")
                         raise SearchError(query, "arxiv", f"HTTP {response.status}")
-                        
+
         except SearchError:
             raise
         except Exception as e:
             logger.error(f"arXiv search error: {e}")
             raise SearchError(query, "arxiv", str(e))
-        
+
         return papers
-    
+
     def _parse_xml(self, xml_data: str) -> List[Paper]:
         """Parse arXiv XML response into Paper objects.
-        
+
         Parameters
         ----------
         xml_data : str
             XML response from arXiv API
-            
+
         Returns
         -------
         List[Paper]
             Parsed Paper objects
         """
         papers = []
-        
+
         try:
             # Parse XML with namespace
             root = ET.fromstring(xml_data)
-            ns = {'atom': 'http://www.w3.org/2005/Atom'}
-            
-            for entry in root.findall('atom:entry', ns):
+            ns = {"atom": "http://www.w3.org/2005/Atom"}
+
+            for entry in root.findall("atom:entry", ns):
                 try:
                     # Extract title
-                    title_elem = entry.find('atom:title', ns)
-                    title = title_elem.text.strip() if title_elem is not None else ''
-                    
+                    title_elem = entry.find("atom:title", ns)
+                    title = title_elem.text.strip() if title_elem is not None else ""
+
                     # Extract authors
                     authors = []
-                    for author_elem in entry.findall('atom:author', ns):
-                        name_elem = author_elem.find('atom:name', ns)
+                    for author_elem in entry.findall("atom:author", ns):
+                        name_elem = author_elem.find("atom:name", ns)
                         if name_elem is not None and name_elem.text:
                             authors.append(name_elem.text)
-                    
+
                     # Extract abstract
-                    summary_elem = entry.find('atom:summary', ns)
-                    abstract = summary_elem.text.strip() if summary_elem is not None else ''
-                    
+                    summary_elem = entry.find("atom:summary", ns)
+                    abstract = (
+                        summary_elem.text.strip() if summary_elem is not None else ""
+                    )
+
                     # Extract year from published date
-                    published_elem = entry.find('atom:published', ns)
+                    published_elem = entry.find("atom:published", ns)
                     year = None
                     if published_elem is not None and published_elem.text:
                         year = published_elem.text[:4]
-                    
+
                     # Extract arXiv ID and create PDF URL
-                    id_elem = entry.find('atom:id', ns)
+                    id_elem = entry.find("atom:id", ns)
                     arxiv_id = None
                     pdf_url = None
                     if id_elem is not None and id_elem.text:
                         # Extract ID from URL (format: http://arxiv.org/abs/1234.5678)
-                        arxiv_id = id_elem.text.split('/')[-1]
+                        arxiv_id = id_elem.text.split("/")[-1]
                         pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
-                    
+
                     # Extract categories as keywords
                     keywords = []
-                    for cat_elem in entry.findall('atom:category', ns):
-                        term = cat_elem.get('term')
+                    for cat_elem in entry.findall("atom:category", ns):
+                        term = cat_elem.get("term")
                         if term:
                             keywords.append(term)
-                    
+
                     paper = Paper(
                         title=title,
                         authors=authors,
                         abstract=abstract,
-                        source='arxiv',
+                        source="arxiv",
                         year=year,
                         arxiv_id=arxiv_id,
                         keywords=keywords,
                         pdf_url=pdf_url,
                         metadata={
-                            'arxiv_categories': keywords,
-                            'arxiv_url': f"https://arxiv.org/abs/{arxiv_id}" if arxiv_id else None
-                        }
+                            "arxiv_categories": keywords,
+                            "arxiv_url": f"https://arxiv.org/abs/{arxiv_id}"
+                            if arxiv_id
+                            else None,
+                        },
                     )
-                    
+
                     papers.append(paper)
-                    
+
                 except Exception as e:
                     logger.warning(f"Failed to parse arXiv entry: {e}")
                     continue
-                    
+
         except Exception as e:
             logger.error(f"Failed to parse arXiv XML: {e}")
-        
+
         return papers
 
     async def fetch_by_id_async(self, identifier: str) -> Optional[Paper]:
@@ -177,14 +181,11 @@ class ArxivSearchEngine(BaseSearchEngine):
         try:
             # Rate limiting
             await self._rate_limit_async()
-            
+
             # arXiv API URL for fetching by ID
             url = f"{self.base_url}query"
-            params = {
-                'id_list': identifier,
-                'max_results': 1
-            }
-            
+            params = {"id_list": identifier, "max_results": 1}
+
             async with aiohttp.ClientSession() as session:
                 async with session.get(url, params=params) as response:
                     if response.status == 200:
@@ -192,9 +193,11 @@ class ArxivSearchEngine(BaseSearchEngine):
                         papers = self._parse_arxiv_response(xml_data)
                         return papers[0] if papers else None
                     else:
-                        logger.error(f"arXiv fetch_by_id_async failed: {response.status}")
+                        logger.error(
+                            f"arXiv fetch_by_id_async failed: {response.status}"
+                        )
                         return None
-                        
+
         except Exception as e:
             logger.error(f"Error fetching paper by ID {identifier}: {e}")
             return None
@@ -206,34 +209,39 @@ class ArxivSearchEngine(BaseSearchEngine):
         logger.debug("arXiv doesn't provide citation counts directly")
         return None
 
-    async def resolve_doi_async(self, title: str, year: Optional[int] = None) -> Optional[str]:
+    async def resolve_doi_async(
+        self, title: str, year: Optional[int] = None
+    ) -> Optional[str]:
         """Resolve title to DOI using arXiv search (note: many arXiv papers don't have DOIs)."""
         try:
             # Search for the paper by title
             papers = await self.search_async(title, limit=5)
-            
+
             # Look for exact or close title matches
             title_lower = title.lower().strip()
             for paper in papers:
                 if paper.doi and paper.title:
                     paper_title_lower = paper.title.lower().strip()
                     # Simple matching - could be improved with fuzzy matching
-                    if title_lower in paper_title_lower or paper_title_lower in title_lower:
+                    if (
+                        title_lower in paper_title_lower
+                        or paper_title_lower in title_lower
+                    ):
                         # If year is specified, prefer papers from that year
                         if year and paper.year and abs(paper.year - year) <= 1:
                             return paper.doi
                         elif not year:
                             return paper.doi
-            
+
             # If no exact match but we have results with DOI, return the first DOI
             for paper in papers:
                 if paper.doi:
                     return paper.doi
-            
+
             # Note: Many arXiv papers don't have DOIs
             logger.debug(f"No DOI found for arXiv paper: {title}")
             return None
-            
+
         except Exception as e:
             logger.error(f"Error resolving DOI for title '{title}': {e}")
             return None
@@ -242,12 +250,12 @@ class ArxivSearchEngine(BaseSearchEngine):
 async def main():
     """Test function for ArxivSearchEngine."""
     from scitex.scholar.config import ScholarConfig
-    
+
     config = ScholarConfig()
     engine = ArxivSearchEngine(config=config)
-    
+
     print("Testing arXiv search engine...")
-    
+
     try:
         papers = await engine.search_async("quantum computing", limit=5)
         print(f"Found {len(papers)} papers:")
@@ -264,7 +272,8 @@ async def main():
 
 if __name__ == "__main__":
     import asyncio
+
     asyncio.run(main())
 
 
-# EOF
+# EOF