scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/.legacy/metadata/urls/_ZoteroTranslatorRunner.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -33,9 +34,7 @@ class ZoteroTranslatorRunner:
 
     def __init__(self, translator_dir: Optional[Path] = None):
         """Initialize with translator directory."""
-        self.translator_dir = translator_dir or (
-            Path(__DIR__) / "zotero_translators"
-        )
+        self.translator_dir = translator_dir or (Path(__DIR__) / "zotero_translators")
         self._translators = self._load_translators()
 
     def _load_translators(self) -> Dict[str, Dict]:
@@ -43,9 +42,7 @@ class ZoteroTranslatorRunner:
         translators = {}
 
         if not self.translator_dir.exists():
-            logger.warning(
-                f"Translator directory not found: {self.translator_dir}"
-            )
+            logger.warning(f"Translator directory not found: {self.translator_dir}")
             return translators
 
         for js_file in self.translator_dir.glob("*.js"):
@@ -58,34 +55,32 @@ class ZoteroTranslatorRunner:
 
                 # Extract metadata JSON - it's always at the beginning and ends with }\n
                 # Find the first line that's just "}" after the opening "{"
-                lines = content.split('\n')
+                lines = content.split("\n")
                 json_end_idx = -1
                 brace_count = 0
-                
+
                 for i, line in enumerate(lines):
-                    if line.strip() == '{':
+                    if line.strip() == "{":
                         brace_count = 1
                     elif brace_count > 0:
-                        brace_count += line.count('{') - line.count('}')
+                        brace_count += line.count("{") - line.count("}")
                         if brace_count == 0:
                             json_end_idx = i
                             break
-                
+
                 if json_end_idx == -1:
                     continue
-                
+
                 # Extract and parse metadata
-                metadata_str = '\n'.join(lines[:json_end_idx + 1])
+                metadata_str = "\n".join(lines[: json_end_idx + 1])
                 # Remove trailing commas before closing braces
-                metadata_str = re.sub(r',(\s*})', r'\1', metadata_str)
+                metadata_str = re.sub(r",(\s*})", r"\1", metadata_str)
                 metadata = json.loads(metadata_str)
 
                 # Only keep web translators
-                if metadata.get("translatorType", 0) & 4 and metadata.get(
-                    "target"
-                ):
+                if metadata.get("translatorType", 0) & 4 and metadata.get("target"):
                     # Extract JavaScript code (after metadata)
-                    js_code = '\n'.join(lines[json_end_idx + 1:]).lstrip()
+                    js_code = "\n".join(lines[json_end_idx + 1 :]).lstrip()
 
                     # Remove test cases section
                     test_idx = js_code.find("/** BEGIN TEST CASES **/")
@@ -109,9 +104,7 @@ class ZoteroTranslatorRunner:
         for name, translator in self._translators.items():
             try:
                 if re.match(translator["target_regex"], url):
-                    logger.debug(
-                        f"URL matches translator: {translator['label']}"
-                    )
+                    logger.debug(f"URL matches translator: {translator['label']}")
                     return translator
             except:
                 continue
@@ -297,4 +290,5 @@ async def find_pdf_urls_with_translator(page: Page) -> List[str]:
     runner = ZoteroTranslatorRunner()
     return await runner.extract_pdf_urls_async(page)
 
+
 # EOF

scitex/scholar/.legacy/metadata/urls/__init__.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -23,17 +24,16 @@ from ._handler import URLHandler
 from ._URLMetadataHandler import (
     URLMetadataHandler,
     add_urls_to_paper,
-    get_papers_for_download
+    get_papers_for_download,
 )
 
 __all__ = [
     # Main abstracted interface
     "URLHandler",
-
     # Backward compatibility
     "URLMetadataHandler",
     "add_urls_to_paper",
-    "get_papers_for_download"
+    "get_papers_for_download",
 ]
 
 # EOF

scitex/scholar/.legacy/metadata/urls/_finder.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -237,9 +238,7 @@ def _get_publisher_pdf_patterns(url: str) -> List[str]:
         pdf_urls.append(url.replace("/full", "/pdf"))
 
     # Springer
-    elif (
-        "springer.com" in url or "link.springer.com" in url
-    ) and "/article/" in url:
+    elif ("springer.com" in url or "link.springer.com" in url) and "/article/" in url:
         if not url.endswith(".pdf"):
             pdf_urls.append(url.rstrip("/") + ".pdf")
 
@@ -263,9 +262,7 @@ def _get_publisher_pdf_patterns(url: str) -> List[str]:
             f"Publisher-specific pattern matching found {len(pdf_urls)} PDF URLs"
         )
     else:
-        logger.warning(
-            f"Publisher-specific patterns did not match any PDF URLs"
-        )
+        logger.warning(f"Publisher-specific patterns did not match any PDF URLs")
 
     return pdf_urls
 
@@ -318,4 +315,5 @@ async def find_all_urls(page: Page) -> Dict[str, List[Dict]]:
         "supplementary": await find_supplementary_urls(page),
     }
 
+
 # EOF

scitex/scholar/.legacy/metadata/urls/_handler.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -99,9 +100,7 @@ class URLHandler:
 
             if self.context:
                 try:
-                    url_publisher = await doi_to_url_publisher(
-                        doi, self.context
-                    )
+                    url_publisher = await doi_to_url_publisher(doi, self.context)
                     if url_publisher:
                         urls["url_publisher"] = url_publisher
 
@@ -118,9 +117,7 @@ class URLHandler:
                             if pdf_urls:
                                 urls["url_pdf"] = pdf_urls
                         except Exception as e:
-                            logger.warning(
-                                f"Failed to navigate to publisher URL: {e}"
-                            )
+                            logger.warning(f"Failed to navigate to publisher URL: {e}")
                             # Try with networkidle instead
                             try:
                                 await page.goto(
@@ -132,9 +129,7 @@ class URLHandler:
                                 if pdf_urls:
                                     urls["url_pdf"] = pdf_urls
                             except Exception as e2:
-                                logger.error(
-                                    f"Could not access publisher page: {e2}"
-                                )
+                                logger.error(f"Could not access publisher page: {e2}")
 
                         # Extract all URLs from page before closing
                         if page:
@@ -147,9 +142,7 @@ class URLHandler:
                                         "supplementary"
                                     ]
                             except Exception as e:
-                                logger.error(
-                                    f"Error extracting URLs from page: {e}"
-                                )
+                                logger.error(f"Error extracting URLs from page: {e}")
 
                         # Now close the page
                         try:
@@ -185,9 +178,7 @@ class URLHandler:
                 )
                 return await find_pdf_urls(page)
             except Exception as e:
-                logger.warning(
-                    f"Failed with domcontentloaded, trying networkidle: {e}"
-                )
+                logger.warning(f"Failed with domcontentloaded, trying networkidle: {e}")
                 try:
                     await page.goto(
                         page_or_url, wait_until="networkidle", timeout=30000
@@ -331,4 +322,5 @@ def update_metadata_urls(metadata_path: Path, urls: Dict) -> bool:
     handler = URLHandler()
     return handler.update_metadata(metadata_path, urls)
 
+
 # EOF

scitex/scholar/.legacy/metadata/urls/_resolver.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -51,9 +52,7 @@ async def doi_to_url_publisher(
         page = await context.new_page()
         try:
             logger.info(f"Resolving DOI: {doi}")
-            await page.goto(
-                url_doi, wait_until="domcontentloaded", timeout=30000
-            )
+            await page.goto(url_doi, wait_until="domcontentloaded", timeout=30000)
             await asyncio.sleep(2)  # Let redirects settle
 
             url_publisher = page.url
@@ -142,9 +141,7 @@ def generate_openurl_query(
                 if first_author.get("lastName"):
                     params.append(f"aulast={quote(first_author['lastName'])}")
                 if first_author.get("firstName"):
-                    params.append(
-                        f"aufirst={quote(first_author['firstName'])}"
-                    )
+                    params.append(f"aufirst={quote(first_author['firstName'])}")
 
     # Add ISSN if available
     if metadata.get("issn"):
@@ -156,9 +153,7 @@ def generate_openurl_query(
     return None
 
 
-async def resolve_openurl(
-    openurl_query: str, context: BrowserContext
-) -> Optional[str]:
+async def resolve_openurl(openurl_query: str, context: BrowserContext) -> Optional[str]:
     """
     Resolve OpenURL query to final authenticated URL.
 
@@ -271,9 +266,7 @@ async def resolve_all_urls(
 
         # Resolve to publisher
         if context:
-            url_publisher = await doi_to_url_publisher(
-                metadata["doi"], context
-            )
+            url_publisher = await doi_to_url_publisher(metadata["doi"], context)
             if url_publisher:
                 urls["url_publisher"] = url_publisher
 
@@ -290,4 +283,5 @@ async def resolve_all_urls(
 
     return urls
 
+
 # EOF

scitex/scholar/.legacy/search/_Embedder.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -24,6 +25,7 @@ logger = logging.getLogger(__name__)
 # Optional imports for embedding models
 try:
     from sentence_transformers import SentenceTransformer
+
     SENTENCE_TRANSFORMERS_AVAILABLE = True
 except ImportError:
     SENTENCE_TRANSFORMERS_AVAILABLE = False
@@ -31,6 +33,7 @@ except ImportError:
 
 try:
     import tiktoken
+
     TIKTOKEN_AVAILABLE = True
 except ImportError:
     TIKTOKEN_AVAILABLE = False
@@ -38,21 +41,21 @@ except ImportError:
 
 class Embedder:
     """Creates embeddings for text using various models.
-    
+
     Supports:
     - Sentence transformers (recommended)
     - Simple TF-IDF embeddings (fallback)
     - OpenAI embeddings (if API key available)
     """
-    
+
     def __init__(
         self,
         model_name: str = "all-MiniLM-L6-v2",
         cache_dir: Optional[Path] = None,
-        use_gpu: bool = False
+        use_gpu: bool = False,
     ):
         """Initialize embedder.
-        
+
         Args:
             model_name: Model to use for embeddings
             cache_dir: Directory to cache embeddings
@@ -60,18 +63,18 @@ class Embedder:
         """
         self.model_name = model_name
         self.use_gpu = use_gpu
-        
+
         # Setup cache
         if cache_dir is None:
             cache_dir = Path.home() / ".scitex" / "scholar" / "embeddings_cache"
         self.cache_dir = Path(cache_dir)
         self.cache_dir.mkdir(parents=True, exist_ok=True)
-        
+
         # Initialize model
         self.model = None
         self.embedding_dim = None
         self._initialize_model()
-    
+
     def _initialize_model(self):
         """Initialize the embedding model."""
         if SENTENCE_TRANSFORMERS_AVAILABLE and self.model_name != "tfidf":
@@ -85,13 +88,13 @@ class Embedder:
                 self._use_simple_embeddings()
         else:
             self._use_simple_embeddings()
-    
+
     def _use_simple_embeddings(self):
         """Fall back to simple embeddings."""
         logger.info("Using simple TF-IDF embeddings")
         self.model_name = "tfidf"
         self.embedding_dim = 768  # Fixed dimension for simple embeddings
-        
+
         # Load vocabulary if exists
         vocab_file = self.cache_dir / "tfidf_vocab.json"
         if vocab_file.exists():
@@ -99,18 +102,16 @@ class Embedder:
                 self.vocab = json.load(f)
         else:
             self.vocab = {}
-    
+
     def embed_text(
-        self,
-        text: Union[str, List[str]],
-        use_cache: bool = True
+        self, text: Union[str, List[str]], use_cache: bool = True
     ) -> np.ndarray:
         """Create embeddings for text.
-        
+
         Args:
             text: Text or list of texts to embed
             use_cache: Whether to use cached embeddings
-            
+
         Returns:
             Embeddings array (n_texts, embedding_dim)
         """
@@ -121,13 +122,13 @@ class Embedder:
         else:
             texts = text
             single = False
-        
+
         # Check cache
         if use_cache:
             embeddings = []
             uncached_texts = []
             uncached_indices = []
-            
+
             for i, t in enumerate(texts):
                 cached_emb = self._get_cached_embedding(t)
                 if cached_emb is not None:
@@ -135,20 +136,20 @@ class Embedder:
                 else:
                     uncached_texts.append(t)
                     uncached_indices.append(i)
-            
+
             # Compute uncached embeddings
             if uncached_texts:
                 new_embeddings = self._compute_embeddings(uncached_texts)
-                
+
                 # Cache and merge
                 for t, emb in zip(uncached_texts, new_embeddings):
                     self._cache_embedding(t, emb)
-                
+
                 # Merge in correct order
                 all_embeddings = np.zeros((len(texts), self.embedding_dim))
                 cached_idx = 0
                 uncached_idx = 0
-                
+
                 for i in range(len(texts)):
                     if i in uncached_indices:
                         all_embeddings[i] = new_embeddings[uncached_idx]
@@ -156,100 +157,101 @@ class Embedder:
                     else:
                         all_embeddings[i] = embeddings[cached_idx]
                         cached_idx += 1
-                
+
                 embeddings = all_embeddings
             else:
                 embeddings = np.array(embeddings)
         else:
             # Compute all embeddings
             embeddings = self._compute_embeddings(texts)
-        
+
         return embeddings[0] if single else embeddings
-    
+
     def _compute_embeddings(self, texts: List[str]) -> np.ndarray:
         """Compute embeddings for texts."""
         if self.model_name == "tfidf":
             return self._compute_tfidf_embeddings(texts)
         elif SENTENCE_TRANSFORMERS_AVAILABLE and self.model is not None:
-            return self.model.encode(texts, convert_to_numpy=True, show_async_progress_bar=False)
+            return self.model.encode(
+                texts, convert_to_numpy=True, show_async_progress_bar=False
+            )
         else:
             # Fallback
             return self._compute_tfidf_embeddings(texts)
-    
+
     def _compute_tfidf_embeddings(self, texts: List[str]) -> np.ndarray:
         """Compute simple TF-IDF based embeddings."""
         from sklearn.feature_extraction.text import TfidfVectorizer
-        
+
         # Update vocabulary
         vectorizer = TfidfVectorizer(
             max_features=self.embedding_dim,
-            vocabulary=self.vocab if self.vocab else None
+            vocabulary=self.vocab if self.vocab else None,
         )
-        
+
         # Fit and transform
         tfidf_matrix = vectorizer.fit_transform(texts)
-        
+
         # Update vocabulary
         if not self.vocab:
             self.vocab = vectorizer.vocabulary_
             vocab_file = self.cache_dir / "tfidf_vocab.json"
-            with open(vocab_file, 'w') as f:
+            with open(vocab_file, "w") as f:
                 json.dump(self.vocab, f)
-        
+
         # Convert to dense array
         embeddings = tfidf_matrix.toarray()
-        
+
         # Normalize
         norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
         embeddings = embeddings / (norms + 1e-8)
-        
+
         return embeddings
-    
-    def embed_paper(self, paper: 'DatabaseEntry', fields: List[str] = None) -> np.ndarray:
+
+    def embed_paper(
+        self, paper: "DatabaseEntry", fields: List[str] = None
+    ) -> np.ndarray:
         """Create embedding for a paper.
-        
+
         Args:
             paper: Paper entry to embed
             fields: Fields to include in embedding
-            
+
         Returns:
             Embedding vector
         """
         if fields is None:
             fields = ["title", "abstract", "keywords"]
-        
+
         # Combine text from specified fields
         text_parts = []
-        
+
         if "title" in fields and paper.title:
             text_parts.append(f"Title: {paper.title}")
-        
+
         if "abstract" in fields and paper.abstract:
             text_parts.append(f"Abstract: {paper.abstract}")
-        
+
         if "keywords" in fields and paper.keywords:
             text_parts.append(f"Keywords: {', '.join(paper.keywords)}")
-        
+
         if "authors" in fields and paper.authors:
             text_parts.append(f"Authors: {', '.join(paper.authors)}")
-        
+
         # Combine and embed
         combined_text = "\n".join(text_parts)
         return self.embed_text(combined_text)
-    
+
     def compute_similarity(
-        self,
-        embeddings1: np.ndarray,
-        embeddings2: np.ndarray,
-        metric: str = "cosine"
+        self, embeddings1: np.ndarray, embeddings2: np.ndarray, metric: str = "cosine"
     ) -> np.ndarray:
         """Compute similarity between embeddings.
-        
+
         Args:
             embeddings1: First set of embeddings (n1, dim)
             embeddings2: Second set of embeddings (n2, dim)
             metric: Similarity metric (cosine, dot, euclidean)
-            
+
         Returns:
             Similarity matrix (n1, n2)
         """
@@ -258,58 +260,61 @@ class Embedder:
             embeddings1 = embeddings1.reshape(1, -1)
         if embeddings2.ndim == 1:
             embeddings2 = embeddings2.reshape(1, -1)
-        
+
         if metric == "cosine":
             # Normalize
-            embeddings1_norm = embeddings1 / (np.linalg.norm(embeddings1, axis=1, keepdims=True) + 1e-8)
-            embeddings2_norm = embeddings2 / (np.linalg.norm(embeddings2, axis=1, keepdims=True) + 1e-8)
+            embeddings1_norm = embeddings1 / (
+                np.linalg.norm(embeddings1, axis=1, keepdims=True) + 1e-8
+            )
+            embeddings2_norm = embeddings2 / (
+                np.linalg.norm(embeddings2, axis=1, keepdims=True) + 1e-8
+            )
             # Compute cosine similarity
             similarities = np.dot(embeddings1_norm, embeddings2_norm.T)
-        
+
         elif metric == "dot":
             similarities = np.dot(embeddings1, embeddings2.T)
-        
+
         elif metric == "euclidean":
             # Convert distance to similarity
             distances = np.linalg.norm(
-                embeddings1[:, np.newaxis] - embeddings2[np.newaxis, :],
-                axis=2
+                embeddings1[:, np.newaxis] - embeddings2[np.newaxis, :], axis=2
             )
             similarities = 1 / (1 + distances)
-        
+
         else:
             raise ValueError(f"Unknown metric: {metric}")
-        
+
         return similarities
-    
+
     def _get_cache_key(self, text: str) -> str:
         """Generate cache key for text."""
         text_hash = hashlib.md5(text.encode()).hexdigest()
         return f"{self.model_name}_{text_hash}"
-    
+
     def _get_cached_embedding(self, text: str) -> Optional[np.ndarray]:
         """Get cached embedding if exists."""
         cache_key = self._get_cache_key(text)
         cache_file = self.cache_dir / f"{cache_key}.npy"
-        
+
         if cache_file.exists():
             try:
                 return np.load(cache_file)
             except Exception as e:
                 logger.debug(f"Could not load cached embedding: {e}")
-        
+
         return None
-    
+
     def _cache_embedding(self, text: str, embedding: np.ndarray):
         """Cache embedding to disk."""
         cache_key = self._get_cache_key(text)
         cache_file = self.cache_dir / f"{cache_key}.npy"
-        
+
         try:
             np.save(cache_file, embedding)
         except Exception as e:
             logger.debug(f"Could not cache embedding: {e}")
-    
+
     def get_model_info(self) -> Dict[str, Any]:
         """Get information about the current model."""
         return {
@@ -317,8 +322,8 @@ class Embedder:
             "embedding_dim": self.embedding_dim,
             "cache_dir": str(self.cache_dir),
             "sentence_transformers": SENTENCE_TRANSFORMERS_AVAILABLE,
-            "device": "cuda" if self.use_gpu else "cpu"
+            "device": "cuda" if self.use_gpu else "cpu",
         }
 
 
-# EOF
+# EOF