scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/.legacy/search/_Embedder.py

@@ -1,329 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-01 04:35:00 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/search/_Embedder.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-"""Text embedder for semantic search."""
-
-import numpy as np
-from typing import List, Union, Optional, Dict, Any
-from pathlib import Path
-import json
-import hashlib
-
-from scitex import logging
-
-logger = logging.getLogger(__name__)
-
-# Optional imports for embedding models
-try:
-    from sentence_transformers import SentenceTransformer
-
-    SENTENCE_TRANSFORMERS_AVAILABLE = True
-except ImportError:
-    SENTENCE_TRANSFORMERS_AVAILABLE = False
-    logger.debug("sentence-transformers not installed - using simple embeddings")
-
-try:
-    import tiktoken
-
-    TIKTOKEN_AVAILABLE = True
-except ImportError:
-    TIKTOKEN_AVAILABLE = False
-
-
-class Embedder:
-    """Creates embeddings for text using various models.
-
-    Supports:
-    - Sentence transformers (recommended)
-    - Simple TF-IDF embeddings (fallback)
-    - OpenAI embeddings (if API key available)
-    """
-
-    def __init__(
-        self,
-        model_name: str = "all-MiniLM-L6-v2",
-        cache_dir: Optional[Path] = None,
-        use_gpu: bool = False,
-    ):
-        """Initialize embedder.
-
-        Args:
-            model_name: Model to use for embeddings
-            cache_dir: Directory to cache embeddings
-            use_gpu: Use GPU if available
-        """
-        self.model_name = model_name
-        self.use_gpu = use_gpu
-
-        # Setup cache
-        if cache_dir is None:
-            cache_dir = Path.home() / ".scitex" / "scholar" / "embeddings_cache"
-        self.cache_dir = Path(cache_dir)
-        self.cache_dir.mkdir(parents=True, exist_ok=True)
-
-        # Initialize model
-        self.model = None
-        self.embedding_dim = None
-        self._initialize_model()
-
-    def _initialize_model(self):
-        """Initialize the embedding model."""
-        if SENTENCE_TRANSFORMERS_AVAILABLE and self.model_name != "tfidf":
-            try:
-                device = "cuda" if self.use_gpu else "cpu"
-                self.model = SentenceTransformer(self.model_name, device=device)
-                self.embedding_dim = self.model.get_sentence_embedding_dimension()
-                logger.info(f"Loaded {self.model_name} on {device}")
-            except Exception as e:
-                logger.warning(f"Could not load {self.model_name}: {e}")
-                self._use_simple_embeddings()
-        else:
-            self._use_simple_embeddings()
-
-    def _use_simple_embeddings(self):
-        """Fall back to simple embeddings."""
-        logger.info("Using simple TF-IDF embeddings")
-        self.model_name = "tfidf"
-        self.embedding_dim = 768  # Fixed dimension for simple embeddings
-
-        # Load vocabulary if exists
-        vocab_file = self.cache_dir / "tfidf_vocab.json"
-        if vocab_file.exists():
-            with open(vocab_file) as f:
-                self.vocab = json.load(f)
-        else:
-            self.vocab = {}
-
-    def embed_text(
-        self, text: Union[str, List[str]], use_cache: bool = True
-    ) -> np.ndarray:
-        """Create embeddings for text.
-
-        Args:
-            text: Text or list of texts to embed
-            use_cache: Whether to use cached embeddings
-
-        Returns:
-            Embeddings array (n_texts, embedding_dim)
-        """
-        # Handle single text
-        if isinstance(text, str):
-            texts = [text]
-            single = True
-        else:
-            texts = text
-            single = False
-
-        # Check cache
-        if use_cache:
-            embeddings = []
-            uncached_texts = []
-            uncached_indices = []
-
-            for i, t in enumerate(texts):
-                cached_emb = self._get_cached_embedding(t)
-                if cached_emb is not None:
-                    embeddings.append(cached_emb)
-                else:
-                    uncached_texts.append(t)
-                    uncached_indices.append(i)
-
-            # Compute uncached embeddings
-            if uncached_texts:
-                new_embeddings = self._compute_embeddings(uncached_texts)
-
-                # Cache and merge
-                for t, emb in zip(uncached_texts, new_embeddings):
-                    self._cache_embedding(t, emb)
-
-                # Merge in correct order
-                all_embeddings = np.zeros((len(texts), self.embedding_dim))
-                cached_idx = 0
-                uncached_idx = 0
-
-                for i in range(len(texts)):
-                    if i in uncached_indices:
-                        all_embeddings[i] = new_embeddings[uncached_idx]
-                        uncached_idx += 1
-                    else:
-                        all_embeddings[i] = embeddings[cached_idx]
-                        cached_idx += 1
-
-                embeddings = all_embeddings
-            else:
-                embeddings = np.array(embeddings)
-        else:
-            # Compute all embeddings
-            embeddings = self._compute_embeddings(texts)
-
-        return embeddings[0] if single else embeddings
-
-    def _compute_embeddings(self, texts: List[str]) -> np.ndarray:
-        """Compute embeddings for texts."""
-        if self.model_name == "tfidf":
-            return self._compute_tfidf_embeddings(texts)
-        elif SENTENCE_TRANSFORMERS_AVAILABLE and self.model is not None:
-            return self.model.encode(
-                texts, convert_to_numpy=True, show_async_progress_bar=False
-            )
-        else:
-            # Fallback
-            return self._compute_tfidf_embeddings(texts)
-
-    def _compute_tfidf_embeddings(self, texts: List[str]) -> np.ndarray:
-        """Compute simple TF-IDF based embeddings."""
-        from sklearn.feature_extraction.text import TfidfVectorizer
-
-        # Update vocabulary
-        vectorizer = TfidfVectorizer(
-            max_features=self.embedding_dim,
-            vocabulary=self.vocab if self.vocab else None,
-        )
-
-        # Fit and transform
-        tfidf_matrix = vectorizer.fit_transform(texts)
-
-        # Update vocabulary
-        if not self.vocab:
-            self.vocab = vectorizer.vocabulary_
-            vocab_file = self.cache_dir / "tfidf_vocab.json"
-            with open(vocab_file, "w") as f:
-                json.dump(self.vocab, f)
-
-        # Convert to dense array
-        embeddings = tfidf_matrix.toarray()
-
-        # Normalize
-        norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
-        embeddings = embeddings / (norms + 1e-8)
-
-        return embeddings
-
-    def embed_paper(
-        self, paper: "DatabaseEntry", fields: List[str] = None
-    ) -> np.ndarray:
-        """Create embedding for a paper.
-
-        Args:
-            paper: Paper entry to embed
-            fields: Fields to include in embedding
-
-        Returns:
-            Embedding vector
-        """
-        if fields is None:
-            fields = ["title", "abstract", "keywords"]
-
-        # Combine text from specified fields
-        text_parts = []
-
-        if "title" in fields and paper.title:
-            text_parts.append(f"Title: {paper.title}")
-
-        if "abstract" in fields and paper.abstract:
-            text_parts.append(f"Abstract: {paper.abstract}")
-
-        if "keywords" in fields and paper.keywords:
-            text_parts.append(f"Keywords: {', '.join(paper.keywords)}")
-
-        if "authors" in fields and paper.authors:
-            text_parts.append(f"Authors: {', '.join(paper.authors)}")
-
-        # Combine and embed
-        combined_text = "\n".join(text_parts)
-        return self.embed_text(combined_text)
-
-    def compute_similarity(
-        self, embeddings1: np.ndarray, embeddings2: np.ndarray, metric: str = "cosine"
-    ) -> np.ndarray:
-        """Compute similarity between embeddings.
-
-        Args:
-            embeddings1: First set of embeddings (n1, dim)
-            embeddings2: Second set of embeddings (n2, dim)
-            metric: Similarity metric (cosine, dot, euclidean)
-
-        Returns:
-            Similarity matrix (n1, n2)
-        """
-        # Ensure 2D arrays
-        if embeddings1.ndim == 1:
-            embeddings1 = embeddings1.reshape(1, -1)
-        if embeddings2.ndim == 1:
-            embeddings2 = embeddings2.reshape(1, -1)
-
-        if metric == "cosine":
-            # Normalize
-            embeddings1_norm = embeddings1 / (
-                np.linalg.norm(embeddings1, axis=1, keepdims=True) + 1e-8
-            )
-            embeddings2_norm = embeddings2 / (
-                np.linalg.norm(embeddings2, axis=1, keepdims=True) + 1e-8
-            )
-            # Compute cosine similarity
-            similarities = np.dot(embeddings1_norm, embeddings2_norm.T)
-
-        elif metric == "dot":
-            similarities = np.dot(embeddings1, embeddings2.T)
-
-        elif metric == "euclidean":
-            # Convert distance to similarity
-            distances = np.linalg.norm(
-                embeddings1[:, np.newaxis] - embeddings2[np.newaxis, :], axis=2
-            )
-            similarities = 1 / (1 + distances)
-
-        else:
-            raise ValueError(f"Unknown metric: {metric}")
-
-        return similarities
-
-    def _get_cache_key(self, text: str) -> str:
-        """Generate cache key for text."""
-        text_hash = hashlib.md5(text.encode()).hexdigest()
-        return f"{self.model_name}_{text_hash}"
-
-    def _get_cached_embedding(self, text: str) -> Optional[np.ndarray]:
-        """Get cached embedding if exists."""
-        cache_key = self._get_cache_key(text)
-        cache_file = self.cache_dir / f"{cache_key}.npy"
-
-        if cache_file.exists():
-            try:
-                return np.load(cache_file)
-            except Exception as e:
-                logger.debug(f"Could not load cached embedding: {e}")
-
-        return None
-
-    def _cache_embedding(self, text: str, embedding: np.ndarray):
-        """Cache embedding to disk."""
-        cache_key = self._get_cache_key(text)
-        cache_file = self.cache_dir / f"{cache_key}.npy"
-
-        try:
-            np.save(cache_file, embedding)
-        except Exception as e:
-            logger.debug(f"Could not cache embedding: {e}")
-
-    def get_model_info(self) -> Dict[str, Any]:
-        """Get information about the current model."""
-        return {
-            "model_name": self.model_name,
-            "embedding_dim": self.embedding_dim,
-            "cache_dir": str(self.cache_dir),
-            "sentence_transformers": SENTENCE_TRANSFORMERS_AVAILABLE,
-            "device": "cuda" if self.use_gpu else "cpu",
-        }
-
-
-# EOF

scitex/scholar/.legacy/search/_SemanticSearch.py

@@ -1,278 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-01 10:25:00 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/search/_SemanticSearch.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-"""Semantic vector search for research papers."""
-
-import numpy as np
-from pathlib import Path
-from typing import List, Dict, Any, Optional, Tuple, Union
-import json
-import pickle
-from sentence_transformers import SentenceTransformer
-from sklearn.metrics.pairwise import cosine_similarity
-import faiss
-from collections import defaultdict
-
-from scitex import logging
-from scitex.errors import ScholarError
-
-logger = logging.getLogger(__name__)
-
-
-class SemanticSearch:
-    """Semantic search for research papers using embeddings.
-
-    Features:
-    - Fast vector similarity search using FAISS
-    - Multiple embedding models support
-    - Query expansion and refinement
-    - Hybrid search with keyword filtering
-    """
-
-    def __init__(
-        self,
-        model_name: str = "allenai/specter2",
-        index_dir: Optional[Union[str, Path]] = None,
-    ):
-        """Initialize semantic search.
-
-        Args:
-            model_name: Sentence transformer model name
-            index_dir: Directory to store/load indices
-        """
-        self.model_name = model_name
-        self.model = None
-        self.index = None
-        self.entry_ids = []
-        self.metadata = {}
-
-        if index_dir is None:
-            index_dir = Path.home() / ".scitex" / "scholar" / "semantic_index"
-        self.index_dir = Path(index_dir)
-        self.index_dir.mkdir(parents=True, exist_ok=True)
-
-        # Index files
-        self.index_file = self.index_dir / f"{model_name.replace('/', '_')}.index"
-        self.metadata_file = self.index_dir / "metadata.json"
-        self.ids_file = self.index_dir / "entry_ids.pkl"
-
-        # Load existing index
-        self._load_index()
-
-    def _load_model(self):
-        """Load sentence transformer model."""
-        if self.model is None:
-            logger.info(f"Loading model: {self.model_name}")
-            self.model = SentenceTransformer(self.model_name)
-
-    def build_index(self, entries: List[Dict[str, Any]], rebuild: bool = False) -> None:
-        """Build semantic index from database entries.
-
-        Args:
-            entries: List of database entries with text fields
-            rebuild: Force rebuild even if index exists
-        """
-        if not rebuild and self.index is not None:
-            logger.info("Index already exists, skipping build")
-            return
-
-        self._load_model()
-
-        # Prepare texts for embedding
-        texts = []
-        valid_entries = []
-
-        for entry in entries:
-            # Combine title and abstract for embedding
-            text_parts = []
-
-            if entry.get("title"):
-                text_parts.append(entry["title"])
-
-            if entry.get("abstract"):
-                text_parts.append(entry["abstract"])
-            elif entry.get("title"):
-                # Use title only if no abstract
-                text_parts.append(entry["title"])
-
-            if text_parts:
-                text = " ".join(text_parts)
-                texts.append(text)
-                valid_entries.append(entry)
-
-        if not texts:
-            raise ScholarError("No valid texts found for indexing")
-
-        logger.info(f"Encoding {len(texts)} papers...")
-        embeddings = self.model.encode(texts, show_async_progress_bar=True)
-
-        # Build FAISS index
-        dimension = embeddings.shape[1]
-        self.index = faiss.IndexFlatL2(dimension)  # L2 distance
-        self.index.add(embeddings.astype("float32"))
-
-        # Store metadata
-        self.entry_ids = [e.get("id", i) for i, e in enumerate(valid_entries)]
-        self.metadata = {
-            "model": self.model_name,
-            "dimension": dimension,
-            "total_entries": len(valid_entries),
-            "has_abstracts": sum(1 for e in valid_entries if e.get("abstract")),
-        }
-
-        # Save index
-        self._save_index()
-
-        logger.info(f"Built index with {len(valid_entries)} papers")
-
-    def search(
-        self, query: str, k: int = 10, filters: Optional[Dict[str, Any]] = None
-    ) -> List[Tuple[str, float]]:
-        """Search for similar papers.
-
-        Args:
-            query: Search query (can be title, abstract, or keywords)
-            k: Number of results to return
-            filters: Optional filters (year, journal, etc.)
-
-        Returns:
-            List of (entry_id, similarity_score) tuples
-        """
-        if self.index is None:
-            raise ScholarError("No index loaded. Build index first.")
-
-        self._load_model()
-
-        # Encode query
-        query_embedding = self.model.encode([query])
-
-        # Search index
-        distances, indices = self.index.search(
-            query_embedding.astype("float32"),
-            min(k * 2, self.index.ntotal),  # Get more for filtering
-        )
-
-        # Convert to similarity scores (1 - normalized distance)
-        max_distance = distances[0].max() if distances[0].max() > 0 else 1.0
-        similarities = 1 - (distances[0] / max_distance)
-
-        # Collect results
-        results = []
-        for idx, sim in zip(indices[0], similarities):
-            if idx < len(self.entry_ids):
-                entry_id = self.entry_ids[idx]
-                results.append((entry_id, float(sim)))
-
-        # Apply filters if provided
-        if filters:
-            results = self._apply_filters(results, filters)
-
-        # Return top k
-        return results[:k]
-
-    def find_similar(self, entry_id: str, k: int = 10) -> List[Tuple[str, float]]:
-        """Find papers similar to a given paper.
-
-        Args:
-            entry_id: ID of the reference paper
-            k: Number of similar papers to return
-
-        Returns:
-            List of (entry_id, similarity_score) tuples
-        """
-        if entry_id not in self.entry_ids:
-            raise ScholarError(f"Entry {entry_id} not in index")
-
-        # Get embedding of reference paper
-        idx = self.entry_ids.index(entry_id)
-
-        # Get the embedding vector
-        embedding = self.index.reconstruct(idx).reshape(1, -1)
-
-        # Search for similar
-        distances, indices = self.index.search(
-            embedding.astype("float32"),
-            k + 1,  # +1 because it will include itself
-        )
-
-        # Convert to similarity scores
-        max_distance = distances[0].max() if distances[0].max() > 0 else 1.0
-        similarities = 1 - (distances[0] / max_distance)
-
-        # Collect results (skip first which is the paper itself)
-        results = []
-        for idx, sim in zip(indices[0][1:], similarities[1:]):
-            if idx < len(self.entry_ids):
-                similar_id = self.entry_ids[idx]
-                results.append((similar_id, float(sim)))
-
-        return results
-
-    def _apply_filters(
-        self, results: List[Tuple[str, float]], filters: Dict[str, Any]
-    ) -> List[Tuple[str, float]]:
-        """Apply filters to search results.
-
-        Note: This is a placeholder. Actual filtering would need
-        access to the database to check entry properties.
-        """
-        # TODO: Implement filtering based on database queries
-        return results
-
-    def _save_index(self):
-        """Save index and metadata to disk."""
-        if self.index is None:
-            return
-
-        logger.info(f"Saving index to {self.index_dir}")
-
-        # Save FAISS index
-        faiss.write_index(self.index, str(self.index_file))
-
-        # Save metadata
-        with open(self.metadata_file, "w") as f:
-            json.dump(self.metadata, f, indent=2)
-
-        # Save entry IDs
-        with open(self.ids_file, "wb") as f:
-            pickle.dump(self.entry_ids, f)
-
-    def _load_index(self):
-        """Load index from disk if exists."""
-        if not all(
-            f.exists() for f in [self.index_file, self.metadata_file, self.ids_file]
-        ):
-            logger.info("No existing index found")
-            return
-
-        try:
-            # Load FAISS index
-            self.index = faiss.read_index(str(self.index_file))
-
-            # Load metadata
-            with open(self.metadata_file, "r") as f:
-                self.metadata = json.load(f)
-
-            # Load entry IDs
-            with open(self.ids_file, "rb") as f:
-                self.entry_ids = pickle.load(f)
-
-            logger.info(f"Loaded index with {len(self.entry_ids)} entries")
-
-        except Exception as e:
-            logger.warning(f"Failed to load index: {e}")
-            self.index = None
-            self.entry_ids = []
-            self.metadata = {}
-
-
-# EOF