scitex 2.13.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/scholar/.legacy/_tmp/search_engine/_UnifiedSearcher.py

@@ -1,392 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-01 17:56:55 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/search_engine/_UnifiedSearcher.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-"""
-Unified search functionality for SciTeX Scholar.
-
-This module provides a unified interface to search across multiple sources
-including PubMed, Semantic Scholar, arXiv, and local collections.
-"""
-
-import asyncio
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
-
-from scitex import logging
-
-from scitex.errors import SearchError
-from scitex.scholar.core import Paper
-from ._BaseSearchEngine import BaseSearchEngine
-from .local._LocalSearchEngine import LocalSearchEngine
-from .local._VectorSearchEngine import VectorSearchEngine
-from .web._ArxivSearchEngine import ArxivSearchEngine
-from .web._CrossRefSearchEngine import CrossRefSearchEngine
-from .web._GoogleScholarSearchEngine import GoogleScholarSearchEngine
-from .web._PubMedSearchEngine import PubMedSearchEngine
-from .web._SemanticScholarSearchEngine import SemanticScholarSearchEngine
-
-logger = logging.getLogger(__name__)
-
-
-class UnifiedSearcher:
-    """
-    Unified searcher that combines results from multiple engines.
-
-    Attributes
-    ----------
-    email : str
-        Email for API authentication
-    semantic_scholar_api_key : str
-        API key for Semantic Scholar
-    crossref_api_key : str
-        API key for CrossRef
-    google_scholar_timeout : int
-        Timeout for Google Scholar searches
-    """
-
-    def __init__(
-        self,
-        config: Optional[Any] = None,
-        email: Optional[str] = None,
-        semantic_scholar_api_key: Optional[str] = None,
-        crossref_api_key: Optional[str] = None,
-        google_scholar_timeout: int = 10,
-    ):
-        """Initialize unified searcher with configuration.
-
-        Args:
-            config: ScholarConfig object (takes precedence over individual params)
-            email: Email for API authentication (fallback)
-            semantic_scholar_api_key: API key for Semantic Scholar (fallback)
-            crossref_api_key: API key for CrossRef (fallback)
-            google_scholar_timeout: Timeout for Google Scholar searches (fallback)
-        """
-        # Use config if provided, otherwise use individual params
-        if config:
-            self.email = getattr(config, "pubmed_email", email)
-            self.semantic_scholar_api_key = getattr(
-                config, "semantic_scholar_api_key", semantic_scholar_api_key
-            )
-            self.crossref_api_key = getattr(
-                config, "crossref_api_key", crossref_api_key
-            )
-            self.google_scholar_timeout = getattr(
-                config, "google_scholar_timeout", google_scholar_timeout
-            )
-        else:
-            self.email = email
-            self.semantic_scholar_api_key = semantic_scholar_api_key
-            self.crossref_api_key = crossref_api_key
-            self.google_scholar_timeout = google_scholar_timeout
-        self._engines = {}  # Lazy-loaded engines
-
-    @property
-    def engines(self):
-        """Get dictionary of available engines."""
-        return self._engines
-
-    def _get_engine(self, source: str) -> BaseSearchEngine:
-        """Get or create engine for a source.
-
-        Parameters
-        ----------
-        source : str
-            Name of the search source
-
-        Returns
-        -------
-        BaseSearchEngine
-            The search engine instance
-
-        Raises
-        ------
-        ValueError
-            If source is not recognized
-        """
-        if source not in self._engines:
-            if source == "semantic_scholar":
-                self._engines[source] = SemanticScholarSearchEngine(
-                    self.semantic_scholar_api_key
-                )
-            elif source == "pubmed":
-                self._engines[source] = PubMedSearchEngine(self.email)
-            elif source == "arxiv":
-                self._engines[source] = ArxivSearchEngine()
-            elif source == "google_scholar":
-                self._engines[source] = GoogleScholarSearchEngine(
-                    timeout=self.google_scholar_timeout
-                )
-            elif source == "crossref":
-                self._engines[source] = CrossRefSearchEngine(
-                    api_key=self.crossref_api_key, email=self.email
-                )
-            elif source == "local":
-                self._engines[source] = LocalSearchEngine()
-            elif source == "vector":
-                self._engines[source] = VectorSearchEngine()
-            else:
-                raise ValueError(f"Unknown source: {source}")
-        return self._engines[source]
-
-    async def search_async(
-        self,
-        query: str,
-        sources: List[str] = None,
-        limit: int = 20,
-        deduplicate: bool = True,
-        **kwargs,
-    ) -> List[Paper]:
-        """
-        Search across multiple sources and merge results.
-
-        Parameters
-        ----------
-        query : str
-            Search query
-        sources : List[str], optional
-            List of sources to search (default: ['pubmed'])
-        limit : int
-            Maximum results per source
-        deduplicate : bool
-            Remove duplicate papers
-        **kwargs : dict
-            Additional parameters for engines
-
-        Returns
-        -------
-        List[Paper]
-            Merged and ranked list of papers
-        """
-        if sources is None:
-            sources = ["pubmed"]  # Default to PubMed only
-
-        # Filter to valid sources
-        valid_sources = [
-            "pubmed",
-            "semantic_scholar",
-            "google_scholar",
-            "crossref",
-            "arxiv",
-            "local",
-            "vector",
-        ]
-        sources = [s for s in sources if s in valid_sources]
-
-        if not sources:
-            logger.warning("No valid search sources specified")
-            return []
-
-        # Search all sources concurrently
-        tasks = []
-        for source in sources:
-            try:
-                engine = self._get_engine(source)
-                task = engine.search_async(query, limit, **kwargs)
-                tasks.append(task)
-            except Exception as e:
-                logger.debug(f"Failed to initialize {source} engine: {e}")
-
-        logger.debug(f"Searching {len(tasks)} sources: {sources}")
-        results = await asyncio.gather(*tasks, return_exceptions=True)
-
-        # Merge results
-        all_papers = []
-        for source, result in zip(sources, results):
-            if isinstance(result, Exception):
-                logger.debug(f"Search failed for {source}: {result}")
-            else:
-                logger.debug(f"{source} returned {len(result)} papers")
-                all_papers.extend(result)
-
-        # Deduplicate if requested
-        if deduplicate:
-            all_papers = self._deduplicate_papers(all_papers)
-
-        # Sort by relevance (using citation count as proxy)
-        all_papers.sort(key=lambda p: p.citation_count or 0, reverse=True)
-
-        return all_papers[:limit]
-
-    def _deduplicate_papers(self, papers: List[Paper]) -> List[Paper]:
-        """Remove duplicate papers based on similarity.
-
-        Parameters
-        ----------
-        papers : List[Paper]
-            List of papers to deduplicate
-
-        Returns
-        -------
-        List[Paper]
-            Deduplicated list of papers
-        """
-        if not papers:
-            return []
-
-        unique_papers = [papers[0]]
-
-        for paper in papers[1:]:
-            is_duplicate = False
-
-            for unique_paper in unique_papers:
-                if paper.similarity_score(unique_paper) > 0.85:
-                    is_duplicate = True
-                    # Keep the one with more information
-                    if (paper.citation_count or 0) > (unique_paper.citation_count or 0):
-                        unique_papers.remove(unique_paper)
-                        unique_papers.append(paper)
-                    break
-
-            if not is_duplicate:
-                unique_papers.append(paper)
-
-        return unique_papers
-
-    def build_local_index(self, pdf_dirs: List[Union[str, Path]]) -> Dict[str, Any]:
-        """Build local search index.
-
-        Parameters
-        ----------
-        pdf_dirs : List[Union[str, Path]]
-            Directories containing PDFs
-
-        Returns
-        -------
-        Dict[str, Any]
-            Indexing statistics
-        """
-        pdf_dirs = [Path(d) for d in pdf_dirs]
-        local_engine = self._get_engine("local")
-        return local_engine.build_index(pdf_dirs)
-
-    def add_to_vector_index(self, papers: List[Paper]) -> None:
-        """Add papers to vector search index.
-
-        Parameters
-        ----------
-        papers : List[Paper]
-            Papers to add to the index
-        """
-        vector_engine = self._get_engine("vector")
-        vector_engine.add_papers(papers)
-
-
-# Convenience functions
-async def search_async(
-    query: str,
-    sources: List[str] = None,
-    limit: int = 20,
-    email: Optional[str] = None,
-    semantic_scholar_api_key: Optional[str] = None,
-    **kwargs,
-) -> List[Paper]:
-    """
-    Async convenience function for searching papers.
-
-    Parameters
-    ----------
-    query : str
-        Search query
-    sources : List[str], optional
-        Sources to search
-    limit : int
-        Maximum results
-    email : str, optional
-        Email for APIs
-    semantic_scholar_api_key : str, optional
-        API key for Semantic Scholar
-    **kwargs : dict
-        Additional search parameters
-
-    Returns
-    -------
-    List[Paper]
-        Search results
-    """
-    searcher = UnifiedSearcher(
-        email=email, semantic_scholar_api_key=semantic_scholar_api_key
-    )
-    return await searcher.search_async(query, sources, limit, **kwargs)
-
-
-def search_sync(
-    query: str,
-    sources: List[str] = None,
-    limit: int = 20,
-    email: Optional[str] = None,
-    semantic_scholar_api_key: Optional[str] = None,
-    **kwargs,
-) -> List[Paper]:
-    """
-    Synchronous convenience function for searching papers.
-
-    Parameters
-    ----------
-    query : str
-        Search query
-    sources : List[str], optional
-        Sources to search
-    limit : int
-        Maximum results
-    email : str, optional
-        Email for APIs
-    semantic_scholar_api_key : str, optional
-        API key for Semantic Scholar
-    **kwargs : dict
-        Additional search parameters
-
-    Returns
-    -------
-    List[Paper]
-        Search results
-    """
-    return asyncio.run(
-        search_async(query, sources, limit, email, semantic_scholar_api_key, **kwargs)
-    )
-
-
-def build_index(
-    paths: List[Union[str, Path]], vector_index: bool = True
-) -> Dict[str, Any]:
-    """
-    Build local search indices.
-
-    Parameters
-    ----------
-    paths : List[Union[str, Path]]
-        Directories containing PDFs
-    vector_index : bool
-        Also build vector similarity index
-
-    Returns
-    -------
-    Dict[str, Any]
-        Statistics about indexing
-    """
-    searcher = UnifiedSearcher()
-    stats = searcher.build_local_index(paths)
-
-    if vector_index:
-        # Add papers to vector index
-        local_engine = searcher._get_engine("local")
-        papers = local_engine._search_sync("*", 9999, {})
-        if papers:
-            searcher.add_to_vector_index(papers)
-            stats["vector_indexed"] = len(papers)
-
-    return stats
-
-
-# Export all classes and functions
-__all__ = ["UnifiedSearcher", "search_async", "search_sync", "build_index"]
-
-# EOF

scitex/scholar/.legacy/_tmp/search_engine/__init__.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-08-01 17:54:17 (ywatanabe)"
-# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/search_engine/__init__.py
-# ----------------------------------------
-from __future__ import annotations
-import os
-
-__FILE__ = __file__
-__DIR__ = os.path.dirname(__FILE__)
-# ----------------------------------------
-
-"""
-Search module for SciTeX Scholar.
-
-This module provides search functionality across multiple academic sources.
-"""
-
-# Import base class
-from ._BaseSearchEngine import BaseSearchEngine
-
-# Import concrete search engines
-from .web._PubMedSearchEngine import PubMedSearchEngine
-from .web._ArxivSearchEngine import ArxivSearchEngine
-from .web._SemanticScholarSearchEngine import SemanticScholarSearchEngine
-from .web._CrossRefSearchEngine import CrossRefSearchEngine
-from .web._GoogleScholarSearchEngine import GoogleScholarSearchEngine
-from .local._LocalSearchEngine import LocalSearchEngine
-from .local._VectorSearchEngine import VectorSearchEngine
-
-# Import unified searcher
-from ._UnifiedSearcher import UnifiedSearcher, search_async, search_sync, build_index
-
-__all__ = [
-    # Base class
-    "BaseSearchEngine",
-    # Concrete search engines
-    "PubMedSearchEngine",
-    "ArxivSearchEngine",
-    "SemanticScholarSearchEngine",
-    "CrossRefSearchEngine",
-    "GoogleScholarSearchEngine",
-    "LocalSearchEngine",
-    "VectorSearchEngine",
-    # Unified searcher
-    "UnifiedSearcher",
-    # Convenience functions
-    "search_async",
-    "search_sync",
-    "build_index",
-]
-
-# EOF

scitex/scholar/.legacy/_tmp/search_engine/local/_LocalSearchEngine.py

@@ -1,134 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-07-26 15:37:00 (ywatanabe)"
-# File: ./src/scitex/scholar/search/_LocalSearchEngine.py
-# ----------------------------------------
-from __future__ import annotations
-
-"""
-Local search engine for PDF files.
-
-This module provides search functionality for local PDF collections.
-"""
-
-import asyncio
-import json
-from scitex import logging
-from pathlib import Path
-from typing import List, Dict, Any, Optional
-
-from .._BaseSearchEngine import BaseSearchEngine
-from scitex.scholar.core import Paper
-from ...utils._paths import get_scholar_dir
-
-logger = logging.getLogger(__name__)
-
-
-class LocalSearchEngine(BaseSearchEngine):
-    """Search engine for local PDF files."""
-
-    def __init__(self, index_path: Optional[Path] = None):
-        super().__init__(name="local", rate_limit=0)
-        self.index_path = index_path or get_scholar_dir() / "local_index.json"
-        self.index = self._load_index()
-
-    async def search_async(self, query: str, limit: int = 20, **kwargs) -> List[Paper]:
-        """Search local PDF collection."""
-        # Local search is synchronous, wrap in async
-        return await asyncio.to_thread(self._search_sync, query, limit, kwargs)
-
-    def _search_sync(self, query: str, limit: int, kwargs: dict) -> List[Paper]:
-        """Synchronous local search implementation."""
-        if not self.index:
-            return []
-
-        # Simple keyword matching
-        query_terms = query.lower().split()
-        scored_papers = []
-
-        for paper_data in self.index.values():
-            # Calculate relevance score
-            score = 0
-            searchable_text = f"{paper_data.get('title', '')} {paper_data.get('abstract', '')} {' '.join(paper_data.get('keywords', []))}".lower()
-
-            for term in query_terms:
-                score += searchable_text.count(term)
-
-            if score > 0:
-                # Create Paper object
-                paper = Paper(
-                    title=paper_data.get("title", "Unknown Title"),
-                    authors=paper_data.get("authors", []),
-                    abstract=paper_data.get("abstract", ""),
-                    source="local",
-                    year=paper_data.get("year"),
-                    keywords=paper_data.get("keywords", []),
-                    pdf_path=Path(paper_data.get("pdf_path", "")),
-                )
-                scored_papers.append((score, paper))
-
-        # Sort by score and return top results
-        scored_papers.sort(key=lambda x: x[0], reverse=True)
-        return [paper for score, paper in scored_papers[:limit]]
-
-    def _load_index(self) -> Dict[str, Any]:
-        """Load local search index."""
-        if self.index_path.exists():
-            try:
-                with open(self.index_path, "r") as f:
-                    return json.load(f)
-            except Exception as e:
-                logger.warning(f"Failed to load local index: {e}")
-        return {}
-
-    def build_index(self, pdf_dirs: List[Path]) -> Dict[str, Any]:
-        """Build search index from PDF directories."""
-        logger.info(f"Building local index from {len(pdf_dirs)} directories")
-
-        index = {}
-        stats = {"files_indexed": 0, "errors": 0}
-
-        for pdf_dir in pdf_dirs:
-            if not pdf_dir.exists():
-                continue
-
-            for pdf_path in pdf_dir.rglob("*.pdf"):
-                try:
-                    # Extract text and metadata
-                    paper_data = self._extract_pdf_metadata(pdf_path)
-                    if paper_data:
-                        index[str(pdf_path)] = paper_data
-                        stats["files_indexed"] += 1
-                except Exception as e:
-                    logger.warning(f"Failed to index {pdf_path}: {e}")
-                    stats["errors"] += 1
-
-        # Save index
-        self.index = index
-        self._save_index()
-
-        logger.info(
-            f"Indexed {stats['files_indexed']} files with {stats['errors']} errors"
-        )
-        return stats
-
-    def _extract_pdf_metadata(self, pdf_path: Path) -> Optional[Dict[str, Any]]:
-        """Extract metadata from PDF file."""
-        # This is a placeholder - in real implementation would use PyPDF2 or similar
-        return {
-            "title": pdf_path.stem.replace("_", " ").title(),
-            "authors": [],
-            "abstract": "",
-            "year": None,
-            "keywords": [],
-            "pdf_path": str(pdf_path),
-        }
-
-    def _save_index(self) -> None:
-        """Save index to disk."""
-        self.index_path.parent.mkdir(parents=True, exist_ok=True)
-        with open(self.index_path, "w") as f:
-            json.dump(self.index, f, indent=2)
-
-
-# EOF

scitex/scholar/.legacy/_tmp/search_engine/local/_VectorSearchEngine.py

@@ -1,129 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-07-26 15:38:00 (ywatanabe)"
-# File: ./src/scitex/scholar/search/_VectorSearchEngine.py
-# ----------------------------------------
-from __future__ import annotations
-
-"""
-Vector similarity search using sentence embeddings.
-
-This module provides semantic search functionality using vector embeddings.
-"""
-
-import asyncio
-from scitex import logging
-import pickle
-from pathlib import Path
-from typing import List, Optional
-
-import numpy as np
-
-from .._BaseSearchEngine import BaseSearchEngine
-from scitex.scholar.core import Paper
-from ...utils._paths import get_scholar_dir
-
-logger = logging.getLogger(__name__)
-
-
-class VectorSearchEngine(BaseSearchEngine):
-    """Vector similarity search using sentence embeddings."""
-
-    def __init__(
-        self, index_path: Optional[Path] = None, model_name: str = "all-MiniLM-L6-v2"
-    ):
-        super().__init__(name="vector", rate_limit=0)
-        self.index_path = index_path or get_scholar_dir() / "vector_index.pkl"
-        self.model_name = model_name
-        self._model = None
-        self._papers = []
-        self._embeddings = None
-        self._load_index()
-
-    async def search_async(self, query: str, limit: int = 20, **kwargs) -> List[Paper]:
-        """Search using vector similarity."""
-        # Vector search is CPU-bound, use thread
-        return await asyncio.to_thread(self._search_sync, query, limit)
-
-    def _search_sync(self, query: str, limit: int) -> List[Paper]:
-        """Synchronous vector search implementation."""
-        if not self._embeddings or not self._papers:
-            return []
-
-        # Lazy load model
-        if self._model is None:
-            self._load_model()
-
-        # Encode query
-        query_embedding = self._model.encode([query])[0]
-
-        # Calculate similarities
-        similarities = np.dot(self._embeddings, query_embedding)
-
-        # Get top results
-        top_indices = np.argsort(similarities)[-limit:][::-1]
-
-        results = []
-        for idx in top_indices:
-            if idx < len(self._papers):
-                results.append(self._papers[idx])
-
-        return results
-
-    def add_papers(self, papers: List[Paper]) -> None:
-        """Add papers to vector index."""
-        if self._model is None:
-            self._load_model()
-
-        # Create searchable text for each paper
-        texts = []
-        for paper in papers:
-            text = f"{paper.title} {paper.abstract}"
-            texts.append(text)
-
-        # Encode papers
-        new_embeddings = self._model.encode(texts)
-
-        # Add to index
-        if self._embeddings is None:
-            self._embeddings = new_embeddings
-            self._papers = papers
-        else:
-            self._embeddings = np.vstack([self._embeddings, new_embeddings])
-            self._papers.extend(papers)
-
-        # Save index
-        self._save_index()
-
-    def _load_model(self) -> None:
-        """Load sentence transformer model."""
-        try:
-            from sentence_transformers import SentenceTransformer
-
-            self._model = SentenceTransformer(self.model_name)
-        except ImportError:
-            logger.warning(
-                "sentence-transformers not installed. Vector search disabled."
-            )
-            self._model = None
-
-    def _load_index(self) -> None:
-        """Load vector index from disk."""
-        if self.index_path.exists():
-            try:
-                with open(self.index_path, "rb") as f:
-                    data = pickle.load(f)
-                    self._papers = data.get("papers", [])
-                    self._embeddings = data.get("embeddings")
-            except Exception as e:
-                logger.warning(f"Failed to load vector index: {e}")
-
-    def _save_index(self) -> None:
-        """Save vector index to disk."""
-        self.index_path.parent.mkdir(parents=True, exist_ok=True)
-        data = {"papers": self._papers, "embeddings": self._embeddings}
-        with open(self.index_path, "wb") as f:
-            pickle.dump(data, f)
-
-
-# EOF