scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/scholar/_local_search.py

@@ -1,454 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2024-12-06 10:15:00"
-# Author: Claude
-# Filename: _local_search.py
-
-"""
-Local search engine for PDF papers and documents.
-"""
-
-import os
-import re
-from pathlib import Path
-from typing import List, Optional, Dict, Any, Tuple
-import logging
-import hashlib
-from datetime import datetime
-import json
-
-from ._paper import Paper
-
-
-logger = logging.getLogger(__name__)
-
-
-class LocalSearchEngine:
-    """Search engine for local PDF papers and documents."""
-    
-    def __init__(
-        self,
-        index_path: Optional[Path] = None,
-        cache_metadata: bool = True,
-    ):
-        """Initialize local search engine.
-        
-        Parameters
-        ----------
-        index_path : Path, optional
-            Path to save/load the metadata index
-        cache_metadata : bool
-            Whether to cache extracted metadata
-        """
-        self.index_path = Path(index_path) if index_path else None
-        self.cache_metadata = cache_metadata
-        
-        # Metadata cache: file_path -> metadata
-        self.metadata_cache: Dict[str, Dict[str, Any]] = {}
-        
-        # Load existing cache if available
-        if self.index_path and self.index_path.exists():
-            self.load_cache()
-        
-        # PDF reader (lazy loaded)
-        self._pdf_reader = None
-    
-    def _get_pdf_reader(self):
-        """Lazy load PDF reader."""
-        if self._pdf_reader is None:
-            try:
-                import fitz  # PyMuPDF
-                self._pdf_reader = "pymupdf"
-            except ImportError:
-                try:
-                    import PyPDF2
-                    self._pdf_reader = "pypdf2"
-                except ImportError:
-                    logger.warning("No PDF reader available (install pymupdf or PyPDF2)")
-                    self._pdf_reader = "none"
-        
-        return self._pdf_reader
-    
-    def search(
-        self,
-        query: str,
-        paths: List[Path],
-        recursive: bool = True,
-        file_pattern: str = "*.pdf",
-        max_results: Optional[int] = None,
-    ) -> List[Tuple[Paper, float]]:
-        """Search for papers in local directories.
-        
-        Parameters
-        ----------
-        query : str
-            Search query
-        paths : List[Path]
-            Directories to search
-        recursive : bool
-            Whether to search recursively
-        file_pattern : str
-            File pattern to match (e.g., "*.pdf")
-        max_results : int, optional
-            Maximum number of results
-        
-        Returns
-        -------
-        List[Tuple[Paper, float]]
-            List of (paper, relevance_score) tuples
-        """
-        results = []
-        query_lower = query.lower()
-        query_terms = set(query_lower.split())
-        
-        # Find all matching files
-        pdf_files = []
-        for path in paths:
-            path = Path(path)
-            if not path.exists():
-                logger.warning(f"Path does not exist: {path}")
-                continue
-            
-            if path.is_file():
-                pdf_files.append(path)
-            else:
-                if recursive:
-                    pdf_files.extend(path.rglob(file_pattern))
-                else:
-                    pdf_files.extend(path.glob(file_pattern))
-        
-        # Search through files
-        for pdf_path in pdf_files:
-            try:
-                # Get metadata (from cache or extract)
-                metadata = self._get_pdf_metadata(pdf_path)
-                
-                if not metadata:
-                    continue
-                
-                # Calculate relevance score
-                score = self._calculate_relevance(query_lower, query_terms, metadata)
-                
-                if score > 0:
-                    # Create Paper object
-                    paper = self._create_paper_from_metadata(pdf_path, metadata)
-                    results.append((paper, score))
-                    
-                    if max_results and len(results) >= max_results:
-                        break
-            
-            except Exception as e:
-                logger.error(f"Error processing {pdf_path}: {e}")
-                continue
-        
-        # Sort by relevance score
-        results.sort(key=lambda x: x[1], reverse=True)
-        
-        if max_results:
-            results = results[:max_results]
-        
-        return results
-    
-    def _get_pdf_metadata(self, pdf_path: Path) -> Optional[Dict[str, Any]]:
-        """Extract or retrieve metadata from PDF."""
-        # Check cache first
-        cache_key = str(pdf_path.absolute())
-        
-        if self.cache_metadata and cache_key in self.metadata_cache:
-            cached = self.metadata_cache[cache_key]
-            # Check if file has been modified
-            if cached.get("mtime") == pdf_path.stat().st_mtime:
-                return cached
-        
-        # Extract metadata
-        metadata = self._extract_pdf_metadata(pdf_path)
-        
-        if metadata and self.cache_metadata:
-            # Add file modification time
-            metadata["mtime"] = pdf_path.stat().st_mtime
-            self.metadata_cache[cache_key] = metadata
-            # Save cache periodically
-            if len(self.metadata_cache) % 10 == 0:
-                self.save_cache()
-        
-        return metadata
-    
-    def _extract_pdf_metadata(self, pdf_path: Path) -> Optional[Dict[str, Any]]:
-        """Extract metadata from PDF file."""
-        reader_type = self._get_pdf_reader()
-        
-        metadata = {
-            "filename": pdf_path.name,
-            "path": str(pdf_path),
-            "size": pdf_path.stat().st_size,
-            "modified": datetime.fromtimestamp(pdf_path.stat().st_mtime).isoformat(),
-        }
-        
-        if reader_type == "none":
-            # Fallback: use filename
-            metadata["title"] = pdf_path.stem.replace("_", " ").replace("-", " ")
-            metadata["content"] = metadata["title"]
-            return metadata
-        
-        try:
-            if reader_type == "pymupdf":
-                import fitz
-                
-                with fitz.open(pdf_path) as doc:
-                    # Get document metadata
-                    info = doc.metadata
-                    metadata["title"] = info.get("title", "") or pdf_path.stem
-                    metadata["author"] = info.get("author", "")
-                    metadata["subject"] = info.get("subject", "")
-                    metadata["keywords"] = info.get("keywords", "")
-                    
-                    # Extract text from first few pages
-                    text_parts = []
-                    for i in range(min(3, len(doc))):  # First 3 pages
-                        page = doc[i]
-                        text = page.get_text()
-                        if text:
-                            text_parts.append(text)
-                    
-                    metadata["content"] = " ".join(text_parts)[:5000]  # Limit content
-                    
-                    # Try to extract abstract
-                    full_text = " ".join(text_parts)
-                    abstract = self._extract_abstract(full_text)
-                    if abstract:
-                        metadata["abstract"] = abstract
-            
-            elif reader_type == "pypdf2":
-                import PyPDF2
-                
-                with open(pdf_path, "rb") as f:
-                    reader = PyPDF2.PdfReader(f)
-                    
-                    # Get metadata
-                    info = reader.metadata
-                    if info:
-                        metadata["title"] = info.get("/Title", "") or pdf_path.stem
-                        metadata["author"] = info.get("/Author", "")
-                        metadata["subject"] = info.get("/Subject", "")
-                        metadata["keywords"] = info.get("/Keywords", "")
-                    
-                    # Extract text from first few pages
-                    text_parts = []
-                    for i in range(min(3, len(reader.pages))):
-                        page = reader.pages[i]
-                        text = page.extract_text()
-                        if text:
-                            text_parts.append(text)
-                    
-                    metadata["content"] = " ".join(text_parts)[:5000]
-                    
-                    # Try to extract abstract
-                    full_text = " ".join(text_parts)
-                    abstract = self._extract_abstract(full_text)
-                    if abstract:
-                        metadata["abstract"] = abstract
-        
-        except Exception as e:
-            logger.error(f"Error extracting metadata from {pdf_path}: {e}")
-            # Fallback to filename
-            metadata["title"] = pdf_path.stem.replace("_", " ").replace("-", " ")
-            metadata["content"] = metadata["title"]
-        
-        return metadata
-    
-    def _extract_abstract(self, text: str) -> Optional[str]:
-        """Try to extract abstract from text."""
-        # Look for abstract section
-        patterns = [
-            r"abstract[:\s]*(.+?)(?=introduction|keywords|1\.|1\s+introduction)",
-            r"summary[:\s]*(.+?)(?=introduction|keywords|1\.|1\s+introduction)",
-        ]
-        
-        for pattern in patterns:
-            match = re.search(pattern, text, re.IGNORECASE | re.DOTALL)
-            if match:
-                abstract = match.group(1).strip()
-                # Clean up
-                abstract = re.sub(r"\s+", " ", abstract)
-                if len(abstract) > 100:  # Reasonable abstract length
-                    return abstract[:1000]  # Limit length
-        
-        return None
-    
-    def _calculate_relevance(
-        self,
-        query_lower: str,
-        query_terms: set,
-        metadata: Dict[str, Any]
-    ) -> float:
-        """Calculate relevance score for a document."""
-        score = 0.0
-        
-        # Search in different fields with different weights
-        fields = [
-            ("title", 5.0),
-            ("abstract", 3.0),
-            ("keywords", 3.0),
-            ("author", 2.0),
-            ("subject", 2.0),
-            ("content", 1.0),
-            ("filename", 1.0),
-        ]
-        
-        for field, weight in fields:
-            field_value = metadata.get(field, "").lower()
-            if not field_value:
-                continue
-            
-            # Exact match
-            if query_lower in field_value:
-                score += weight * 2
-            
-            # Term matches
-            field_terms = set(field_value.split())
-            matching_terms = query_terms & field_terms
-            if matching_terms:
-                score += weight * len(matching_terms) / len(query_terms)
-        
-        return score
-    
-    def _create_paper_from_metadata(
-        self,
-        pdf_path: Path,
-        metadata: Dict[str, Any]
-    ) -> Paper:
-        """Create Paper object from metadata."""
-        # Parse authors
-        authors = []
-        author_str = metadata.get("author", "")
-        if author_str:
-            # Try to split authors
-            if ";" in author_str:
-                authors = [a.strip() for a in author_str.split(";")]
-            elif "," in author_str and author_str.count(",") > 1:
-                authors = [a.strip() for a in author_str.split(",")]
-            else:
-                authors = [author_str.strip()]
-        
-        # Parse keywords
-        keywords = []
-        keyword_str = metadata.get("keywords", "")
-        if keyword_str:
-            keywords = [k.strip() for k in re.split(r"[;,]", keyword_str) if k.strip()]
-        
-        # Create paper
-        paper = Paper(
-            title=metadata.get("title", pdf_path.stem),
-            authors=authors,
-            abstract=metadata.get("abstract", metadata.get("content", "")[:500]),
-            source="local",
-            keywords=keywords,
-            pdf_path=pdf_path,
-            metadata={
-                "filename": metadata.get("filename"),
-                "size": metadata.get("size"),
-                "modified": metadata.get("modified"),
-                "subject": metadata.get("subject"),
-            },
-        )
-        
-        return paper
-    
-    def build_index(self, paths: List[Path], recursive: bool = True) -> int:
-        """Build metadata index for given paths.
-        
-        Parameters
-        ----------
-        paths : List[Path]
-            Directories to index
-        recursive : bool
-            Whether to search recursively
-        
-        Returns
-        -------
-        int
-            Number of files indexed
-        """
-        count = 0
-        
-        for path in paths:
-            path = Path(path)
-            if not path.exists():
-                continue
-            
-            if path.is_file() and path.suffix.lower() == ".pdf":
-                pdf_files = [path]
-            else:
-                if recursive:
-                    pdf_files = list(path.rglob("*.pdf"))
-                else:
-                    pdf_files = list(path.glob("*.pdf"))
-            
-            for pdf_path in pdf_files:
-                try:
-                    metadata = self._get_pdf_metadata(pdf_path)
-                    if metadata:
-                        count += 1
-                        if count % 10 == 0:
-                            logger.info(f"Indexed {count} files...")
-                except Exception as e:
-                    logger.error(f"Error indexing {pdf_path}: {e}")
-        
-        # Save cache
-        if self.cache_metadata:
-            self.save_cache()
-        
-        logger.info(f"Indexed {count} PDF files")
-        return count
-    
-    def save_cache(self) -> None:
-        """Save metadata cache to disk."""
-        if not self.index_path:
-            return
-        
-        self.index_path.parent.mkdir(parents=True, exist_ok=True)
-        
-        with open(self.index_path, "w") as f:
-            json.dump(self.metadata_cache, f, indent=2)
-        
-        logger.debug(f"Saved cache with {len(self.metadata_cache)} entries")
-    
-    def load_cache(self) -> None:
-        """Load metadata cache from disk."""
-        if not self.index_path or not self.index_path.exists():
-            return
-        
-        try:
-            with open(self.index_path, "r") as f:
-                self.metadata_cache = json.load(f)
-            logger.debug(f"Loaded cache with {len(self.metadata_cache)} entries")
-        except Exception as e:
-            logger.error(f"Error loading cache: {e}")
-    
-    def clear_cache(self) -> None:
-        """Clear the metadata cache."""
-        self.metadata_cache.clear()
-        if self.index_path and self.index_path.exists():
-            self.index_path.unlink()
-
-
-# Example usage
-if __name__ == "__main__":
-    # Create local search engine
-    engine = LocalSearchEngine()
-    
-    # Search in current directory
-    results = engine.search("machine learning", [Path(".")], max_results=5)
-    
-    print(f"Found {len(results)} papers:")
-    for paper, score in results:
-        print(f"\nScore: {score:.2f}")
-        print(f"Title: {paper.title}")
-        print(f"Path: {paper.pdf_path}")
-        if paper.authors:
-            print(f"Authors: {', '.join(paper.authors)}")
-    
-    # Build index for faster future searches
-    print("\nBuilding index...")
-    count = engine.build_index([Path(".")])
-    print(f"Indexed {count} files")

scitex/scholar/_paper.py

@@ -1,244 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2024-12-06 10:00:00"
-# Author: Claude
-# Filename: _paper.py
-
-"""
-Paper class for representing scientific papers.
-"""
-
-from typing import Dict, Optional, List, Any
-from datetime import datetime
-from pathlib import Path
-import json
-
-
-class Paper:
-    """Represents a scientific paper with metadata and content."""
-    
-    def __init__(
-        self,
-        title: str,
-        authors: List[str],
-        abstract: str,
-        source: str,  # 'pubmed', 'arxiv', 'local', etc.
-        year: Optional[int] = None,
-        doi: Optional[str] = None,
-        pmid: Optional[str] = None,
-        arxiv_id: Optional[str] = None,
-        journal: Optional[str] = None,
-        keywords: Optional[List[str]] = None,
-        pdf_path: Optional[Path] = None,
-        embedding: Optional[Any] = None,  # numpy array or tensor
-        metadata: Optional[Dict[str, Any]] = None,
-    ):
-        """Initialize a Paper object.
-        
-        Parameters
-        ----------
-        title : str
-            Paper title
-        authors : List[str]
-            List of author names
-        abstract : str
-            Paper abstract
-        source : str
-            Source of the paper (pubmed, arxiv, local, etc.)
-        year : int, optional
-            Publication year
-        doi : str, optional
-            Digital Object Identifier
-        pmid : str, optional
-            PubMed ID
-        arxiv_id : str, optional
-            arXiv identifier
-        journal : str, optional
-            Journal name
-        keywords : List[str], optional
-            Keywords/tags
-        pdf_path : Path, optional
-            Path to local PDF file
-        embedding : Any, optional
-            Vector embedding of the paper
-        metadata : Dict[str, Any], optional
-            Additional metadata
-        """
-        self.title = title
-        self.authors = authors
-        self.abstract = abstract
-        self.source = source
-        self.year = year
-        self.doi = doi
-        self.pmid = pmid
-        self.arxiv_id = arxiv_id
-        self.journal = journal
-        self.keywords = keywords or []
-        self.pdf_path = Path(pdf_path) if pdf_path else None
-        self.embedding = embedding
-        self.metadata = metadata or {}
-        self.retrieved_at = datetime.now()
-    
-    def __repr__(self) -> str:
-        """String representation of the paper."""
-        return f"Paper(title='{self.title[:50]}...', authors={len(self.authors)}, source='{self.source}')"
-    
-    def __str__(self) -> str:
-        """Human-readable string representation."""
-        authors_str = ", ".join(self.authors[:3])
-        if len(self.authors) > 3:
-            authors_str += f" et al. ({len(self.authors)} authors)"
-        
-        parts = [
-            f"Title: {self.title}",
-            f"Authors: {authors_str}",
-            f"Year: {self.year or 'Unknown'}",
-            f"Source: {self.source}",
-        ]
-        
-        if self.journal:
-            parts.append(f"Journal: {self.journal}")
-        if self.doi:
-            parts.append(f"DOI: {self.doi}")
-        if self.pmid:
-            parts.append(f"PMID: {self.pmid}")
-        if self.arxiv_id:
-            parts.append(f"arXiv: {self.arxiv_id}")
-        
-        return "\n".join(parts)
-    
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert paper to dictionary representation."""
-        return {
-            "title": self.title,
-            "authors": self.authors,
-            "abstract": self.abstract,
-            "source": self.source,
-            "year": self.year,
-            "doi": self.doi,
-            "pmid": self.pmid,
-            "arxiv_id": self.arxiv_id,
-            "journal": self.journal,
-            "keywords": self.keywords,
-            "pdf_path": str(self.pdf_path) if self.pdf_path else None,
-            "has_embedding": self.embedding is not None,
-            "metadata": self.metadata,
-            "retrieved_at": self.retrieved_at.isoformat(),
-        }
-    
-    @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> "Paper":
-        """Create Paper from dictionary representation."""
-        # Remove computed fields
-        data = data.copy()
-        data.pop("has_embedding", None)
-        data.pop("retrieved_at", None)
-        
-        # Convert pdf_path back to Path if present
-        if data.get("pdf_path"):
-            data["pdf_path"] = Path(data["pdf_path"])
-        
-        return cls(**data)
-    
-    def to_bibtex(self) -> str:
-        """Generate BibTeX entry for the paper."""
-        # Generate citation key
-        first_author = self.authors[0].split()[-1] if self.authors else "Unknown"
-        year = self.year or "0000"
-        title_word = self.title.split()[0].lower() if self.title else "paper"
-        cite_key = f"{first_author}{year}{title_word}"
-        
-        # Determine entry type
-        if self.arxiv_id:
-            entry_type = "@misc"
-        else:
-            entry_type = "@article"
-        
-        # Build BibTeX entry
-        lines = [f"{entry_type}{{{cite_key},"]
-        lines.append(f'  title = "{{{self.title}}}",')
-        
-        if self.authors:
-            authors_str = " and ".join(self.authors)
-            lines.append(f'  author = "{{{authors_str}}}",')
-        
-        if self.year:
-            lines.append(f'  year = "{{{self.year}}}",')
-        
-        if self.journal:
-            lines.append(f'  journal = "{{{self.journal}}}",')
-        
-        if self.doi:
-            lines.append(f'  doi = "{{{self.doi}}}",')
-        
-        if self.arxiv_id:
-            lines.append(f'  eprint = "{{{self.arxiv_id}}}",')
-            lines.append('  archivePrefix = "{arXiv}",')
-        
-        lines.append("}")
-        
-        return "\n".join(lines)
-    
-    def get_identifier(self) -> str:
-        """Get a unique identifier for the paper."""
-        if self.doi:
-            return f"doi:{self.doi}"
-        elif self.pmid:
-            return f"pmid:{self.pmid}"
-        elif self.arxiv_id:
-            return f"arxiv:{self.arxiv_id}"
-        else:
-            # Create a hash from title and authors
-            import hashlib
-            content = f"{self.title}_{';'.join(self.authors)}"
-            return f"hash:{hashlib.md5(content.encode()).hexdigest()[:12]}"
-    
-    def has_pdf(self) -> bool:
-        """Check if paper has an associated PDF file."""
-        return self.pdf_path is not None and self.pdf_path.exists()
-    
-    def similarity_score(self, other: "Paper") -> float:
-        """Calculate similarity score with another paper (0-1)."""
-        score = 0.0
-        weights = {"title": 0.4, "authors": 0.3, "abstract": 0.3}
-        
-        # Title similarity (simple word overlap)
-        title_words1 = set(self.title.lower().split())
-        title_words2 = set(other.title.lower().split())
-        if title_words1 and title_words2:
-            title_overlap = len(title_words1 & title_words2) / len(title_words1 | title_words2)
-            score += weights["title"] * title_overlap
-        
-        # Author similarity
-        authors1 = set(self.authors)
-        authors2 = set(other.authors)
-        if authors1 and authors2:
-            author_overlap = len(authors1 & authors2) / len(authors1 | authors2)
-            score += weights["authors"] * author_overlap
-        
-        # Abstract similarity (simple word overlap)
-        abstract_words1 = set(self.abstract.lower().split())
-        abstract_words2 = set(other.abstract.lower().split())
-        if abstract_words1 and abstract_words2:
-            abstract_overlap = len(abstract_words1 & abstract_words2) / len(abstract_words1 | abstract_words2)
-            score += weights["abstract"] * abstract_overlap
-        
-        return score
-
-
-# Example usage
-if __name__ == "__main__":
-    paper = Paper(
-        title="Deep Learning for Scientific Discovery",
-        authors=["John Doe", "Jane Smith", "Bob Johnson"],
-        abstract="This paper explores the application of deep learning...",
-        source="arxiv",
-        year=2024,
-        arxiv_id="2401.12345",
-        keywords=["deep learning", "scientific computing", "AI"],
-    )
-    
-    print(paper)
-    print("\nBibTeX:")
-    print(paper.to_bibtex())
-    print("\nIdentifier:", paper.get_identifier())