scitex 2.0.0__py2.py3-none-any.whl

scitex/scholar/_local_search.py

@@ -0,0 +1,454 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-12-06 10:15:00"
+# Author: Claude
+# Filename: _local_search.py
+
+"""
+Local search engine for PDF papers and documents.
+"""
+
+import os
+import re
+from pathlib import Path
+from typing import List, Optional, Dict, Any, Tuple
+import logging
+import hashlib
+from datetime import datetime
+import json
+
+from ._paper import Paper
+
+
+logger = logging.getLogger(__name__)
+
+
+class LocalSearchEngine:
+    """Search engine for local PDF papers and documents."""
+    
+    def __init__(
+        self,
+        index_path: Optional[Path] = None,
+        cache_metadata: bool = True,
+    ):
+        """Initialize local search engine.
+        
+        Parameters
+        ----------
+        index_path : Path, optional
+            Path to save/load the metadata index
+        cache_metadata : bool
+            Whether to cache extracted metadata
+        """
+        self.index_path = Path(index_path) if index_path else None
+        self.cache_metadata = cache_metadata
+        
+        # Metadata cache: file_path -> metadata
+        self.metadata_cache: Dict[str, Dict[str, Any]] = {}
+        
+        # Load existing cache if available
+        if self.index_path and self.index_path.exists():
+            self.load_cache()
+        
+        # PDF reader (lazy loaded)
+        self._pdf_reader = None
+    
+    def _get_pdf_reader(self):
+        """Lazy load PDF reader."""
+        if self._pdf_reader is None:
+            try:
+                import fitz  # PyMuPDF
+                self._pdf_reader = "pymupdf"
+            except ImportError:
+                try:
+                    import PyPDF2
+                    self._pdf_reader = "pypdf2"
+                except ImportError:
+                    logger.warning("No PDF reader available (install pymupdf or PyPDF2)")
+                    self._pdf_reader = "none"
+        
+        return self._pdf_reader
+    
+    def search(
+        self,
+        query: str,
+        paths: List[Path],
+        recursive: bool = True,
+        file_pattern: str = "*.pdf",
+        max_results: Optional[int] = None,
+    ) -> List[Tuple[Paper, float]]:
+        """Search for papers in local directories.
+        
+        Parameters
+        ----------
+        query : str
+            Search query
+        paths : List[Path]
+            Directories to search
+        recursive : bool
+            Whether to search recursively
+        file_pattern : str
+            File pattern to match (e.g., "*.pdf")
+        max_results : int, optional
+            Maximum number of results
+        
+        Returns
+        -------
+        List[Tuple[Paper, float]]
+            List of (paper, relevance_score) tuples
+        """
+        results = []
+        query_lower = query.lower()
+        query_terms = set(query_lower.split())
+        
+        # Find all matching files
+        pdf_files = []
+        for path in paths:
+            path = Path(path)
+            if not path.exists():
+                logger.warning(f"Path does not exist: {path}")
+                continue
+            
+            if path.is_file():
+                pdf_files.append(path)
+            else:
+                if recursive:
+                    pdf_files.extend(path.rglob(file_pattern))
+                else:
+                    pdf_files.extend(path.glob(file_pattern))
+        
+        # Search through files
+        for pdf_path in pdf_files:
+            try:
+                # Get metadata (from cache or extract)
+                metadata = self._get_pdf_metadata(pdf_path)
+                
+                if not metadata:
+                    continue
+                
+                # Calculate relevance score
+                score = self._calculate_relevance(query_lower, query_terms, metadata)
+                
+                if score > 0:
+                    # Create Paper object
+                    paper = self._create_paper_from_metadata(pdf_path, metadata)
+                    results.append((paper, score))
+                    
+                    if max_results and len(results) >= max_results:
+                        break
+            
+            except Exception as e:
+                logger.error(f"Error processing {pdf_path}: {e}")
+                continue
+        
+        # Sort by relevance score
+        results.sort(key=lambda x: x[1], reverse=True)
+        
+        if max_results:
+            results = results[:max_results]
+        
+        return results
+    
+    def _get_pdf_metadata(self, pdf_path: Path) -> Optional[Dict[str, Any]]:
+        """Extract or retrieve metadata from PDF."""
+        # Check cache first
+        cache_key = str(pdf_path.absolute())
+        
+        if self.cache_metadata and cache_key in self.metadata_cache:
+            cached = self.metadata_cache[cache_key]
+            # Check if file has been modified
+            if cached.get("mtime") == pdf_path.stat().st_mtime:
+                return cached
+        
+        # Extract metadata
+        metadata = self._extract_pdf_metadata(pdf_path)
+        
+        if metadata and self.cache_metadata:
+            # Add file modification time
+            metadata["mtime"] = pdf_path.stat().st_mtime
+            self.metadata_cache[cache_key] = metadata
+            # Save cache periodically
+            if len(self.metadata_cache) % 10 == 0:
+                self.save_cache()
+        
+        return metadata
+    
+    def _extract_pdf_metadata(self, pdf_path: Path) -> Optional[Dict[str, Any]]:
+        """Extract metadata from PDF file."""
+        reader_type = self._get_pdf_reader()
+        
+        metadata = {
+            "filename": pdf_path.name,
+            "path": str(pdf_path),
+            "size": pdf_path.stat().st_size,
+            "modified": datetime.fromtimestamp(pdf_path.stat().st_mtime).isoformat(),
+        }
+        
+        if reader_type == "none":
+            # Fallback: use filename
+            metadata["title"] = pdf_path.stem.replace("_", " ").replace("-", " ")
+            metadata["content"] = metadata["title"]
+            return metadata
+        
+        try:
+            if reader_type == "pymupdf":
+                import fitz
+                
+                with fitz.open(pdf_path) as doc:
+                    # Get document metadata
+                    info = doc.metadata
+                    metadata["title"] = info.get("title", "") or pdf_path.stem
+                    metadata["author"] = info.get("author", "")
+                    metadata["subject"] = info.get("subject", "")
+                    metadata["keywords"] = info.get("keywords", "")
+                    
+                    # Extract text from first few pages
+                    text_parts = []
+                    for i in range(min(3, len(doc))):  # First 3 pages
+                        page = doc[i]
+                        text = page.get_text()
+                        if text:
+                            text_parts.append(text)
+                    
+                    metadata["content"] = " ".join(text_parts)[:5000]  # Limit content
+                    
+                    # Try to extract abstract
+                    full_text = " ".join(text_parts)
+                    abstract = self._extract_abstract(full_text)
+                    if abstract:
+                        metadata["abstract"] = abstract
+            
+            elif reader_type == "pypdf2":
+                import PyPDF2
+                
+                with open(pdf_path, "rb") as f:
+                    reader = PyPDF2.PdfReader(f)
+                    
+                    # Get metadata
+                    info = reader.metadata
+                    if info:
+                        metadata["title"] = info.get("/Title", "") or pdf_path.stem
+                        metadata["author"] = info.get("/Author", "")
+                        metadata["subject"] = info.get("/Subject", "")
+                        metadata["keywords"] = info.get("/Keywords", "")
+                    
+                    # Extract text from first few pages
+                    text_parts = []
+                    for i in range(min(3, len(reader.pages))):
+                        page = reader.pages[i]
+                        text = page.extract_text()
+                        if text:
+                            text_parts.append(text)
+                    
+                    metadata["content"] = " ".join(text_parts)[:5000]
+                    
+                    # Try to extract abstract
+                    full_text = " ".join(text_parts)
+                    abstract = self._extract_abstract(full_text)
+                    if abstract:
+                        metadata["abstract"] = abstract
+        
+        except Exception as e:
+            logger.error(f"Error extracting metadata from {pdf_path}: {e}")
+            # Fallback to filename
+            metadata["title"] = pdf_path.stem.replace("_", " ").replace("-", " ")
+            metadata["content"] = metadata["title"]
+        
+        return metadata
+    
+    def _extract_abstract(self, text: str) -> Optional[str]:
+        """Try to extract abstract from text."""
+        # Look for abstract section
+        patterns = [
+            r"abstract[:\s]*(.+?)(?=introduction|keywords|1\.|1\s+introduction)",
+            r"summary[:\s]*(.+?)(?=introduction|keywords|1\.|1\s+introduction)",
+        ]
+        
+        for pattern in patterns:
+            match = re.search(pattern, text, re.IGNORECASE | re.DOTALL)
+            if match:
+                abstract = match.group(1).strip()
+                # Clean up
+                abstract = re.sub(r"\s+", " ", abstract)
+                if len(abstract) > 100:  # Reasonable abstract length
+                    return abstract[:1000]  # Limit length
+        
+        return None
+    
+    def _calculate_relevance(
+        self,
+        query_lower: str,
+        query_terms: set,
+        metadata: Dict[str, Any]
+    ) -> float:
+        """Calculate relevance score for a document."""
+        score = 0.0
+        
+        # Search in different fields with different weights
+        fields = [
+            ("title", 5.0),
+            ("abstract", 3.0),
+            ("keywords", 3.0),
+            ("author", 2.0),
+            ("subject", 2.0),
+            ("content", 1.0),
+            ("filename", 1.0),
+        ]
+        
+        for field, weight in fields:
+            field_value = metadata.get(field, "").lower()
+            if not field_value:
+                continue
+            
+            # Exact match
+            if query_lower in field_value:
+                score += weight * 2
+            
+            # Term matches
+            field_terms = set(field_value.split())
+            matching_terms = query_terms & field_terms
+            if matching_terms:
+                score += weight * len(matching_terms) / len(query_terms)
+        
+        return score
+    
+    def _create_paper_from_metadata(
+        self,
+        pdf_path: Path,
+        metadata: Dict[str, Any]
+    ) -> Paper:
+        """Create Paper object from metadata."""
+        # Parse authors
+        authors = []
+        author_str = metadata.get("author", "")
+        if author_str:
+            # Try to split authors
+            if ";" in author_str:
+                authors = [a.strip() for a in author_str.split(";")]
+            elif "," in author_str and author_str.count(",") > 1:
+                authors = [a.strip() for a in author_str.split(",")]
+            else:
+                authors = [author_str.strip()]
+        
+        # Parse keywords
+        keywords = []
+        keyword_str = metadata.get("keywords", "")
+        if keyword_str:
+            keywords = [k.strip() for k in re.split(r"[;,]", keyword_str) if k.strip()]
+        
+        # Create paper
+        paper = Paper(
+            title=metadata.get("title", pdf_path.stem),
+            authors=authors,
+            abstract=metadata.get("abstract", metadata.get("content", "")[:500]),
+            source="local",
+            keywords=keywords,
+            pdf_path=pdf_path,
+            metadata={
+                "filename": metadata.get("filename"),
+                "size": metadata.get("size"),
+                "modified": metadata.get("modified"),
+                "subject": metadata.get("subject"),
+            },
+        )
+        
+        return paper
+    
+    def build_index(self, paths: List[Path], recursive: bool = True) -> int:
+        """Build metadata index for given paths.
+        
+        Parameters
+        ----------
+        paths : List[Path]
+            Directories to index
+        recursive : bool
+            Whether to search recursively
+        
+        Returns
+        -------
+        int
+            Number of files indexed
+        """
+        count = 0
+        
+        for path in paths:
+            path = Path(path)
+            if not path.exists():
+                continue
+            
+            if path.is_file() and path.suffix.lower() == ".pdf":
+                pdf_files = [path]
+            else:
+                if recursive:
+                    pdf_files = list(path.rglob("*.pdf"))
+                else:
+                    pdf_files = list(path.glob("*.pdf"))
+            
+            for pdf_path in pdf_files:
+                try:
+                    metadata = self._get_pdf_metadata(pdf_path)
+                    if metadata:
+                        count += 1
+                        if count % 10 == 0:
+                            logger.info(f"Indexed {count} files...")
+                except Exception as e:
+                    logger.error(f"Error indexing {pdf_path}: {e}")
+        
+        # Save cache
+        if self.cache_metadata:
+            self.save_cache()
+        
+        logger.info(f"Indexed {count} PDF files")
+        return count
+    
+    def save_cache(self) -> None:
+        """Save metadata cache to disk."""
+        if not self.index_path:
+            return
+        
+        self.index_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        with open(self.index_path, "w") as f:
+            json.dump(self.metadata_cache, f, indent=2)
+        
+        logger.debug(f"Saved cache with {len(self.metadata_cache)} entries")
+    
+    def load_cache(self) -> None:
+        """Load metadata cache from disk."""
+        if not self.index_path or not self.index_path.exists():
+            return
+        
+        try:
+            with open(self.index_path, "r") as f:
+                self.metadata_cache = json.load(f)
+            logger.debug(f"Loaded cache with {len(self.metadata_cache)} entries")
+        except Exception as e:
+            logger.error(f"Error loading cache: {e}")
+    
+    def clear_cache(self) -> None:
+        """Clear the metadata cache."""
+        self.metadata_cache.clear()
+        if self.index_path and self.index_path.exists():
+            self.index_path.unlink()
+
+
+# Example usage
+if __name__ == "__main__":
+    # Create local search engine
+    engine = LocalSearchEngine()
+    
+    # Search in current directory
+    results = engine.search("machine learning", [Path(".")], max_results=5)
+    
+    print(f"Found {len(results)} papers:")
+    for paper, score in results:
+        print(f"\nScore: {score:.2f}")
+        print(f"Title: {paper.title}")
+        print(f"Path: {paper.pdf_path}")
+        if paper.authors:
+            print(f"Authors: {', '.join(paper.authors)}")
+    
+    # Build index for faster future searches
+    print("\nBuilding index...")
+    count = engine.build_index([Path(".")])
+    print(f"Indexed {count} files")

scitex/scholar/_paper.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-12-06 10:00:00"
+# Author: Claude
+# Filename: _paper.py
+
+"""
+Paper class for representing scientific papers.
+"""
+
+from typing import Dict, Optional, List, Any
+from datetime import datetime
+from pathlib import Path
+import json
+
+
+class Paper:
+    """Represents a scientific paper with metadata and content."""
+    
+    def __init__(
+        self,
+        title: str,
+        authors: List[str],
+        abstract: str,
+        source: str,  # 'pubmed', 'arxiv', 'local', etc.
+        year: Optional[int] = None,
+        doi: Optional[str] = None,
+        pmid: Optional[str] = None,
+        arxiv_id: Optional[str] = None,
+        journal: Optional[str] = None,
+        keywords: Optional[List[str]] = None,
+        pdf_path: Optional[Path] = None,
+        embedding: Optional[Any] = None,  # numpy array or tensor
+        metadata: Optional[Dict[str, Any]] = None,
+    ):
+        """Initialize a Paper object.
+        
+        Parameters
+        ----------
+        title : str
+            Paper title
+        authors : List[str]
+            List of author names
+        abstract : str
+            Paper abstract
+        source : str
+            Source of the paper (pubmed, arxiv, local, etc.)
+        year : int, optional
+            Publication year
+        doi : str, optional
+            Digital Object Identifier
+        pmid : str, optional
+            PubMed ID
+        arxiv_id : str, optional
+            arXiv identifier
+        journal : str, optional
+            Journal name
+        keywords : List[str], optional
+            Keywords/tags
+        pdf_path : Path, optional
+            Path to local PDF file
+        embedding : Any, optional
+            Vector embedding of the paper
+        metadata : Dict[str, Any], optional
+            Additional metadata
+        """
+        self.title = title
+        self.authors = authors
+        self.abstract = abstract
+        self.source = source
+        self.year = year
+        self.doi = doi
+        self.pmid = pmid
+        self.arxiv_id = arxiv_id
+        self.journal = journal
+        self.keywords = keywords or []
+        self.pdf_path = Path(pdf_path) if pdf_path else None
+        self.embedding = embedding
+        self.metadata = metadata or {}
+        self.retrieved_at = datetime.now()
+    
+    def __repr__(self) -> str:
+        """String representation of the paper."""
+        return f"Paper(title='{self.title[:50]}...', authors={len(self.authors)}, source='{self.source}')"
+    
+    def __str__(self) -> str:
+        """Human-readable string representation."""
+        authors_str = ", ".join(self.authors[:3])
+        if len(self.authors) > 3:
+            authors_str += f" et al. ({len(self.authors)} authors)"
+        
+        parts = [
+            f"Title: {self.title}",
+            f"Authors: {authors_str}",
+            f"Year: {self.year or 'Unknown'}",
+            f"Source: {self.source}",
+        ]
+        
+        if self.journal:
+            parts.append(f"Journal: {self.journal}")
+        if self.doi:
+            parts.append(f"DOI: {self.doi}")
+        if self.pmid:
+            parts.append(f"PMID: {self.pmid}")
+        if self.arxiv_id:
+            parts.append(f"arXiv: {self.arxiv_id}")
+        
+        return "\n".join(parts)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert paper to dictionary representation."""
+        return {
+            "title": self.title,
+            "authors": self.authors,
+            "abstract": self.abstract,
+            "source": self.source,
+            "year": self.year,
+            "doi": self.doi,
+            "pmid": self.pmid,
+            "arxiv_id": self.arxiv_id,
+            "journal": self.journal,
+            "keywords": self.keywords,
+            "pdf_path": str(self.pdf_path) if self.pdf_path else None,
+            "has_embedding": self.embedding is not None,
+            "metadata": self.metadata,
+            "retrieved_at": self.retrieved_at.isoformat(),
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Paper":
+        """Create Paper from dictionary representation."""
+        # Remove computed fields
+        data = data.copy()
+        data.pop("has_embedding", None)
+        data.pop("retrieved_at", None)
+        
+        # Convert pdf_path back to Path if present
+        if data.get("pdf_path"):
+            data["pdf_path"] = Path(data["pdf_path"])
+        
+        return cls(**data)
+    
+    def to_bibtex(self) -> str:
+        """Generate BibTeX entry for the paper."""
+        # Generate citation key
+        first_author = self.authors[0].split()[-1] if self.authors else "Unknown"
+        year = self.year or "0000"
+        title_word = self.title.split()[0].lower() if self.title else "paper"
+        cite_key = f"{first_author}{year}{title_word}"
+        
+        # Determine entry type
+        if self.arxiv_id:
+            entry_type = "@misc"
+        else:
+            entry_type = "@article"
+        
+        # Build BibTeX entry
+        lines = [f"{entry_type}{{{cite_key},"]
+        lines.append(f'  title = "{{{self.title}}}",')
+        
+        if self.authors:
+            authors_str = " and ".join(self.authors)
+            lines.append(f'  author = "{{{authors_str}}}",')
+        
+        if self.year:
+            lines.append(f'  year = "{{{self.year}}}",')
+        
+        if self.journal:
+            lines.append(f'  journal = "{{{self.journal}}}",')
+        
+        if self.doi:
+            lines.append(f'  doi = "{{{self.doi}}}",')
+        
+        if self.arxiv_id:
+            lines.append(f'  eprint = "{{{self.arxiv_id}}}",')
+            lines.append('  archivePrefix = "{arXiv}",')
+        
+        lines.append("}")
+        
+        return "\n".join(lines)
+    
+    def get_identifier(self) -> str:
+        """Get a unique identifier for the paper."""
+        if self.doi:
+            return f"doi:{self.doi}"
+        elif self.pmid:
+            return f"pmid:{self.pmid}"
+        elif self.arxiv_id:
+            return f"arxiv:{self.arxiv_id}"
+        else:
+            # Create a hash from title and authors
+            import hashlib
+            content = f"{self.title}_{';'.join(self.authors)}"
+            return f"hash:{hashlib.md5(content.encode()).hexdigest()[:12]}"
+    
+    def has_pdf(self) -> bool:
+        """Check if paper has an associated PDF file."""
+        return self.pdf_path is not None and self.pdf_path.exists()
+    
+    def similarity_score(self, other: "Paper") -> float:
+        """Calculate similarity score with another paper (0-1)."""
+        score = 0.0
+        weights = {"title": 0.4, "authors": 0.3, "abstract": 0.3}
+        
+        # Title similarity (simple word overlap)
+        title_words1 = set(self.title.lower().split())
+        title_words2 = set(other.title.lower().split())
+        if title_words1 and title_words2:
+            title_overlap = len(title_words1 & title_words2) / len(title_words1 | title_words2)
+            score += weights["title"] * title_overlap
+        
+        # Author similarity
+        authors1 = set(self.authors)
+        authors2 = set(other.authors)
+        if authors1 and authors2:
+            author_overlap = len(authors1 & authors2) / len(authors1 | authors2)
+            score += weights["authors"] * author_overlap
+        
+        # Abstract similarity (simple word overlap)
+        abstract_words1 = set(self.abstract.lower().split())
+        abstract_words2 = set(other.abstract.lower().split())
+        if abstract_words1 and abstract_words2:
+            abstract_overlap = len(abstract_words1 & abstract_words2) / len(abstract_words1 | abstract_words2)
+            score += weights["abstract"] * abstract_overlap
+        
+        return score
+
+
+# Example usage
+if __name__ == "__main__":
+    paper = Paper(
+        title="Deep Learning for Scientific Discovery",
+        authors=["John Doe", "Jane Smith", "Bob Johnson"],
+        abstract="This paper explores the application of deep learning...",
+        source="arxiv",
+        year=2024,
+        arxiv_id="2401.12345",
+        keywords=["deep learning", "scientific computing", "AI"],
+    )
+    
+    print(paper)
+    print("\nBibTeX:")
+    print(paper.to_bibtex())
+    print("\nIdentifier:", paper.get_identifier())