academic-refchecker 1.2.65__py3-none-any.whl → 1.2.66__py3-none-any.whl

backend/thumbnail.py

@@ -0,0 +1,517 @@
+"""
+Thumbnail generation utilities for PDF and web page previews.
+
+Uses PyMuPDF (fitz) to extract the first page of PDFs as thumbnails.
+Thumbnails are cached on disk to avoid regeneration.
+"""
+import os
+import hashlib
+import logging
+import tempfile
+from pathlib import Path
+from typing import Optional
+import asyncio
+
+logger = logging.getLogger(__name__)
+
+# Default thumbnail cache directory
+THUMBNAIL_CACHE_DIR = Path(tempfile.gettempdir()) / "refchecker_thumbnails"
+
+# Thumbnail settings
+THUMBNAIL_WIDTH = 200  # Target width in pixels for small thumbnails
+THUMBNAIL_DPI = 150  # Higher DPI for sharper text rendering
+
+# Preview settings (larger image for overlay view)
+PREVIEW_WIDTH = 1600  # Target width in pixels for preview/overlay
+
+
+def get_thumbnail_cache_path(source_identifier: str, check_id: Optional[int] = None) -> Path:
+    """
+    Get the cache path for a thumbnail.
+    
+    Args:
+        source_identifier: A unique identifier for the source (URL, file path, or hash)
+        check_id: Optional check ID for more unique naming
+        
+    Returns:
+        Path to the thumbnail file (may not exist yet)
+    """
+    # Create cache directory if it doesn't exist
+    THUMBNAIL_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    
+    # Create a hash of the source for the filename
+    source_hash = hashlib.md5(source_identifier.encode()).hexdigest()[:12]
+    
+    if check_id:
+        filename = f"thumb_{check_id}_{source_hash}.png"
+    else:
+        filename = f"thumb_{source_hash}.png"
+    
+    return THUMBNAIL_CACHE_DIR / filename
+
+
+def get_preview_cache_path(source_identifier: str, check_id: Optional[int] = None) -> Path:
+    """
+    Get the cache path for a preview (larger image for overlay).
+    
+    Args:
+        source_identifier: A unique identifier for the source (URL, file path, or hash)
+        check_id: Optional check ID for more unique naming
+        
+    Returns:
+        Path to the preview file (may not exist yet)
+    """
+    # Create cache directory if it doesn't exist
+    THUMBNAIL_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    
+    # Create a hash of the source for the filename
+    source_hash = hashlib.md5(source_identifier.encode()).hexdigest()[:12]
+    
+    if check_id:
+        filename = f"preview_{check_id}_{source_hash}.png"
+    else:
+        filename = f"preview_{source_hash}.png"
+    
+    return THUMBNAIL_CACHE_DIR / filename
+
+
+def generate_pdf_thumbnail(pdf_path: str, output_path: Optional[str] = None) -> Optional[str]:
+    """
+    Generate a thumbnail from the first page of a PDF.
+    
+    Args:
+        pdf_path: Path to the PDF file
+        output_path: Optional path for the output thumbnail. If not provided,
+                     uses the cache directory.
+                     
+    Returns:
+        Path to the generated thumbnail, or None if generation failed
+    """
+    try:
+        import fitz  # PyMuPDF
+        
+        if not os.path.exists(pdf_path):
+            logger.error(f"PDF file not found: {pdf_path}")
+            return None
+        
+        # Determine output path
+        if output_path is None:
+            output_path = str(get_thumbnail_cache_path(pdf_path))
+        
+        # Check if thumbnail already exists
+        if os.path.exists(output_path):
+            logger.debug(f"Thumbnail already exists: {output_path}")
+            return output_path
+        
+        # Open the PDF
+        doc = fitz.open(pdf_path)
+        
+        if len(doc) == 0:
+            logger.warning(f"PDF has no pages: {pdf_path}")
+            doc.close()
+            return None
+        
+        # Get the first page
+        page = doc[0]
+        
+        # Calculate zoom factor to get desired width
+        page_width = page.rect.width
+        zoom = THUMBNAIL_WIDTH / page_width
+        
+        # Create transformation matrix
+        mat = fitz.Matrix(zoom, zoom)
+        
+        # Render page to pixmap with higher quality
+        pix = page.get_pixmap(matrix=mat, alpha=False)
+        
+        # Enhance contrast to make text darker/more readable
+        try:
+            from PIL import Image, ImageEnhance
+            import io
+            
+            # Convert pixmap to PIL Image
+            img_data = pix.tobytes("png")
+            img = Image.open(io.BytesIO(img_data))
+            
+            # Increase contrast (1.3 = 30% more contrast)
+            enhancer = ImageEnhance.Contrast(img)
+            img = enhancer.enhance(1.3)
+            
+            # Slightly increase sharpness
+            enhancer = ImageEnhance.Sharpness(img)
+            img = enhancer.enhance(1.2)
+            
+            # Save enhanced image
+            img.save(output_path, "PNG")
+        except ImportError:
+            # Fallback: save without enhancement if PIL not available
+            pix.save(output_path)
+        
+        doc.close()
+        
+        logger.info(f"Generated thumbnail: {output_path} ({pix.width}x{pix.height})")
+        return output_path
+        
+    except ImportError:
+        logger.error("PyMuPDF (fitz) is not installed. Install with: pip install pymupdf")
+        return None
+    except Exception as e:
+        logger.error(f"Error generating PDF thumbnail: {e}")
+        return None
+
+
+async def generate_pdf_thumbnail_async(pdf_path: str, output_path: Optional[str] = None) -> Optional[str]:
+    """
+    Async wrapper for PDF thumbnail generation.
+    
+    Args:
+        pdf_path: Path to the PDF file
+        output_path: Optional path for the output thumbnail
+        
+    Returns:
+        Path to the generated thumbnail, or None if generation failed
+    """
+    return await asyncio.to_thread(generate_pdf_thumbnail, pdf_path, output_path)
+
+
+def generate_pdf_preview(pdf_path: str, output_path: Optional[str] = None) -> Optional[str]:
+    """
+    Generate a high-resolution preview from the first page of a PDF.
+    
+    Args:
+        pdf_path: Path to the PDF file
+        output_path: Optional path for the output preview. If not provided,
+                     uses the cache directory.
+                     
+    Returns:
+        Path to the generated preview, or None if generation failed
+    """
+    try:
+        import fitz  # PyMuPDF
+        
+        if not os.path.exists(pdf_path):
+            logger.error(f"PDF file not found: {pdf_path}")
+            return None
+        
+        # Determine output path
+        if output_path is None:
+            output_path = str(get_preview_cache_path(pdf_path))
+        
+        # Check if preview already exists
+        if os.path.exists(output_path):
+            logger.debug(f"Preview already exists: {output_path}")
+            return output_path
+        
+        # Open the PDF
+        doc = fitz.open(pdf_path)
+        
+        if len(doc) == 0:
+            logger.warning(f"PDF has no pages: {pdf_path}")
+            doc.close()
+            return None
+        
+        # Get the first page
+        page = doc[0]
+        
+        # Calculate zoom factor to get desired width (larger for preview)
+        page_width = page.rect.width
+        zoom = PREVIEW_WIDTH / page_width
+        
+        # Create transformation matrix
+        mat = fitz.Matrix(zoom, zoom)
+        
+        # Render page to pixmap with higher quality
+        pix = page.get_pixmap(matrix=mat, alpha=False)
+        
+        # Enhance contrast to make text darker/more readable
+        try:
+            from PIL import Image, ImageEnhance
+            import io
+            
+            # Convert pixmap to PIL Image
+            img_data = pix.tobytes("png")
+            img = Image.open(io.BytesIO(img_data))
+            
+            # Increase contrast (1.2 = 20% more contrast)
+            enhancer = ImageEnhance.Contrast(img)
+            img = enhancer.enhance(1.2)
+            
+            # Slightly increase sharpness
+            enhancer = ImageEnhance.Sharpness(img)
+            img = enhancer.enhance(1.1)
+            
+            # Save enhanced image
+            img.save(output_path, "PNG", optimize=True)
+        except ImportError:
+            # Fallback: save without enhancement if PIL not available
+            pix.save(output_path)
+        
+        doc.close()
+        
+        logger.info(f"Generated preview: {output_path} ({pix.width}x{pix.height})")
+        return output_path
+        
+    except ImportError:
+        logger.error("PyMuPDF (fitz) is not installed. Install with: pip install pymupdf")
+        return None
+    except Exception as e:
+        logger.error(f"Error generating PDF preview: {e}")
+        return None
+
+
+async def generate_pdf_preview_async(pdf_path: str, output_path: Optional[str] = None) -> Optional[str]:
+    """
+    Async wrapper for PDF preview generation.
+    
+    Args:
+        pdf_path: Path to the PDF file
+        output_path: Optional path for the output preview
+        
+    Returns:
+        Path to the generated preview, or None if generation failed
+    """
+    return await asyncio.to_thread(generate_pdf_preview, pdf_path, output_path)
+
+
+def generate_arxiv_thumbnail(arxiv_id: str, check_id: Optional[int] = None) -> Optional[str]:
+    """
+    Generate a thumbnail for an ArXiv paper.
+    
+    Downloads the PDF and generates a thumbnail of the first page.
+    
+    Args:
+        arxiv_id: ArXiv paper ID (e.g., "2311.12022")
+        check_id: Optional check ID for cache naming
+        
+    Returns:
+        Path to the generated thumbnail, or None if generation failed
+    """
+    try:
+        import arxiv as arxiv_lib
+        
+        # Check if thumbnail already exists
+        output_path = get_thumbnail_cache_path(f"arxiv_{arxiv_id}", check_id)
+        if output_path.exists():
+            logger.debug(f"ArXiv thumbnail already exists: {output_path}")
+            return str(output_path)
+        
+        # Download the PDF to a temporary location
+        pdf_dir = Path(tempfile.gettempdir()) / "refchecker_pdfs"
+        pdf_dir.mkdir(parents=True, exist_ok=True)
+        pdf_path = pdf_dir / f"arxiv_{arxiv_id}.pdf"
+        
+        # Check if PDF is already downloaded
+        if not pdf_path.exists():
+            logger.info(f"Downloading ArXiv PDF: {arxiv_id}")
+            search = arxiv_lib.Search(id_list=[arxiv_id])
+            paper = next(search.results())
+            paper.download_pdf(filename=str(pdf_path))
+        
+        # Generate thumbnail from the PDF
+        return generate_pdf_thumbnail(str(pdf_path), str(output_path))
+        
+    except Exception as e:
+        logger.error(f"Error generating ArXiv thumbnail: {e}")
+        return None
+
+
+async def generate_arxiv_thumbnail_async(arxiv_id: str, check_id: Optional[int] = None) -> Optional[str]:
+    """
+    Async wrapper for ArXiv thumbnail generation.
+    
+    Args:
+        arxiv_id: ArXiv paper ID
+        check_id: Optional check ID for cache naming
+        
+    Returns:
+        Path to the generated thumbnail, or None if generation failed
+    """
+    return await asyncio.to_thread(generate_arxiv_thumbnail, arxiv_id, check_id)
+
+
+def generate_arxiv_preview(arxiv_id: str, check_id: Optional[int] = None) -> Optional[str]:
+    """
+    Generate a high-resolution preview for an ArXiv paper.
+    
+    Downloads the PDF and generates a preview of the first page.
+    
+    Args:
+        arxiv_id: ArXiv paper ID (e.g., "2311.12022")
+        check_id: Optional check ID for cache naming
+        
+    Returns:
+        Path to the generated preview, or None if generation failed
+    """
+    try:
+        import arxiv as arxiv_lib
+        
+        # Check if preview already exists
+        output_path = get_preview_cache_path(f"arxiv_{arxiv_id}", check_id)
+        if output_path.exists():
+            logger.debug(f"ArXiv preview already exists: {output_path}")
+            return str(output_path)
+        
+        # Download the PDF to a temporary location
+        pdf_dir = Path(tempfile.gettempdir()) / "refchecker_pdfs"
+        pdf_dir.mkdir(parents=True, exist_ok=True)
+        pdf_path = pdf_dir / f"arxiv_{arxiv_id}.pdf"
+        
+        # Check if PDF is already downloaded
+        if not pdf_path.exists():
+            logger.info(f"Downloading ArXiv PDF: {arxiv_id}")
+            search = arxiv_lib.Search(id_list=[arxiv_id])
+            paper = next(search.results())
+            paper.download_pdf(filename=str(pdf_path))
+        
+        # Generate preview from the PDF
+        return generate_pdf_preview(str(pdf_path), str(output_path))
+        
+    except Exception as e:
+        logger.error(f"Error generating ArXiv preview: {e}")
+        return None
+
+
+async def generate_arxiv_preview_async(arxiv_id: str, check_id: Optional[int] = None) -> Optional[str]:
+    """
+    Async wrapper for ArXiv preview generation.
+    
+    Args:
+        arxiv_id: ArXiv paper ID
+        check_id: Optional check ID for cache naming
+        
+    Returns:
+        Path to the generated preview, or None if generation failed
+    """
+    return await asyncio.to_thread(generate_arxiv_preview, arxiv_id, check_id)
+
+
+def get_text_thumbnail(check_id: int, text_preview: str = "", text_file_path: str = "") -> Optional[str]:
+    """
+    Generate a thumbnail for pasted text showing actual content.
+    
+    Creates an image with the first few lines of the text content.
+    
+    Args:
+        check_id: Check ID for naming
+        text_preview: Optional first few lines of text to display
+        text_file_path: Optional path to the text file to read content from
+        
+    Returns:
+        Path to the generated thumbnail, or None if generation failed
+    """
+    try:
+        import fitz
+        
+        output_path = get_thumbnail_cache_path(f"text_{check_id}", check_id)
+        
+        if output_path.exists():
+            return str(output_path)
+        
+        # Try to read text content from file
+        text_content = text_preview
+        if text_file_path and os.path.exists(text_file_path):
+            try:
+                with open(text_file_path, 'r', encoding='utf-8') as f:
+                    text_content = f.read()
+            except Exception as e:
+                logger.warning(f"Could not read text file: {e}")
+        
+        # Create a document-like image with actual text content
+        doc = fitz.open()
+        page = doc.new_page(width=THUMBNAIL_WIDTH, height=int(THUMBNAIL_WIDTH * 1.4))
+        
+        # Fill with white/off-white background
+        page.draw_rect(page.rect, color=(0.95, 0.95, 0.95), fill=(0.99, 0.99, 0.99))
+        
+        # Draw border
+        page.draw_rect(page.rect, color=(0.8, 0.8, 0.8), width=1)
+        
+        # Draw actual text content if available
+        margin = 10
+        if text_content:
+            # Create a text box for the content
+            text_rect = fitz.Rect(margin, margin, THUMBNAIL_WIDTH - margin, int(THUMBNAIL_WIDTH * 1.4) - margin)
+            
+            # Truncate to first ~500 chars for thumbnail
+            display_text = text_content[:500]
+            if len(text_content) > 500:
+                display_text += "..."
+            
+            # Insert text with small font
+            page.insert_textbox(
+                text_rect,
+                display_text,
+                fontsize=6,
+                color=(0.2, 0.2, 0.2),
+                fontname="helv"
+            )
+        else:
+            # Fallback: Draw placeholder lines
+            line_height = 12
+            y = margin + 30
+            
+            # Draw a "T" icon at top
+            text_rect = fitz.Rect(margin, margin, margin + 30, margin + 25)
+            page.insert_textbox(text_rect, "T", fontsize=20, color=(0.4, 0.4, 0.6))
+            
+            for i in range(10):
+                line_width = THUMBNAIL_WIDTH - 2 * margin
+                if i % 3 == 2:
+                    line_width = line_width * 0.7
+                
+                page.draw_line(
+                    fitz.Point(margin, y),
+                    fitz.Point(margin + line_width, y),
+                    color=(0.7, 0.7, 0.7),
+                    width=2
+                )
+                y += line_height
+        
+        # Render to pixmap and save
+        pix = page.get_pixmap(alpha=False)
+        pix.save(str(output_path))
+        doc.close()
+        
+        logger.info(f"Generated text thumbnail: {output_path}")
+        return str(output_path)
+        
+    except ImportError:
+        logger.error("PyMuPDF (fitz) is not installed")
+        return None
+    except Exception as e:
+        logger.error(f"Error generating text thumbnail: {e}")
+        return None
+
+
+async def get_text_thumbnail_async(check_id: int, text_preview: str = "", text_file_path: str = "") -> Optional[str]:
+    """Async wrapper for text thumbnail generation."""
+    return await asyncio.to_thread(get_text_thumbnail, check_id, text_preview, text_file_path)
+
+
+def cleanup_old_thumbnails(max_age_days: int = 30):
+    """
+    Clean up old thumbnails from the cache.
+    
+    Args:
+        max_age_days: Maximum age in days before thumbnails are deleted
+    """
+    try:
+        import time
+        
+        if not THUMBNAIL_CACHE_DIR.exists():
+            return
+        
+        max_age_seconds = max_age_days * 24 * 60 * 60
+        current_time = time.time()
+        
+        for thumb_path in THUMBNAIL_CACHE_DIR.glob("thumb_*.png"):
+            try:
+                file_age = current_time - thumb_path.stat().st_mtime
+                if file_age > max_age_seconds:
+                    thumb_path.unlink()
+                    logger.debug(f"Deleted old thumbnail: {thumb_path}")
+            except Exception as e:
+                logger.warning(f"Error deleting thumbnail {thumb_path}: {e}")
+                
+    except Exception as e:
+        logger.error(f"Error cleaning up thumbnails: {e}")

backend/websocket_manager.py

@@ -0,0 +1,104 @@
+"""
+WebSocket connection manager for real-time updates
+"""
+import asyncio
+import json
+from typing import Dict, Set
+from fastapi import WebSocket
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ConnectionManager:
+    """Manages WebSocket connections for real-time updates"""
+
+    def __init__(self):
+        # Map of session_id -> set of websocket connections
+        self.active_connections: Dict[str, Set[WebSocket]] = {}
+
+    async def connect(self, websocket: WebSocket, session_id: str):
+        """Accept a new WebSocket connection"""
+        await websocket.accept()
+        if session_id not in self.active_connections:
+            self.active_connections[session_id] = set()
+        self.active_connections[session_id].add(websocket)
+        logger.info(f"WebSocket connected for session: {session_id}")
+
+    def disconnect(self, websocket: WebSocket, session_id: str):
+        """Remove a WebSocket connection"""
+        if session_id in self.active_connections:
+            self.active_connections[session_id].discard(websocket)
+            if not self.active_connections[session_id]:
+                del self.active_connections[session_id]
+        logger.info(f"WebSocket disconnected for session: {session_id}")
+
+    async def send_message(self, session_id: str, message_type: str, data: dict):
+        """Send a message to all connections for a session"""
+        if session_id not in self.active_connections:
+            logger.debug(f"No active connections for session: {session_id}")
+            return
+
+        # Flatten structure: frontend expects {type, session_id, ...data}
+        # Include session_id so the client can ignore stale messages from old sessions
+        message = {"type": message_type, "session_id": session_id, **data}
+        message_json = json.dumps(message)
+        
+        logger.debug(f"Sending {message_type} to session {session_id}: {message_json[:200]}...")
+
+        # Send to all connections for this session
+        disconnected = set()
+        for websocket in self.active_connections[session_id]:
+            try:
+                await websocket.send_text(message_json)
+            except Exception as e:
+                logger.error(f"Error sending message to websocket: {e}")
+                disconnected.add(websocket)
+
+        # Clean up disconnected websockets
+        for ws in disconnected:
+            self.disconnect(ws, session_id)
+
+    async def broadcast_started(self, session_id: str, paper_title: str, paper_source: str):
+        """Broadcast that checking has started"""
+        await self.send_message(session_id, "started", {
+            "paper_title": paper_title,
+            "paper_source": paper_source
+        })
+
+    async def broadcast_extracting(self, session_id: str):
+        """Broadcast that references are being extracted"""
+        await self.send_message(session_id, "extracting", {
+            "message": "Extracting references from paper..."
+        })
+
+    async def broadcast_progress(self, session_id: str, current: int, total: int):
+        """Broadcast progress update"""
+        await self.send_message(session_id, "progress", {
+            "current": current,
+            "total": total,
+            "percent": round((current / total * 100) if total > 0 else 0, 1)
+        })
+
+    async def broadcast_reference_result(self, session_id: str, reference_data: dict):
+        """Broadcast a reference checking result"""
+        await self.send_message(session_id, "reference_result", reference_data)
+
+    async def broadcast_summary_update(self, session_id: str, summary: dict):
+        """Broadcast updated summary statistics"""
+        await self.send_message(session_id, "summary_update", summary)
+
+    async def broadcast_completed(self, session_id: str, final_summary: dict):
+        """Broadcast that checking is complete"""
+        await self.send_message(session_id, "completed", final_summary)
+
+    async def broadcast_error(self, session_id: str, error_message: str, error_details: str = ""):
+        """Broadcast an error"""
+        await self.send_message(session_id, "error", {
+            "message": error_message,
+            "details": error_details
+        })
+
+
+# Global connection manager instance
+manager = ConnectionManager()

refchecker/__version__.py

@@ -1,5 +1,5 @@
 """Version information for RefChecker."""
 
-__version__ = "1.2.65"
+__version__ = "1.2.66"
 
-__version__ = "1.2.65"
+__version__ = "1.2.66"

refchecker/checkers/crossref.py

@@ -487,13 +487,22 @@ class CrossRefReferenceChecker:
         work_doi = work_data.get('DOI')
         if doi and work_doi:
             # Compare DOIs using the proper comparison function
-            from refchecker.utils.doi_utils import compare_dois
+            from refchecker.utils.doi_utils import compare_dois, validate_doi_resolves
             if not compare_dois(doi, work_doi):
-                errors.append({
-                    'error_type': 'doi',
-                    'error_details': format_doi_mismatch(doi, work_doi),
-                    'ref_doi_correct': work_doi
-                })
+                # If cited DOI resolves, it's likely a valid alternate DOI (e.g., arXiv vs conference)
+                # Treat as warning instead of error
+                if validate_doi_resolves(doi):
+                    errors.append({
+                        'warning_type': 'doi',
+                        'warning_details': format_doi_mismatch(doi, work_doi),
+                        'ref_doi_correct': work_doi
+                    })
+                else:
+                    errors.append({
+                        'error_type': 'doi',
+                        'error_details': format_doi_mismatch(doi, work_doi),
+                        'ref_doi_correct': work_doi
+                    })
         
         # Extract URL from work data
         work_url = self.extract_url_from_work(work_data)

refchecker/checkers/enhanced_hybrid_checker.py

@@ -256,6 +256,21 @@ class EnhancedHybridReferenceChecker:
         Returns:
             Tuple of (verified_data, errors, url)
         """
+        # Check if this is a URL-only reference (should skip verification)
+        authors = reference.get('authors', [])
+        if authors and "URL Reference" in authors:
+            # Skip verification for URL references - they're just links, not papers
+            logger.debug("Enhanced Hybrid: Skipping verification for URL reference")
+            return None, [], reference.get('cited_url') or reference.get('url')
+        
+        # Also check if it looks like a URL-only reference (no title, just URL)
+        title = reference.get('title', '').strip()
+        cited_url = reference.get('cited_url') or reference.get('url')
+        if not title and cited_url:
+            # This is a URL-only reference without a title
+            logger.debug(f"Enhanced Hybrid: Skipping verification for URL-only reference: {cited_url}")
+            return None, [], cited_url
+        
         # Track all APIs that failed and could be retried
         failed_apis = []
         
@@ -533,10 +548,9 @@ class EnhancedHybridReferenceChecker:
         if self.semantic_scholar:
             return self.semantic_scholar.normalize_paper_title(title)
         else:
-            # Basic normalization if Semantic Scholar is not available
-            import re
-            title = re.sub(r'\s+', ' ', title.strip().lower())
-            return re.sub(r'[^\w\s]', '', title)
+            # Use the centralized normalization function from text_utils
+            from refchecker.utils.text_utils import normalize_paper_title as normalize_title
+            return normalize_title(title)
     
     def compare_authors(self, cited_authors: List[str], correct_authors: List[Any]) -> Tuple[bool, str]:
         """