academic-refchecker 2.0.7__py3-none-any.whl

refchecker/scripts/__init__.py

@@ -0,0 +1 @@
+"""Scripts for refchecker package."""

refchecker/scripts/start_vllm_server.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""
+Standalone vLLM server launcher script
+Runs outside of debugger environment to avoid pydevd conflicts
+"""
+
+import sys
+import subprocess
+import os
+import time
+import argparse
+import signal
+
+def start_vllm_server(model_name, port=8000, tensor_parallel_size=1, max_model_len=None, gpu_memory_util=0.9, daemon=False):
+    """Start vLLM server with specified parameters"""
+    
+    # Kill any existing server on the port
+    try:
+        subprocess.run(["pkill", "-f", "vllm.entrypoints.openai.api_server"], 
+                      timeout=10, capture_output=True)
+        time.sleep(2)
+    except:
+        pass
+    
+    # Build command
+    cmd = [
+        sys.executable, "-m", "vllm.entrypoints.openai.api_server",
+        "--model", model_name,
+        "--host", "0.0.0.0",
+        "--port", str(port),
+        "--tensor-parallel-size", str(tensor_parallel_size),
+        "--gpu-memory-utilization", str(gpu_memory_util)
+    ]
+    
+    if max_model_len:
+        cmd.extend(["--max-model-len", str(max_model_len)])
+    
+    print(f"Starting vLLM server: {' '.join(cmd)}")
+    
+    # Create clean environment without debugger variables
+    clean_env = {}
+    for key, value in os.environ.items():
+        if not any(debug_key in key.upper() for debug_key in ['DEBUGPY', 'PYDEVD']):
+            clean_env[key] = value
+    
+    # Remove debugger paths from PYTHONPATH if present
+    if 'PYTHONPATH' in clean_env:
+        pythonpath_parts = clean_env['PYTHONPATH'].split(':')
+        clean_pythonpath = [p for p in pythonpath_parts if 'debugpy' not in p and 'pydevd' not in p]
+        if clean_pythonpath:
+            clean_env['PYTHONPATH'] = ':'.join(clean_pythonpath)
+        else:
+            clean_env.pop('PYTHONPATH', None)
+    
+    # Start server
+    if daemon:
+        # For daemon mode, redirect output to /dev/null to avoid blocking
+        with open(os.devnull, 'w') as devnull:
+            process = subprocess.Popen(
+                cmd,
+                env=clean_env,
+                start_new_session=True,
+                stdout=devnull,
+                stderr=devnull
+            )
+        return process
+    else:
+        # For non-daemon mode, keep stdout/stderr for monitoring with real-time streaming
+        process = subprocess.Popen(
+            cmd,
+            env=clean_env,
+            start_new_session=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=0,  # Unbuffered for real-time output
+            universal_newlines=True
+        )
+        return process
+
+def main():
+    parser = argparse.ArgumentParser(description="Start vLLM server")
+    parser.add_argument("--model", required=True, help="Model name to serve")
+    parser.add_argument("--port", type=int, default=8000, help="Port to serve on")
+    parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallel size")
+    parser.add_argument("--max-model-len", type=int, help="Maximum model length")
+    parser.add_argument("--gpu-memory-util", type=float, default=0.9, help="GPU memory utilization")
+    parser.add_argument("--daemon", action="store_true", help="Run as daemon")
+    
+    args = parser.parse_args()
+    
+    process = start_vllm_server(
+        model_name=args.model,
+        port=args.port,
+        tensor_parallel_size=args.tensor_parallel_size,
+        max_model_len=args.max_model_len,
+        gpu_memory_util=args.gpu_memory_util,
+        daemon=args.daemon
+    )
+    
+    if args.daemon:
+        print(f"vLLM server started as daemon with PID: {process.pid}")
+        print(f"Server URL: http://localhost:{args.port}")
+        return 0
+    else:
+        print(f"vLLM server starting with PID: {process.pid}")
+        print(f"Server URL: http://localhost:{args.port}")
+        print("Press Ctrl+C to stop...")
+        
+        try:
+            # Stream output with immediate flushing
+            for line in process.stdout:
+                print(line.rstrip(), flush=True)
+        except KeyboardInterrupt:
+            print("\nStopping vLLM server...")
+            process.terminate()
+            process.wait()
+            return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

refchecker/services/__init__.py

@@ -0,0 +1,8 @@
+"""
+Services module for ArXiv Reference Checker
+Contains service classes for modular functionality
+"""
+
+from .pdf_processor import PDFProcessor, Paper
+
+__all__ = ['PDFProcessor', 'Paper']

refchecker/services/pdf_processor.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+"""
+PDF Processing Service for ArXiv Reference Checker
+Extracted from core.refchecker to improve modularity
+"""
+
+import os
+import logging
+from typing import Optional, Dict, Any
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class Paper:
+    """Represents a paper with metadata"""
+    title: str
+    authors: list
+    abstract: str = ""
+    year: Optional[int] = None
+    venue: str = ""
+    url: str = ""
+    doi: str = ""
+    arxiv_id: str = ""
+    pdf_path: str = ""
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert paper to dictionary format"""
+        return {
+            'title': self.title,
+            'authors': self.authors,
+            'abstract': self.abstract,
+            'year': self.year,
+            'venue': self.venue,
+            'url': self.url,
+            'doi': self.doi,
+            'arxiv_id': self.arxiv_id,
+            'pdf_path': self.pdf_path
+        }
+
+class PDFProcessor:
+    """Service for processing PDF files and extracting text"""
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        self.config = config or {}
+        self.cache = {}
+        
+    def extract_text_from_pdf(self, pdf_path: str) -> str:
+        """
+        Extract text from PDF file
+        
+        Args:
+            pdf_path: Path to PDF file
+            
+        Returns:
+            Extracted text content
+        """
+        if not os.path.exists(pdf_path):
+            raise FileNotFoundError(f"PDF file not found: {pdf_path}")
+        
+        # Check cache first
+        if pdf_path in self.cache:
+            logger.debug(f"Using cached text for {pdf_path}")
+            return self.cache[pdf_path]
+        
+        try:
+            import pypdf
+            
+            with open(pdf_path, 'rb') as file:
+                pdf_reader = pypdf.PdfReader(file)
+                text = ""
+                failed_pages = []
+                
+                for page_num in range(len(pdf_reader.pages)):
+                    try:
+                        page = pdf_reader.pages[page_num]
+                        page_text = page.extract_text()
+                        if page_text:
+                            text += page_text + "\n"
+                    except TypeError as e:
+                        # Handle pypdf errors like "NumberObject is not iterable"
+                        # which can occur with malformed PDF pages
+                        failed_pages.append(page_num + 1)  # 1-indexed for logging
+                        logger.warning(f"Skipping page {page_num + 1} due to PDF parsing error: {e}")
+                        continue
+                    except Exception as e:
+                        failed_pages.append(page_num + 1)
+                        logger.warning(f"Error extracting text from page {page_num + 1}: {e}")
+                        continue
+                
+                if failed_pages:
+                    logger.warning(f"Failed to extract text from {len(failed_pages)} pages: {failed_pages[:10]}{'...' if len(failed_pages) > 10 else ''}")
+                
+                if not text.strip():
+                    raise ValueError(f"No text could be extracted from any pages of {pdf_path}")
+                
+                # Cache the result
+                self.cache[pdf_path] = text
+                logger.debug(f"Extracted {len(text)} characters from {pdf_path}")
+                return text
+                
+        except ImportError:
+            logger.error("pypdf not installed. Install with: pip install pypdf")
+            raise
+        except Exception as e:
+            logger.error(f"Error extracting text from PDF {pdf_path}: {e}")
+            raise
+    
+    def create_local_file_paper(self, file_path: str, metadata: Optional[Dict[str, Any]] = None) -> Paper:
+        """
+        Create a Paper object from a local file
+        
+        Args:
+            file_path: Path to the file
+            metadata: Optional metadata dictionary
+            
+        Returns:
+            Paper object
+        """
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"File not found: {file_path}")
+        
+        # Extract text if it's a PDF
+        text_content = ""
+        if file_path.lower().endswith('.pdf'):
+            try:
+                text_content = self.extract_text_from_pdf(file_path)
+            except Exception as e:
+                logger.warning(f"Could not extract text from {file_path}: {e}")
+        
+        # Use metadata if provided, otherwise extract from filename
+        if metadata:
+            title = metadata.get('title', os.path.basename(file_path))
+            authors = metadata.get('authors', [])
+            abstract = metadata.get('abstract', '')
+            year = metadata.get('year')
+            venue = metadata.get('venue', '')
+            url = metadata.get('url', '')
+            doi = metadata.get('doi', '')
+            arxiv_id = metadata.get('arxiv_id', '')
+        else:
+            # Basic extraction from filename
+            title = os.path.splitext(os.path.basename(file_path))[0]
+            authors = []
+            abstract = text_content[:500] if text_content else ""  # First 500 chars as abstract
+            year = None
+            venue = ""
+            url = ""
+            doi = ""
+            arxiv_id = ""
+        
+        return Paper(
+            title=title,
+            authors=authors,
+            abstract=abstract,
+            year=year,
+            venue=venue,
+            url=url,
+            doi=doi,
+            arxiv_id=arxiv_id,
+            pdf_path=file_path
+        )
+    
+    def extract_bibliography_from_text(self, text: str) -> str:
+        """
+        Extract bibliography section from text
+        
+        Args:
+            text: Full text content
+            
+        Returns:
+            Bibliography section text
+        """
+        if not text:
+            return ""
+        
+        # Common bibliography section headers
+        bib_headers = [
+            r'\n\s*REFERENCES\s*\n',
+            r'\n\s*References\s*\n',
+            r'\n\s*BIBLIOGRAPHY\s*\n',
+            r'\n\s*Bibliography\s*\n',
+            r'\n\s*WORKS CITED\s*\n',
+            r'\n\s*Works Cited\s*\n'
+        ]
+        
+        import re
+        
+        # Find bibliography section
+        for header in bib_headers:
+            match = re.search(header, text, re.IGNORECASE)
+            if match:
+                # Extract from bibliography header
+                bib_start = match.end()
+                full_bib_text = text[bib_start:].strip()
+                
+                # Find the end of the bibliography section by looking for common section headers
+                # that typically follow references
+                end_markers = [
+                    r'\n\s*APPENDIX\s*[A-Z]?\s*\n',
+                    r'\n\s*Appendix\s*[A-Z]?\s*\n',
+                    r'\n\s*[A-Z]\s+[A-Z]{2,}.*\n',  # Pattern like "A LRE Dataset", "B ADDITIONAL RESULTS"
+                    r'\n\s*[A-Z]\.\d+\s+.*\n',  # Pattern like "A.1 Dataset Details"
+                    r'\nTable\s+\d+:.*\n[A-Z]\s+[A-Z]',  # Table followed by appendix section like "Table 7: ...\nA LRE"
+                    r'\n\s*SUPPLEMENTARY\s+MATERIAL\s*\n',
+                    r'\n\s*Supplementary\s+Material\s*\n',  
+                    r'\n\s*SUPPLEMENTAL\s+MATERIAL\s*\n',
+                    r'\n\s*Supplemental\s+Material\s*\n',
+                    r'\n\s*ACKNOWLEDGMENTS?\s*\n',
+                    r'\n\s*Acknowledgments?\s*\n',
+                    r'\n\s*AUTHOR\s+CONTRIBUTIONS?\s*\n',
+                    r'\n\s*Author\s+Contributions?\s*\n',
+                    r'\n\s*FUNDING\s*\n',
+                    r'\n\s*Funding\s*\n',
+                    r'\n\s*ETHICS\s+STATEMENT\s*\n',
+                    r'\n\s*Ethics\s+Statement\s*\n',
+                    r'\n\s*CONFLICT\s+OF\s+INTEREST\s*\n',
+                    r'\n\s*Conflict\s+of\s+Interest\s*\n',
+                    r'\n\s*DATA\s+AVAILABILITY\s*\n',
+                    r'\n\s*Data\s+Availability\s*\n'
+                ]
+                
+                bib_text = full_bib_text
+                bib_end = len(full_bib_text)
+                
+                # Look for section markers that indicate end of bibliography
+                for end_marker in end_markers:
+                    end_match = re.search(end_marker, full_bib_text, re.IGNORECASE)
+                    if end_match and end_match.start() < bib_end:
+                        bib_end = end_match.start()
+                
+                # If we found an end marker, truncate there
+                if bib_end < len(full_bib_text):
+                    bib_text = full_bib_text[:bib_end].strip()
+                    logger.debug(f"Bibliography section truncated at position {bib_end}")
+                
+                # Also try to detect bibliography end by finding the last numbered reference
+                # Look for the highest numbered reference in the text
+                ref_numbers = re.findall(r'\[(\d+)\]', bib_text)
+                if ref_numbers:
+                    max_ref_num = max(int(num) for num in ref_numbers)
+                    logger.debug(f"Found references up to [{max_ref_num}]")
+                    
+                    # Look for the end of the last numbered reference
+                    last_ref_pattern = rf'\[{max_ref_num}\][^[]*?(?=\n\s*[A-Z]{{2,}}|\n\s*\w+\s*\n\s*[A-Z]|\Z)'
+                    last_ref_match = re.search(last_ref_pattern, bib_text, re.DOTALL)
+                    if last_ref_match:
+                        potential_end = last_ref_match.end()
+                        # Only use this if it's before our section marker end
+                        if potential_end < bib_end:
+                            bib_text = bib_text[:potential_end].strip()
+                            logger.debug(f"Bibliography truncated after reference [{max_ref_num}]")
+                
+                # Final fallback: limit to reasonable length
+                if len(bib_text) > 50000:  # Limit to ~50KB
+                    bib_text = bib_text[:50000]
+                    logger.debug("Bibliography section truncated to 50KB limit")
+                
+                logger.debug(f"Found bibliography section: {len(bib_text)} characters")
+                return bib_text
+        
+        logger.warning("No bibliography section found in text")
+        return ""
+    
+    def clear_cache(self):
+        """Clear the text extraction cache"""
+        self.cache.clear()
+        logger.debug("PDF text cache cleared")

refchecker/utils/__init__.py

@@ -0,0 +1,27 @@
+"""
+Utility functions for text processing, author comparison, mocking, and configuration validation
+"""
+
+from .text_utils import (
+    clean_author_name, clean_title, normalize_text, 
+    clean_conference_markers_from_title,
+    remove_year_from_title
+)
+from .url_utils import extract_arxiv_id_from_url
+from .author_utils import compare_authors, levenshtein_distance, extract_authors_list
+from .mock_objects import (
+    MockPaper, MockReference, MockLLMProvider, MockSemanticScholarAPI, MockArxivAPI,
+    create_mock_config, create_mock_paper, create_mock_reference, 
+    create_mock_bibliography, create_mock_extracted_references
+)
+from .config_validator import ConfigValidator, ValidationResult
+
+__all__ = [
+    "clean_author_name", "clean_title", "normalize_text", 
+    "extract_arxiv_id_from_url", "clean_conference_markers_from_title",
+    "remove_year_from_title", "compare_authors", "levenshtein_distance", 
+    "extract_authors_list", "MockPaper", "MockReference", "MockLLMProvider", 
+    "MockSemanticScholarAPI", "MockArxivAPI", "create_mock_config", 
+    "create_mock_paper", "create_mock_reference", "create_mock_bibliography", 
+    "create_mock_extracted_references", "ConfigValidator", "ValidationResult"
+]