academic-refchecker 2.0.7__py3-none-any.whl

refchecker/config/__init__.py

@@ -0,0 +1 @@
+# Config module

refchecker/config/logging.conf

@@ -0,0 +1,36 @@
+[loggers]
+keys=root,refchecker
+
+[handlers]
+keys=consoleHandler,fileHandler
+
+[formatters]
+keys=simpleFormatter,detailedFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[logger_refchecker]
+level=DEBUG
+handlers=consoleHandler,fileHandler
+qualname=refchecker
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=INFO
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=FileHandler
+level=DEBUG
+formatter=detailedFormatter
+args=('logs/refchecker.log',)
+
+[formatter_simpleFormatter]
+format=%(asctime)s - %(levelname)s - %(message)s
+
+[formatter_detailedFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s

refchecker/config/settings.py

@@ -0,0 +1,170 @@
+"""
+Configuration settings for RefChecker
+"""
+
+import os
+from typing import Dict, Any
+
+# Default configuration
+DEFAULT_CONFIG = {
+    # API Settings
+    "semantic_scholar": {
+        "base_url": "https://api.semanticscholar.org/graph/v1",
+        "rate_limit_delay": 1.0,
+        "max_retries": 3,
+        "timeout": 30,
+    },
+    
+    "arxiv": {
+        "base_url": "https://export.arxiv.org/api/query",
+        "rate_limit_delay": 3.0,
+        "max_retries": 5,
+        "timeout": 30,
+    },
+    
+    # Processing Settings
+    "processing": {
+        "max_papers": 50,
+        "days_back": 365,
+        "batch_size": 100,
+    },
+    
+    # Output Settings
+    "output": {
+        "debug_dir": "debug",
+        "logs_dir": "logs", 
+        "output_dir": "output",
+        "validation_output_dir": "validation_output",
+    },
+    
+    # Database Settings
+    "database": {
+        "default_path": "semantic_scholar_db/semantic_scholar.db",
+        "download_batch_size": 100,
+    },
+    
+    # Text Processing Settings
+    "text_processing": {
+        "similarity_threshold": 0.8,
+        "max_title_similarity": 0.8,
+        "max_author_similarity": 0.7,
+        "year_tolerance": 1,
+    },
+    
+    # LLM Settings
+    "llm": {
+        "enabled": False,
+        "provider": "openai",
+        "fallback_enabled": True,
+        "parallel_chunks": True,  # Enable parallel chunk processing
+        "max_chunk_workers": 4,   # Maximum number of parallel workers for chunk processing
+        "openai": {
+            "model": "gpt-4.1",
+            "max_tokens": 4000,
+            "temperature": 0.1,
+            "timeout": 30,
+        },
+        "anthropic": {
+            "model": "claude-sonnet-4-20250514",
+            "max_tokens": 4000,
+            "temperature": 0.1,
+            "timeout": 30,
+        },
+        "google": {
+            "model": "gemini-2.5-flash",
+            "max_tokens": 4000,
+            "temperature": 0.1,
+            "timeout": 30,
+        },
+        "azure": {
+            "model": "gpt-4o",
+            "max_tokens": 4000,
+            "temperature": 0.1,
+            "timeout": 30,
+        },
+        "vllm": {
+            "model": "meta-llama/Llama-3.1-8B-Instruct",
+            "max_tokens": 4000,
+            "temperature": 0.1,
+            "timeout": 30,
+            "server_url": "http://localhost:8000",
+            "download_path": "./models",
+            "auto_download": True,
+        }
+    }
+}
+
+def get_config() -> Dict[str, Any]:
+    """Get configuration with environment variable overrides"""
+    config = DEFAULT_CONFIG.copy()
+    
+    # Override with environment variables if present
+    if os.getenv("SEMANTIC_SCHOLAR_API_KEY"):
+        config["semantic_scholar"]["api_key"] = os.getenv("SEMANTIC_SCHOLAR_API_KEY")
+    
+    if os.getenv("REFCHECKER_DEBUG"):
+        config["debug"] = os.getenv("REFCHECKER_DEBUG").lower() == "true"
+    
+    if os.getenv("REFCHECKER_OUTPUT_DIR"):
+        config["output"]["output_dir"] = os.getenv("REFCHECKER_OUTPUT_DIR")
+    
+    # LLM configuration from environment variables
+    if os.getenv("REFCHECKER_USE_LLM"):
+        config["llm"]["enabled"] = os.getenv("REFCHECKER_USE_LLM").lower() == "true"
+    
+    if os.getenv("REFCHECKER_LLM_PROVIDER"):
+        config["llm"]["provider"] = os.getenv("REFCHECKER_LLM_PROVIDER")
+    
+    if os.getenv("REFCHECKER_LLM_FALLBACK_ON_ERROR"):
+        config["llm"]["fallback_enabled"] = os.getenv("REFCHECKER_LLM_FALLBACK_ON_ERROR").lower() == "true"
+    
+    # Provider-specific API keys - check native variables first, then fallback to refchecker-prefixed
+    if os.getenv("OPENAI_API_KEY") or os.getenv("REFCHECKER_OPENAI_API_KEY"):
+        config["llm"]["openai"]["api_key"] = os.getenv("OPENAI_API_KEY") or os.getenv("REFCHECKER_OPENAI_API_KEY")
+    
+    if os.getenv("ANTHROPIC_API_KEY") or os.getenv("REFCHECKER_ANTHROPIC_API_KEY"):
+        config["llm"]["anthropic"]["api_key"] = os.getenv("ANTHROPIC_API_KEY") or os.getenv("REFCHECKER_ANTHROPIC_API_KEY")
+    
+    if os.getenv("GOOGLE_API_KEY") or os.getenv("REFCHECKER_GOOGLE_API_KEY"):
+        config["llm"]["google"]["api_key"] = os.getenv("GOOGLE_API_KEY") or os.getenv("REFCHECKER_GOOGLE_API_KEY")
+    
+    if os.getenv("AZURE_OPENAI_API_KEY") or os.getenv("REFCHECKER_AZURE_API_KEY"):
+        config["llm"]["azure"]["api_key"] = os.getenv("AZURE_OPENAI_API_KEY") or os.getenv("REFCHECKER_AZURE_API_KEY")
+    
+    if os.getenv("AZURE_OPENAI_ENDPOINT") or os.getenv("REFCHECKER_AZURE_ENDPOINT"):
+        config["llm"]["azure"]["endpoint"] = os.getenv("AZURE_OPENAI_ENDPOINT") or os.getenv("REFCHECKER_AZURE_ENDPOINT")
+    
+    # vLLM configuration
+    if os.getenv("REFCHECKER_VLLM_SERVER_URL"):
+        config["llm"]["vllm"]["server_url"] = os.getenv("REFCHECKER_VLLM_SERVER_URL")
+    
+    if os.getenv("REFCHECKER_VLLM_DOWNLOAD_PATH"):
+        config["llm"]["vllm"]["download_path"] = os.getenv("REFCHECKER_VLLM_DOWNLOAD_PATH")
+    
+    if os.getenv("REFCHECKER_VLLM_AUTO_DOWNLOAD"):
+        config["llm"]["vllm"]["auto_download"] = os.getenv("REFCHECKER_VLLM_AUTO_DOWNLOAD").lower() == "true"
+    
+    # Parallel processing configuration
+    if os.getenv("REFCHECKER_LLM_PARALLEL_CHUNKS"):
+        config["llm"]["parallel_chunks"] = os.getenv("REFCHECKER_LLM_PARALLEL_CHUNKS").lower() == "true"
+    
+    if os.getenv("REFCHECKER_LLM_MAX_CHUNK_WORKERS"):
+        config["llm"]["max_chunk_workers"] = int(os.getenv("REFCHECKER_LLM_MAX_CHUNK_WORKERS"))
+    
+    # Model configuration
+    if os.getenv("REFCHECKER_LLM_MODEL"):
+        provider = config["llm"]["provider"]
+        if provider in config["llm"]:
+            config["llm"][provider]["model"] = os.getenv("REFCHECKER_LLM_MODEL")
+    
+    if os.getenv("REFCHECKER_LLM_MAX_TOKENS"):
+        provider = config["llm"]["provider"]
+        if provider in config["llm"]:
+            config["llm"][provider]["max_tokens"] = int(os.getenv("REFCHECKER_LLM_MAX_TOKENS"))
+    
+    if os.getenv("REFCHECKER_LLM_TEMPERATURE"):
+        provider = config["llm"]["provider"]
+        if provider in config["llm"]:
+            config["llm"][provider]["temperature"] = float(os.getenv("REFCHECKER_LLM_TEMPERATURE"))
+    
+    return config

refchecker/core/__init__.py

@@ -0,0 +1,7 @@
+"""
+Core functionality for RefChecker
+"""
+
+from .refchecker import ArxivReferenceChecker, setup_logging
+
+__all__ = ["ArxivReferenceChecker", "setup_logging"]

refchecker/core/db_connection_pool.py

@@ -0,0 +1,141 @@
+"""
+Thread-safe database connection pool for parallel reference checking.
+
+This module provides a connection pool that allows multiple worker threads
+to safely access SQLite databases in parallel by maintaining per-thread connections.
+"""
+
+import threading
+import sqlite3
+import logging
+from typing import Dict, Optional
+from contextlib import contextmanager
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseConnectionPool:
+    """
+    Thread-safe SQLite connection pool.
+    
+    Each thread gets its own database connection to avoid SQLite's
+    "objects created in a thread can only be used in that same thread" restriction.
+    """
+    
+    def __init__(self, db_path: str):
+        """
+        Initialize the connection pool.
+        
+        Args:
+            db_path: Path to the SQLite database file
+        """
+        self.db_path = db_path
+        self._connections: Dict[int, sqlite3.Connection] = {}
+        self._lock = threading.Lock()
+    
+    def get_connection(self) -> sqlite3.Connection:
+        """
+        Get a database connection for the current thread.
+        
+        Returns:
+            SQLite connection object for the current thread
+        """
+        thread_id = threading.get_ident()
+        
+        with self._lock:
+            if thread_id not in self._connections:
+                logger.debug(f"Creating new database connection for thread {thread_id}")
+                conn = sqlite3.connect(self.db_path)
+                conn.row_factory = sqlite3.Row  # Return rows as dictionaries
+                self._connections[thread_id] = conn
+            
+            return self._connections[thread_id]
+    
+    @contextmanager
+    def connection(self):
+        """
+        Context manager for database connections.
+        
+        Usage:
+            with pool.connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT ...")
+        """
+        conn = self.get_connection()
+        try:
+            yield conn
+        except Exception:
+            # Rollback on error
+            conn.rollback()
+            raise
+        else:
+            # Commit on success
+            conn.commit()
+    
+    def close_all(self):
+        """Close all connections in the pool."""
+        with self._lock:
+            # Don't try to close connections from different threads - SQLite doesn't allow it
+            # The connections will be cleaned up when the worker threads exit
+            logger.debug(f"Clearing connection pool (connections will close when threads exit)")
+            self._connections.clear()
+    
+    def close_current_thread(self):
+        """Close the connection for the current thread."""
+        thread_id = threading.get_ident()
+        
+        with self._lock:
+            if thread_id in self._connections:
+                try:
+                    self._connections[thread_id].close()
+                    del self._connections[thread_id]
+                    logger.debug(f"Closed database connection for current thread {thread_id}")
+                except Exception as e:
+                    logger.error(f"Error closing connection for thread {thread_id}: {e}")
+
+
+class ThreadSafeLocalChecker:
+    """
+    Thread-safe wrapper for LocalNonArxivReferenceChecker that uses connection pooling.
+    """
+    
+    def __init__(self, db_path: str):
+        """
+        Initialize the thread-safe checker.
+        
+        Args:
+            db_path: Path to the SQLite database file
+        """
+        self.db_path = db_path
+        self.connection_pool = DatabaseConnectionPool(db_path)
+    
+    def verify_reference(self, reference):
+        """
+        Thread-safe reference verification.
+        
+        Args:
+            reference: Reference dictionary to verify
+            
+        Returns:
+            Tuple of (verified_data, errors, paper_url)
+        """
+        # Import here to avoid circular imports
+        from checkers.local_semantic_scholar import LocalNonArxivReferenceChecker
+        
+        # Get thread-local connection
+        conn = self.connection_pool.get_connection()
+        
+        # Create a properly initialized checker instance with thread-local connection
+        checker = LocalNonArxivReferenceChecker.__new__(LocalNonArxivReferenceChecker)
+        
+        # Initialize the essential attributes (same as __init__ does)
+        checker.db_path = self.db_path
+        checker.conn = conn
+        # The connection should already have row_factory set from the pool
+        
+        # Call the verification method
+        return checker.verify_reference(reference)
+    
+    def close(self):
+        """Close all database connections."""
+        self.connection_pool.close_all()