academic-refchecker 2.0.7__py3-none-any.whl

refchecker/utils/unicode_utils.py

@@ -0,0 +1,335 @@
+#!/usr/bin/env python3
+"""
+Unicode parsing utility functions for handling text processing in pipelines.
+Provides robust Unicode support for various text processing scenarios.
+"""
+
+import unicodedata
+import re
+import json
+import codecs
+from typing import Any, Dict, List, Optional, Union
+
+
+def normalize_unicode_text(text: str, form: str = 'NFKC') -> str:
+    """
+    Normalize Unicode text to handle various Unicode forms.
+    
+    Args:
+        text: Input text to normalize
+        form: Unicode normalization form ('NFC', 'NFKC', 'NFD', 'NFKD')
+    
+    Returns:
+        Normalized Unicode text
+    """
+    if not isinstance(text, str):
+        text = str(text)
+    
+    try:
+        # Normalize Unicode characters
+        normalized = unicodedata.normalize(form, text)
+        return normalized
+    except Exception as e:
+        print(f"Warning: Unicode normalization failed: {e}")
+        return text
+
+
+def clean_unicode_control_chars(text: str) -> str:
+    """
+    Remove or replace problematic Unicode control characters.
+    
+    Args:
+        text: Input text to clean
+        
+    Returns:
+        Cleaned text with control characters handled
+    """
+    if not isinstance(text, str):
+        text = str(text)
+    
+    # Remove common problematic control characters
+    # Keep essential whitespace characters (space, tab, newline, carriage return)
+    control_char_pattern = re.compile(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]')
+    cleaned = control_char_pattern.sub('', text)
+    
+    # Replace non-breaking spaces and similar with regular spaces
+    cleaned = re.sub(r'[\u00A0\u2000-\u200B\u2028\u2029\u202F\u205F\u3000]', ' ', cleaned)
+    
+    return cleaned
+
+
+def safe_encode_decode(text: str, encoding: str = 'utf-8', errors: str = 'replace') -> str:
+    """
+    Safely encode and decode text to handle encoding issues.
+    
+    Args:
+        text: Input text
+        encoding: Target encoding (default: utf-8)
+        errors: Error handling strategy ('ignore', 'replace', 'strict')
+        
+    Returns:
+        Safely encoded/decoded text
+    """
+    if not isinstance(text, str):
+        text = str(text)
+    
+    try:
+        # Encode then decode to handle any encoding issues
+        encoded = text.encode(encoding, errors=errors)
+        decoded = encoded.decode(encoding, errors=errors)
+        return decoded
+    except Exception as e:
+        print(f"Warning: Encoding/decoding failed: {e}")
+        return text
+
+
+def fix_mojibake(text: str) -> str:
+    """
+    Attempt to fix common mojibake (character encoding corruption) issues.
+    
+    Args:
+        text: Input text that may contain mojibake
+        
+    Returns:
+        Text with mojibake corrections attempted
+    """
+    if not isinstance(text, str):
+        text = str(text)
+    
+    # Common mojibake patterns and their fixes
+    mojibake_fixes = {
+        # UTF-8 interpreted as Latin-1 then re-encoded
+        'á': 'á',
+        'é': 'é',
+        'í': 'í',
+        'ó': 'ó',
+        'ú': 'ú',
+        'ñ': 'ñ',
+        'ü': 'ü',
+        'Â': '',  # Often spurious  characters
+        '’': "'",  # Right single quotation mark
+        '“': '"',  # Left double quotation mark
+        'â€': '"',   # Right double quotation mark
+        'â€"': '—',  # Em dash
+        'â€"': '–',  # En dash
+    }
+    
+    for broken, fixed in mojibake_fixes.items():
+        text = text.replace(broken, fixed)
+    
+    return text
+
+
+def safe_json_loads(text: str) -> Any:
+    """
+    Safely load JSON with Unicode handling.
+    
+    Args:
+        text: JSON string to parse
+        
+    Returns:
+        Parsed JSON object, or None if parsing fails
+    """
+    if not isinstance(text, str):
+        text = str(text)
+    
+    try:
+        # Clean the text first
+        cleaned_text = normalize_unicode_text(text)
+        cleaned_text = clean_unicode_control_chars(cleaned_text)
+        
+        # Try to parse JSON
+        return json.loads(cleaned_text)
+    except json.JSONDecodeError as e:
+        print(f"Warning: JSON parsing failed: {e}")
+        # Try with mojibake fixes
+        try:
+            fixed_text = fix_mojibake(cleaned_text)
+            return json.loads(fixed_text)
+        except json.JSONDecodeError:
+            print("Warning: JSON parsing failed even after mojibake fixes")
+            return None
+    except Exception as e:
+        print(f"Warning: Unexpected error in JSON parsing: {e}")
+        return None
+
+
+def safe_file_read(file_path: str, encoding: str = 'utf-8', fallback_encodings: Optional[List[str]] = None) -> str:
+    """
+    Safely read a file with Unicode handling and encoding detection.
+    
+    Args:
+        file_path: Path to file to read
+        encoding: Primary encoding to try
+        fallback_encodings: List of fallback encodings to try
+        
+    Returns:
+        File contents as string
+    """
+    if fallback_encodings is None:
+        fallback_encodings = ['utf-8-sig', 'latin-1', 'cp1252', 'iso-8859-1']
+    
+    encodings_to_try = [encoding] + [enc for enc in fallback_encodings if enc != encoding]
+    
+    for enc in encodings_to_try:
+        try:
+            with codecs.open(file_path, 'r', encoding=enc, errors='replace') as f:
+                content = f.read()
+            
+            # Clean and normalize the content
+            content = normalize_unicode_text(content)
+            content = clean_unicode_control_chars(content)
+            
+            return content
+        except UnicodeDecodeError:
+            continue
+        except Exception as e:
+            print(f"Warning: Error reading file with encoding {enc}: {e}")
+            continue
+    
+    raise ValueError(f"Could not read file {file_path} with any of the attempted encodings")
+
+
+def safe_file_write(file_path: str, content: str, encoding: str = 'utf-8') -> None:
+    """
+    Safely write content to file with Unicode handling.
+    
+    Args:
+        file_path: Path to file to write
+        content: Content to write
+        encoding: Encoding to use for writing
+    """
+    if not isinstance(content, str):
+        content = str(content)
+    
+    # Normalize content before writing
+    content = normalize_unicode_text(content)
+    
+    try:
+        with codecs.open(file_path, 'w', encoding=encoding, errors='replace') as f:
+            f.write(content)
+    except Exception as e:
+        print(f"Warning: Error writing file {file_path}: {e}")
+        # Fallback: write with error replacement
+        with codecs.open(file_path, 'w', encoding=encoding, errors='replace') as f:
+            f.write(content)
+
+
+def process_text_robust(text: Union[str, bytes, Any], 
+                       normalize: bool = True,
+                       clean_control_chars: bool = True,
+                       fix_mojibake_issues: bool = True,
+                       safe_encoding: bool = True) -> str:
+    """
+    Robustly process text with comprehensive Unicode handling.
+    
+    Args:
+        text: Input text to process
+        normalize: Whether to normalize Unicode
+        clean_control_chars: Whether to clean control characters
+        fix_mojibake_issues: Whether to attempt mojibake fixes
+        safe_encoding: Whether to apply safe encoding/decoding
+        
+    Returns:
+        Processed text string
+    """
+    # Handle bytes input
+    if isinstance(text, bytes):
+        try:
+            text = text.decode('utf-8', errors='replace')
+        except:
+            text = str(text)
+    elif not isinstance(text, str):
+        text = str(text)
+    
+    # Apply processing steps
+    if normalize:
+        text = normalize_unicode_text(text)
+    
+    if clean_control_chars:
+        text = clean_unicode_control_chars(text)
+    
+    if fix_mojibake_issues:
+        text = fix_mojibake(text)
+    
+    if safe_encoding:
+        text = safe_encode_decode(text)
+    
+    return text
+
+
+def validate_unicode_text(text: str) -> Dict[str, Any]:
+    """
+    Validate and analyze Unicode text for potential issues.
+    
+    Args:
+        text: Text to validate
+        
+    Returns:
+        Dictionary with validation results and statistics
+    """
+    if not isinstance(text, str):
+        text = str(text)
+    
+    results = {
+        'length': len(text),
+        'is_ascii': text.isascii(),
+        'encoding_issues': [],
+        'control_chars_count': 0,
+        'non_printable_count': 0,
+        'unicode_categories': {},
+    }
+    
+    # Count control characters
+    control_char_pattern = re.compile(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]')
+    results['control_chars_count'] = len(control_char_pattern.findall(text))
+    
+    # Count non-printable characters
+    results['non_printable_count'] = sum(1 for c in text if not c.isprintable())
+    
+    # Analyze Unicode categories
+    for char in text[:1000]:  # Sample first 1000 chars for performance
+        category = unicodedata.category(char)
+        results['unicode_categories'][category] = results['unicode_categories'].get(category, 0) + 1
+    
+    # Check for common encoding issues
+    if 'Ã' in text and any(char in text for char in ['¡', '©', '­', '³', 'º', '±', '¼']):
+        results['encoding_issues'].append('Possible UTF-8 to Latin-1 mojibake')
+    
+    if 'â€' in text:
+        results['encoding_issues'].append('Possible smart quote encoding issues')
+    
+    return results
+
+
+# Example usage and testing functions
+def test_unicode_utils():
+    """Test function to verify Unicode utilities work correctly."""
+    
+    # Test cases
+    test_cases = [
+        "Normal ASCII text",
+        "Unicode: café, naïve, résumé",
+        "Mojibake: caf© na√Øve r©sum©",
+        "Control chars: Hello\x00\x01World",
+        "Smart quotes: \"Hello\" 'World'",
+        "Mixed: café\u00A0with\u2000spaces",
+    ]
+    
+    print("Testing Unicode utilities...")
+    for i, test_text in enumerate(test_cases):
+        print(f"\nTest {i+1}: {repr(test_text[:50])}")
+        
+        # Process the text
+        processed = process_text_robust(test_text)
+        print(f"Processed: {repr(processed[:50])}")
+        
+        # Validate the text
+        validation = validate_unicode_text(test_text)
+        print(f"Issues found: {len(validation['encoding_issues'])}")
+        if validation['encoding_issues']:
+            print(f"Encoding issues: {validation['encoding_issues']}")
+
+
+if __name__ == "__main__":
+    test_unicode_utils()

refchecker/utils/url_utils.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+"""
+URL Utilities for Reference Checking
+
+This module provides utilities for URL construction, validation, and manipulation
+related to academic references.
+"""
+
+import re
+from typing import Optional
+from .doi_utils import normalize_doi
+
+
+def construct_doi_url(doi: str) -> str:
+    """
+    Construct a proper DOI URL from a DOI string.
+    
+    Args:
+        doi: DOI string
+        
+    Returns:
+        Full DOI URL
+    """
+    if not doi:
+        return ""
+    
+    # Normalize the DOI first
+    normalized_doi = normalize_doi(doi)
+    
+    # Construct URL
+    return f"https://doi.org/{normalized_doi}"
+
+
+def extract_arxiv_id_from_url(url: str) -> Optional[str]:
+    """
+    Extract ArXiv ID from an ArXiv URL or text containing ArXiv reference.
+    
+    This is the common function that handles all ArXiv ID extraction patterns:
+    - URLs: https://arxiv.org/abs/1234.5678, https://arxiv.org/pdf/1234.5678.pdf, https://arxiv.org/html/1234.5678
+    - Text references: arXiv:1234.5678, arXiv preprint arXiv:1234.5678
+    - Version handling: removes version numbers (v1, v2, etc.)
+    
+    Args:
+        url: ArXiv URL or text containing ArXiv reference
+        
+    Returns:
+        ArXiv ID (without version) if found, None otherwise
+    """
+    if not url or not isinstance(url, str):
+        return None
+    
+    # Pattern 1: arXiv: format (e.g., "arXiv:1610.10099" or "arXiv preprint arXiv:1610.10099")
+    arxiv_text_match = re.search(r'arXiv:(\d{4}\.\d{4,5})', url, re.IGNORECASE)
+    if arxiv_text_match:
+        arxiv_id = arxiv_text_match.group(1)
+        # Remove version number if present
+        return re.sub(r'v\d+$', '', arxiv_id)
+    
+    # Pattern 2: arxiv.org URLs (abs, pdf, html)
+    # Handle URLs with version numbers and various formats
+    arxiv_url_match = re.search(r'arxiv\.org/(?:abs|pdf|html)/([^\s/?#]+?)(?:\.pdf|v\d+)?(?:[?\#]|$)', url, re.IGNORECASE)
+    if arxiv_url_match:
+        arxiv_id = arxiv_url_match.group(1)
+        # Remove version number if present
+        return re.sub(r'v\d+$', '', arxiv_id)
+    
+    # Pattern 3: Fallback for simpler URL patterns
+    fallback_match = re.search(r'arxiv\.org/(?:abs|pdf|html)/([^/?#]+)', url, re.IGNORECASE)
+    if fallback_match:
+        arxiv_id = fallback_match.group(1).replace('.pdf', '')
+        # Remove version number if present
+        return re.sub(r'v\d+$', '', arxiv_id)
+    
+    return None
+
+
+def construct_arxiv_url(arxiv_id: str, url_type: str = "abs") -> str:
+    """
+    Construct an ArXiv URL from an ArXiv ID.
+    
+    Args:
+        arxiv_id: ArXiv identifier
+        url_type: Type of URL ('abs' for abstract, 'pdf' for PDF)
+        
+    Returns:
+        Full ArXiv URL
+    """
+    if not arxiv_id:
+        return ""
+    
+    # Remove version number if present for consistency
+    clean_id = arxiv_id.replace('v1', '').replace('v2', '').replace('v3', '')
+    
+    if url_type == "pdf":
+        return f"https://arxiv.org/pdf/{clean_id}.pdf"
+    else:
+        return f"https://arxiv.org/abs/{clean_id}"
+
+
+def construct_semantic_scholar_url(paper_id: str) -> str:
+    """
+    Construct a Semantic Scholar URL from a paper ID.
+    
+    Args:
+        paper_id: Semantic Scholar paper ID (SHA hash, NOT CorpusId)
+                  The paperId is the 40-character hex hash that works in web URLs.
+                  CorpusId (numeric) does NOT work in web URLs.
+        
+    Returns:
+        Full Semantic Scholar URL
+    """
+    if not paper_id:
+        return ""
+    
+    return f"https://www.semanticscholar.org/paper/{paper_id}"
+
+
+def construct_openalex_url(work_id: str) -> str:
+    """
+    Construct an OpenAlex URL from a work ID.
+    
+    Args:
+        work_id: OpenAlex work identifier
+        
+    Returns:
+        Full OpenAlex URL
+    """
+    if not work_id:
+        return ""
+    
+    # Remove prefix if present
+    clean_id = work_id.replace('https://openalex.org/', '')
+    
+    return f"https://openalex.org/{clean_id}"
+
+
+def construct_pubmed_url(pmid: str) -> str:
+    """
+    Construct a PubMed URL from a PMID.
+    
+    Args:
+        pmid: PubMed identifier
+        
+    Returns:
+        Full PubMed URL
+    """
+    if not pmid:
+        return ""
+    
+    # Remove PMID prefix if present
+    clean_pmid = pmid.replace('PMID:', '').strip()
+    
+    return f"https://pubmed.ncbi.nlm.nih.gov/{clean_pmid}/"
+
+
+def get_best_available_url(external_ids: dict, open_access_pdf: Optional[str] = None, paper_id: Optional[str] = None) -> Optional[str]:
+    """
+    Get the best available URL from a paper's external IDs and open access information.
+    Priority: Open Access PDF > DOI > ArXiv > Semantic Scholar > OpenAlex > PubMed
+    
+    Args:
+        external_ids: Dictionary of external identifiers
+        open_access_pdf: Open access PDF URL if available
+        paper_id: Semantic Scholar paperId (SHA hash) if available
+        
+    Returns:
+        Best available URL or None if no valid URL found
+    """
+    # Priority 1: Open access PDF
+    if open_access_pdf:
+        return open_access_pdf
+    
+    # Priority 2: DOI URL
+    if external_ids.get('DOI'):
+        return construct_doi_url(external_ids['DOI'])
+    
+    # Priority 3: ArXiv URL
+    if external_ids.get('ArXiv'):
+        return construct_arxiv_url(external_ids['ArXiv'])
+    
+    # Priority 4: Semantic Scholar URL (using paperId, not CorpusId)
+    if paper_id:
+        return construct_semantic_scholar_url(paper_id)
+    
+    # Priority 5: OpenAlex URL
+    if external_ids.get('OpenAlex'):
+        return construct_openalex_url(external_ids['OpenAlex'])
+    
+    # Priority 6: PubMed URL
+    if external_ids.get('PubMed'):
+        return construct_pubmed_url(external_ids['PubMed'])
+    
+    return None
+
+
+def validate_url_format(url: str) -> bool:
+    """
+    Basic validation of URL format.
+    
+    Args:
+        url: URL to validate
+        
+    Returns:
+        True if URL appears to be valid, False otherwise
+    """
+    if not url:
+        return False
+    
+    # Basic URL format check
+    return url.startswith(('http://', 'https://')) and '.' in url
+
+
+def clean_url(url: str) -> str:
+    """
+    Clean a URL by removing common issues like extra spaces, fragments, malformed LaTeX, etc.
+    
+    This function handles:
+    - Whitespace trimming
+    - Malformed LaTeX URL wrappers like \\url{https://...}
+    - Markdown-style links like [text](url)
+    - Trailing punctuation from academic references
+    - DOI URL query parameter cleanup
+    
+    Args:
+        url: URL to clean
+        
+    Returns:
+        Cleaned URL
+    """
+    if not url:
+        return ""
+    
+    # Remove leading/trailing whitespace
+    url = url.strip()
+    
+    # Handle malformed URLs that contain \url{} wrappers within the URL text
+    # e.g., "https://\url{https://www.example.com/}" -> "https://www.example.com/"
+    import re
+    url_pattern = r'https?://\\url\{(https?://[^}]+)\}'
+    url_match = re.search(url_pattern, url)
+    if url_match:
+        url = url_match.group(1)
+    
+    # Handle markdown-style links like [text](url) or [url](url)
+    # e.g., "[https://example.com](https://example.com)" -> "https://example.com"
+    markdown_pattern = r'\[([^\]]*)\]\((https?://[^)]+)\)'
+    markdown_match = re.search(markdown_pattern, url)
+    if markdown_match:
+        # Use the URL from parentheses
+        url = markdown_match.group(2)
+    
+    # Remove trailing punctuation that's commonly part of sentence structure
+    # but preserve legitimate URL characters
+    url = url.rstrip('.,;!?)')
+    
+    # Note: Preserving query parameters for all URLs now
+    # Previously this function removed query parameters for non-DOI URLs,
+    # but this was causing issues with OpenReview and other URLs that need their parameters
+    # Only remove query parameters for DOI URLs where they're typically not needed
+    if '?' in url and 'doi.org' in url:
+        base_url, params = url.split('?', 1)
+        url = base_url
+    
+    return url
+
+
+def clean_url_punctuation(url: str) -> str:
+    """
+    Clean trailing punctuation from URLs that often gets included during extraction.
+    
+    This function removes trailing punctuation that commonly gets extracted with URLs
+    from academic references (periods, commas, semicolons, etc.) while preserving
+    legitimate URL characters including query parameters.
+    
+    Args:
+        url: URL string that may have trailing punctuation
+        
+    Returns:
+        Cleaned URL with trailing punctuation removed
+    """
+    if not url:
+        return ""
+    
+    # Remove leading/trailing whitespace
+    url = url.strip()
+    
+    # Handle malformed URLs that contain \\url{} wrappers within the URL text
+    # e.g., "https://\\url{https://www.example.com/}" -> "https://www.example.com/"
+    import re
+    url_pattern = r'https?://\\url\{(https?://[^}]+)\}'
+    url_match = re.search(url_pattern, url)
+    if url_match:
+        url = url_match.group(1)
+    
+    # Handle markdown-style links like [text](url) or [url](url)
+    # e.g., "[https://example.com](https://example.com)" -> "https://example.com"
+    markdown_pattern = r'\[([^\]]*)\]\((https?://[^)]+)\)'
+    markdown_match = re.search(markdown_pattern, url)
+    if markdown_match:
+        # Use the URL from parentheses
+        url = markdown_match.group(2)
+    
+    # Remove trailing punctuation that's commonly part of sentence structure
+    # but preserve legitimate URL characters
+    url = url.rstrip('.,;!?)')
+    
+    return url