PyPI - academic-refchecker - Versions diffs - 2.0.14__py3-none-any.whl → 2.0.16__py3-none-any.whl - Mend

academic-refchecker 2.0.14py3-none-any.whl → 2.0.16py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (14) hide show

backend/static/index.html CHANGED Viewed

@@ -6,8 +6,8 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>RefChecker - Academic Reference Validator</title>
     <meta name="description" content="Validate academic paper references using AI-powered verification" />
-    <script type="module" crossorigin src="/assets/index-hk21nqxR.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-2P6L_39v.css">
+    <script type="module" crossorigin src="/assets/index-DMZJNrR0.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-BuguAhjS.css">
   </head>
   <body>
     <div id="root"></div>

refchecker/__version__.py CHANGED Viewed

@@ -1,3 +1,3 @@
 """Version information for RefChecker."""
-__version__ = "2.0.14"
+__version__ = "2.0.16"

refchecker/checkers/arxiv_citation.py CHANGED Viewed

@@ -31,6 +31,7 @@ import re
 import logging
 import requests
 import html
+import time
 from typing import Dict, List, Tuple, Optional, Any
 import bibtexparser
@@ -356,7 +357,13 @@ class ArXivCitationChecker:
         version_str = f"v{version_num}"
         url = f"{self.abs_url}/{arxiv_id}{version_str}"
+        # Use shorter delay for version metadata (HTML parsing is lightweight)
+        # Save original delay, use 1 second, then restore
+        original_delay = self.rate_limiter.delay
+        self.rate_limiter.delay = 1.0  # Faster rate for version checking
         self.rate_limiter.wait()
+        self.rate_limiter.delay = original_delay  # Restore original delay
         try:
             logger.debug(f"Checking historical version: {url}")
             response = requests.get(url, timeout=self.timeout)
@@ -421,16 +428,59 @@ class ArXivCitationChecker:
             logger.warning(f"Failed to get latest version for {arxiv_id}: {e}")
             return None
+    def _calculate_match_score(
+            self, cited_title: str, cited_authors: List[str],
+            authoritative_title: str, authoritative_authors: List[Dict]) -> float:
+        """
+        Calculate a numeric match score between cited reference and authoritative data.
+        Used to find the BEST matching historical version, not just the first one that
+        passes a threshold.
+        Args:
+            cited_title: Title from the reference
+            cited_authors: Authors from the reference
+            authoritative_title: Title from ArXiv version
+            authoritative_authors: Authors from ArXiv version
+        Returns:
+            A score between 0.0 and 1.0 where higher is better.
+        """
+        if not cited_title or not authoritative_title:
+            return 0.0
+        # Primary: Title similarity (weighted at 80%)
+        title_similarity = compare_titles_with_latex_cleaning(cited_title, authoritative_title)
+        # Secondary: Author count match (weighted at 20%)
+        author_score = 0.0
+        if cited_authors and authoritative_authors:
+            cited_count = len(cited_authors)
+            auth_count = len(authoritative_authors)
+            if cited_count == auth_count:
+                author_score = 1.0
+            elif abs(cited_count - auth_count) == 1:
+                author_score = 0.7
+            elif abs(cited_count - auth_count) == 2:
+                author_score = 0.4
+            else:
+                author_score = 0.1
+        # Weighted combination
+        return 0.8 * title_similarity + 0.2 * author_score
     def _compare_info_match(
             self, cited_title: str, cited_authors: List[str], cited_year: Optional[int],
-            authoritative_title: str, authoritative_authors: List[str], authoritative_year: Optional[int]) -> bool:
+            authoritative_title: str, authoritative_authors: List[Dict], authoritative_year: Optional[int]) -> bool:
         """
         Compare the information of a cited paper with the authoritative information.
+        Uses title as the primary matching criterion. Authors are used as a secondary
+        check, but year is not required to match (year often has discrepancies).
         Args:
             cited_title: Title from the reference
             cited_authors: Authors from the reference
-            cited_year: Year from the reference
             authoritative_title: Title from ArXiv version
             authoritative_authors: Authors from ArXiv version
             authoritative_year: Year from ArXiv version
@@ -438,22 +488,31 @@ class ArXivCitationChecker:
         Returns:
             True if the information matches, False otherwise.
         """
-        # Compare title
+        # Primary criterion: Title MUST match
         if cited_title and authoritative_title:
             title_similarity = compare_titles_with_latex_cleaning(cited_title, authoritative_title)
             if title_similarity < SIMILARITY_THRESHOLD:
                 return False
+        else:
+            # If no title to compare, can't determine match
+            return False
-        # Compare authors
+        # Secondary criterion: If authors are provided, they should reasonably match
+        # (be lenient - allow partial matches since author lists can vary)
         if cited_authors and authoritative_authors:
             authors_match, _ = compare_authors(cited_authors, authoritative_authors)
-            if not authors_match:
+            # If authors don't match at all, this might not be the right version
+            # But be lenient - just having similar author count is a good sign
+            cited_count = len(cited_authors)
+            auth_count = len(authoritative_authors)
+            # Allow if authors match OR if author counts are within 1 of each other
+            if not authors_match and abs(cited_count - auth_count) > 1:
                 return False
-        # Compare year
-        if cited_year and authoritative_year:
-            if cited_year != authoritative_year:
-                return False
+        # Year is NOT used as a matching criterion because:
+        # 1. ArXiv shows submission date, citations often use publication year
+        # 2. People often cite with incorrect years
+        # 3. The same ArXiv version can be cited with different years
         return True
@@ -556,37 +615,68 @@ class ArXivCitationChecker:
         latest_version_num = self._get_latest_version_number(arxiv_id)
         if latest_version_num and latest_version_num > 1:
+            # Find the BEST matching version, not just the first one
+            best_match_version = None
+            best_match_score = 0.0
+            best_match_data = None
+            # Add timeout for version checking (30 seconds max)
+            # This prevents blocking when rate-limited with many concurrent ArXiv requests
+            version_check_start = time.time()
+            VERSION_CHECK_TIMEOUT = 30.0
             # Check historical versions (1 to latest-1)
-            for version_num in range(1, latest_version_num):
+            # Start from newest historical version (more likely to match recent citations)
+            for version_num in range(latest_version_num - 1, 0, -1):
+                # Check if we've exceeded the version checking timeout
+                if time.time() - version_check_start > VERSION_CHECK_TIMEOUT:
+                    logger.debug(f"ArXivCitationChecker: Version checking timed out after {VERSION_CHECK_TIMEOUT}s")
+                    break
                 version_data = self._fetch_version_metadata_from_html(arxiv_id, version_num)
                 if not version_data:
                     continue
-                # Check if reference matches this historical version
-                if self._compare_info_match(
-                        cited_title, cited_authors, cited_year,
-                        version_data['title'], version_data['authors'], version_data['year']):
-                    logger.debug(f"ArXivCitationChecker: Reference matches historical version v{version_num}")
-                    # Convert errors to warnings with version update info
-                    # Version update issues are informational, not errors - the citation was correct for its time
-                    version_suffix = f" (v{version_num} vs v{latest_version_num} update)"
-                    warnings = []
-                    for error in errors:
-                        warning = {
-                            'warning_type': error.get('error_type', 'unknown') + version_suffix,
-                            'warning_details': error.get('error_details', ''),
-                        }
-                        # Preserve correction hints
-                        for key in ['ref_title_correct', 'ref_authors_correct', 'ref_year_correct']:
-                            if key in error:
-                                warning[key] = error[key]
-                        warnings.append(warning)
-                    # Return with warnings instead of errors - URL points to the matched version
-                    matched_url = f"https://arxiv.org/abs/{arxiv_id}v{version_num}"
-                    return latest_data, warnings, matched_url
+                # Calculate match score for this version
+                match_score = self._calculate_match_score(
+                    cited_title, cited_authors,
+                    version_data['title'], version_data['authors'])
+                if match_score > best_match_score:
+                    best_match_score = match_score
+                    best_match_version = version_num
+                    best_match_data = version_data
+                # Early termination: if we found an excellent match (>= 0.98), stop searching
+                # This saves HTTP requests when we've found a near-perfect version match
+                if best_match_score >= 0.98:
+                    logger.debug(f"ArXivCitationChecker: Found excellent version match v{best_match_version} (score: {best_match_score:.3f}), stopping search")
+                    break
+            # If we found a matching version (above threshold), convert errors to warnings
+            if best_match_version and best_match_score >= SIMILARITY_THRESHOLD:
+                logger.debug(f"ArXivCitationChecker: Reference best matches historical version v{best_match_version} (score: {best_match_score:.3f})")
+                # Convert errors to warnings with version update info
+                # Version update issues are informational, not errors - the citation was correct for its time
+                version_suffix = f" (v{best_match_version} vs v{latest_version_num} update)"
+                warnings = []
+                for error in errors:
+                    # Get the error/warning type - handle both error_type and warning_type
+                    err_type = error.get('error_type') or error.get('warning_type', 'unknown')
+                    warning = {
+                        'warning_type': err_type + version_suffix,
+                        'warning_details': error.get('error_details') or error.get('warning_details', ''),
+                    }
+                    # Preserve correction hints
+                    for key in ['ref_title_correct', 'ref_authors_correct', 'ref_year_correct']:
+                        if key in error:
+                            warning[key] = error[key]
+                    warnings.append(warning)
+                # Return with warnings instead of errors - URL points to the matched version
+                matched_url = f"https://arxiv.org/abs/{arxiv_id}v{best_match_version}"
+                return latest_data, warnings, matched_url
         logger.debug(f"ArXivCitationChecker: Verified {arxiv_id} with {len(errors)} errors/warnings")
         return latest_data, errors, paper_url

refchecker/checkers/semantic_scholar.py CHANGED Viewed

@@ -27,9 +27,11 @@ import requests
 import time
 import logging
 import re
+import html
 from typing import Dict, List, Tuple, Optional, Any, Union
 from refchecker.utils.text_utils import normalize_text, clean_title_basic, find_best_match, is_name_match, are_venues_substantially_different, calculate_title_similarity, compare_authors, clean_title_for_search, strip_latex_commands, compare_titles_with_latex_cleaning
 from refchecker.utils.error_utils import format_title_mismatch
+from refchecker.utils.arxiv_rate_limiter import ArXivRateLimiter
 from refchecker.config.settings import get_config
 # Set up logging
@@ -67,6 +69,11 @@ class NonArxivReferenceChecker:
         # Track API failures for Enhanced Hybrid Checker
         self._api_failed = False
         self._failure_reason = None
+        # ArXiv rate limiter for version checks
+        self.arxiv_rate_limiter = ArXivRateLimiter.get_instance()
+        self.arxiv_abs_url = "https://arxiv.org/abs"
+        self.arxiv_timeout = 30
     def search_paper(self, query: str, year: Optional[int] = None) -> List[Dict[str, Any]]:
         """
@@ -267,6 +274,258 @@ class NonArxivReferenceChecker:
         return paper_venue if paper_venue else None
+    def _extract_arxiv_id_and_version(self, reference: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
+        """
+        Extract ArXiv ID and version from a reference.
+        Args:
+            reference: Reference dictionary containing url, raw_text, etc.
+        Returns:
+            Tuple of (arxiv_id_without_version, version_string_or_None)
+            For example: ("2301.12345", "v2") or ("2301.12345", None)
+        """
+        # Patterns to extract arXiv IDs with versions
+        arxiv_id_patterns = [
+            r'arxiv\.org/abs/([0-9]{4}\.[0-9]{4,5})(v\d+)?',
+            r'arxiv\.org/pdf/([0-9]{4}\.[0-9]{4,5})(v\d+)?',
+            r'arxiv\.org/abs/([a-z-]+/[0-9]{7})(v\d+)?',
+            r'arxiv\.org/pdf/([a-z-]+/[0-9]{7})(v\d+)?',
+            r'arXiv:([0-9]{4}\.[0-9]{4,5})(v\d+)?',
+            r'arXiv:([a-z-]+/[0-9]{7})(v\d+)?',
+        ]
+        sources = [
+            reference.get('url', ''),
+            reference.get('cited_url', ''),
+            reference.get('raw_text', ''),
+        ]
+        for source in sources:
+            if not source:
+                continue
+            for pattern in arxiv_id_patterns:
+                match = re.search(pattern, source, re.IGNORECASE)
+                if match:
+                    arxiv_id = match.group(1)
+                    version = match.group(2) if len(match.groups()) > 1 else None
+                    return arxiv_id, version
+        return None, None
+    def _get_latest_arxiv_version_number(self, arxiv_id: str) -> Optional[int]:
+        """
+        Get the latest version number for an ArXiv paper.
+        Args:
+            arxiv_id: ArXiv ID without version
+        Returns:
+            Latest version number as integer, or None if couldn't determine
+        """
+        url = f"{self.arxiv_abs_url}/{arxiv_id}"
+        self.arxiv_rate_limiter.wait()
+        try:
+            response = requests.get(url, timeout=self.arxiv_timeout)
+            response.raise_for_status()
+            # Look for version links like "[v1]", "[v2]", etc.
+            versions = re.findall(r'\[v(\d+)\]', response.text)
+            if versions:
+                return max(int(v) for v in versions)
+            return None
+        except Exception as e:
+            logger.debug(f"Failed to get latest version for {arxiv_id}: {e}")
+            return None
+    def _fetch_arxiv_version_metadata(self, arxiv_id: str, version_num: int) -> Optional[Dict[str, Any]]:
+        """
+        Fetch metadata for a specific ArXiv version using HTML scraping.
+        Args:
+            arxiv_id: ArXiv ID without version
+            version_num: Version number to fetch (1, 2, 3, etc.)
+        Returns:
+            Dictionary with version metadata or None if version doesn't exist
+        """
+        version_str = f"v{version_num}"
+        url = f"{self.arxiv_abs_url}/{arxiv_id}{version_str}"
+        self.arxiv_rate_limiter.wait()
+        try:
+            logger.debug(f"Checking ArXiv version: {url}")
+            response = requests.get(url, timeout=self.arxiv_timeout)
+            if response.status_code == 404:
+                return None
+            response.raise_for_status()
+            html_content = response.text
+            # Parse meta tags for metadata
+            title_match = re.search(r'<meta name="citation_title" content="(.*?)"', html_content)
+            title = html.unescape(title_match.group(1)).strip() if title_match else ""
+            authors = []
+            for auth in re.findall(r'<meta name="citation_author" content="(.*?)"', html_content):
+                authors.append({'name': html.unescape(auth).strip()})
+            date_match = re.search(r'<meta name="citation_date" content="(.*?)"', html_content)
+            year = None
+            if date_match:
+                ym = re.search(r'^(\d{4})', date_match.group(1))
+                if ym:
+                    year = int(ym.group(1))
+            return {
+                'version': version_str,
+                'version_num': version_num,
+                'title': title,
+                'authors': authors,
+                'year': year,
+                'url': url,
+            }
+        except Exception as e:
+            logger.debug(f"Failed to fetch ArXiv version {version_str}: {e}")
+            return None
+    def _check_arxiv_version_update(self, reference: Dict[str, Any], paper_data: Dict[str, Any], arxiv_id: str, errors: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], Optional[int]]:
+        """
+        Check if a reference is citing an older version of an ArXiv paper that has been updated.
+        If the reference matches a historical version, converts errors to warnings with version annotation.
+        Args:
+            reference: The original reference dictionary
+            paper_data: The verified paper data from Semantic Scholar (latest version)
+            arxiv_id: The ArXiv ID from the paper
+            errors: The current list of errors found against the latest version
+        Returns:
+            Tuple of (modified_errors_or_warnings, matched_version_num)
+            - If reference matches a historical version: returns (warnings_with_version_suffix, matched_version)
+            - Otherwise: returns (original_errors, None)
+        """
+        # Extract cited version from reference
+        _, cited_version = self._extract_arxiv_id_and_version(reference)
+        # Get the latest version number
+        latest_version_num = self._get_latest_arxiv_version_number(arxiv_id)
+        if not latest_version_num or latest_version_num <= 1:
+            # Only one version exists or couldn't determine
+            return errors, None
+        # Check if reference explicitly cites a specific older version
+        cited_version_num = None
+        if cited_version:
+            match = re.match(r'v(\d+)', cited_version)
+            if match:
+                cited_version_num = int(match.group(1))
+        # If a specific older version is cited in the URL, convert errors to warnings
+        if cited_version_num and cited_version_num < latest_version_num:
+            version_suffix = f" (v{cited_version_num} vs v{latest_version_num} update)"
+            warnings = self._convert_errors_to_version_warnings(errors, version_suffix)
+            return warnings, cited_version_num
+        # If no explicit version or no errors to check, return original
+        if not errors:
+            return errors, None
+        # Check if reference metadata matches a historical version
+        cited_title = reference.get('title', '').strip()
+        if not cited_title:
+            return errors, None
+        from refchecker.utils.text_utils import compare_titles_with_latex_cleaning
+        # Find the BEST matching version by comparing against all versions
+        # (not just checking if latest exceeds threshold)
+        best_match_version = None
+        best_match_score = 0.0
+        # Check latest version first
+        latest_version_data = self._fetch_arxiv_version_metadata(arxiv_id, latest_version_num)
+        if latest_version_data:
+            latest_title = latest_version_data.get('title', '').strip()
+            if latest_title:
+                latest_score = compare_titles_with_latex_cleaning(cited_title, latest_title)
+                if latest_score >= SIMILARITY_THRESHOLD:
+                    best_match_version = latest_version_num
+                    best_match_score = latest_score
+        # Check historical versions to find if any is a BETTER match
+        for version_num in range(1, latest_version_num):
+            version_data = self._fetch_arxiv_version_metadata(arxiv_id, version_num)
+            if not version_data:
+                continue
+            version_title = version_data.get('title', '').strip()
+            if not version_title:
+                continue
+            version_score = compare_titles_with_latex_cleaning(cited_title, version_title)
+            # If this version is a better match than current best
+            if version_score > best_match_score and version_score >= SIMILARITY_THRESHOLD:
+                best_match_version = version_num
+                best_match_score = version_score
+        # If best match is a historical version (not latest), convert errors to warnings
+        if best_match_version is not None and best_match_version < latest_version_num:
+            logger.debug(f"Reference best matches ArXiv v{best_match_version} (score: {best_match_score:.3f}, latest is v{latest_version_num})")
+            version_suffix = f" (v{best_match_version} vs v{latest_version_num} update)"
+            warnings = self._convert_errors_to_version_warnings(errors, version_suffix)
+            return warnings, best_match_version
+        return errors, None
+    def _convert_errors_to_version_warnings(self, errors: List[Dict[str, Any]], version_suffix: str) -> List[Dict[str, Any]]:
+        """
+        Convert error dictionaries to warning dictionaries with version suffix.
+        Args:
+            errors: List of error dictionaries
+            version_suffix: Version suffix to append (e.g., " (v1 vs v3 update)")
+        Returns:
+            List of warning dictionaries with version annotation
+        """
+        warnings = []
+        for error in errors:
+            error_type = error.get('error_type', '')
+            # Skip info_type entries (suggestions) - keep them as-is
+            if 'info_type' in error:
+                warnings.append(error)
+                continue
+            # Skip entries that are already warnings
+            if 'warning_type' in error:
+                # Just append the version suffix
+                warning = error.copy()
+                warning['warning_type'] = error['warning_type'] + version_suffix
+                warnings.append(warning)
+                continue
+            # Convert error to warning with version suffix
+            warning = {
+                'warning_type': error_type + version_suffix,
+                'warning_details': error.get('error_details', ''),
+            }
+            # Preserve correction hints
+            for key in ['ref_title_correct', 'ref_authors_correct', 'ref_year_correct',
+                       'ref_venue_correct', 'ref_doi_correct', 'ref_url_correct']:
+                if key in error:
+                    warning[key] = error[key]
+            warnings.append(warning)
+        return warnings
     def verify_reference(self, reference: Dict[str, Any]) -> Tuple[Optional[Dict[str, Any]], List[Dict[str, Any]], Optional[str]]:
         """
         Verify a non-arXiv reference using Semantic Scholar
@@ -642,6 +901,10 @@ class NonArxivReferenceChecker:
                     'info_details': f"Reference could include arXiv URL: {arxiv_url}",
                     'ref_url_correct': arxiv_url
                 })
+            # Check for ArXiv version updates - if reference matches an older version,
+            # convert errors to warnings with version annotation (like ArXiv citation checker)
+            errors, matched_version = self._check_arxiv_version_update(reference, paper_data, arxiv_id, errors)
         # Verify DOI
         paper_doi = None

{academic_refchecker-2.0.14.dist-info → academic_refchecker-2.0.16.dist-info}/WHEEL RENAMED Viewed

File without changes

{academic_refchecker-2.0.14.dist-info → academic_refchecker-2.0.16.dist-info}/entry_points.txt RENAMED Viewed

File without changes

{academic_refchecker-2.0.14.dist-info → academic_refchecker-2.0.16.dist-info}/licenses/LICENSE RENAMED Viewed

File without changes

{academic_refchecker-2.0.14.dist-info → academic_refchecker-2.0.16.dist-info}/top_level.txt RENAMED Viewed

File without changes

academic-refchecker 2.0.14__py3-none-any.whl → 2.0.16__py3-none-any.whl

academic-refchecker 2.0.14py3-none-any.whl → 2.0.16py3-none-any.whl