academic-refchecker 1.2.48__py3-none-any.whl → 1.2.50__py3-none-any.whl

__version__.py

@@ -1,3 +1,3 @@
 """Version information for RefChecker."""
 
-__version__ = "1.2.48"
+__version__ = "1.2.50"

{academic_refchecker-1.2.48.dist-info → academic_refchecker-1.2.50.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: academic-refchecker
-Version: 1.2.48
+Version: 1.2.50
 Summary: A comprehensive tool for validating reference accuracy in academic papers
 Author-email: Mark Russinovich <markrussinovich@hotmail.com>
 License-Expression: MIT

{academic_refchecker-1.2.48.dist-info → academic_refchecker-1.2.50.dist-info}/RECORD

@@ -1,21 +1,21 @@
-__version__.py,sha256=YqYMb7RgdYzVlFjjkcZ3XA6aq61WIeNqUMqbccqtZcg,65
-academic_refchecker-1.2.48.dist-info/licenses/LICENSE,sha256=Kwrx3fePVCeEFDCZvCW4OuoTNBiSoYbpGBI6qzGhWF0,1067
+__version__.py,sha256=ZQ6vcRuuZpexVshgiVwj1EkuR3vzgsRUj6ll7aoa8Dw,65
+academic_refchecker-1.2.50.dist-info/licenses/LICENSE,sha256=Kwrx3fePVCeEFDCZvCW4OuoTNBiSoYbpGBI6qzGhWF0,1067
 checkers/__init__.py,sha256=T0PAHTFt6UiGvn-WGoJU8CdhXNmf6zaHmcGVoWHhmJQ,533
 checkers/crossref.py,sha256=cLYmSzE8ehJ5sNko_R3fEiGBGiPH5_HxLhFM-pCfDRM,20378
-checkers/enhanced_hybrid_checker.py,sha256=6yf5tV4jLSVzjX1xR_kQq0NOgQIst-z_WmkiqqMc8hQ,23469
+checkers/enhanced_hybrid_checker.py,sha256=rbXkzpNkd0bn4e2OooX-CcdGTwwYpgmVaFvX_xCAFsA,27777
 checkers/github_checker.py,sha256=GoepG4aRRUqAomkM4HgOSNf20BPxQgocZEpsk0ZTZZU,14003
 checkers/local_semantic_scholar.py,sha256=D8py8-yMCgN1lvhXCiMUOEA4wBkH7AQvrkM4-3LCDsU,21015
 checkers/openalex.py,sha256=Fbc7iscZzmXjAZxH32PDX2r2Nwo9b5Ku-Sh1Ut9KpLA,19550
-checkers/openreview_checker.py,sha256=EYRcVu7sZzrFjqdOEKp72vd0S3sXSSjTgI9jsAVpdD0,20524
-checkers/semantic_scholar.py,sha256=99KOHLAiYs31nSdx-gcMR_TWIlV8G4juNL0bmV4AoUs,34768
+checkers/openreview_checker.py,sha256=mu33gytnIEond5A2gAZtxgLkKrirmmUSIUc4frL1GsA,40030
+checkers/semantic_scholar.py,sha256=dDDOxURwr-Kx7fIiJTAh_4_9V8VxGWYabQJiQ1VdSbM,34762
 checkers/webpage_checker.py,sha256=Ivzhu0xcpeWZcCTlAt9C06Lfsz5gKEiSko9gJ7EyQEw,22324
 config/__init__.py,sha256=r7sONsX2-ITviUJRU1KEz76uAuTRqZlzU-TVkvFRGYY,15
 config/logging.conf,sha256=r1tP0ApLHtlz7rV-oKS1MVO7oXJOgahbZFTtYmKnf9U,687
 config/settings.py,sha256=-vODFoXbWbGPUElpmchE5zbCj_n4Vtxr8HU1hQDFp_c,6164
 core/__init__.py,sha256=1T2MSQyDk0u_PupbHvm4CvNNN--dxsw78fqKUrqoYrM,157
 core/db_connection_pool.py,sha256=XRiOdehikkSz3obH4WKgf8woa3694if50Q15rBT-4XQ,4697
-core/parallel_processor.py,sha256=AOnjqhBHXlSb1c-PSunat9Eug5y04gOygwbHdPUqxgk,17202
-core/refchecker.py,sha256=OXCMKAlX6XnX5mfBDvW2kfhJwU1Qe45yFy0lfQl_Xcs,277331
+core/parallel_processor.py,sha256=VHjsHc_wCKumeF__fXh8RjMpM4dYE8ua5amgotd4PTg,17474
+core/refchecker.py,sha256=sOenr6DgXqReiOCKcPVVFaaAAYJXEwONdw9gjur4KYE,279937
 database/__init__.py,sha256=mEuVHlEBuS44t_2ZT_JnvQQrlRCjo1SJq1NmaJ6r8OY,125
 database/download_semantic_scholar_db.py,sha256=waN4I97KC_36YMiPbiBDUUmgfzu1nub5yeKdAsIR2aw,75276
 llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -26,7 +26,7 @@ scripts/start_vllm_server.py,sha256=ZepWp2y2cKFW0Kgsoima2RbmF02fTU29UFcLLpsBhFU,
 services/__init__.py,sha256=jGi9S74Msak3YR-C4Qb68VU7HB4oLaX9o1rlVAFpOFI,187
 services/pdf_processor.py,sha256=vu_JnhFGZY6jFVbDbPvG-mlQojvB-3Dzc8_946KVV2E,9427
 utils/__init__.py,sha256=1RrGoIIn1_gVzxd56b6a7HeAS-wu7uDP-nxLbR3fJ-8,1199
-utils/arxiv_utils.py,sha256=MxyD3Q0EzrmE0xORMJw8wdVtZ4Fp-ux_cn6jLMQimV8,18168
+utils/arxiv_utils.py,sha256=EzH1PhEAW0df5mmSP-kKHmuwqd4u2CSotRNwQ5IMJx8,19766
 utils/author_utils.py,sha256=DLTo1xsxef2wxoe4s_MWrh36maj4fgnvFlsDLpDE-qQ,5507
 utils/biblatex_parser.py,sha256=OkHXQcjiBrEDuhBfEk0RtmAYxufu5lAxAjb8__DzMjI,25537
 utils/bibliography_utils.py,sha256=mpmdAklzAs1CT3gqrOcjujGhouL95OuliCx0LE9Pg90,11705
@@ -34,13 +34,13 @@ utils/bibtex_parser.py,sha256=a89NLy_q2kwED4QFJgxWFgPQOJBV73bIUL3RS_Urmro,15231
 utils/config_validator.py,sha256=rxf7K3DYmJ-BNPsmtaCNipY2BTVT-pJZ7wN-M9Y3GC8,11167
 utils/db_utils.py,sha256=_wSupfBlm0ILFvntQTvoj7tLDCbrYPRQrp9NDvphF_E,6281
 utils/doi_utils.py,sha256=ezUiRnYRpoO0U_Rqgxv1FxqmeTwPh6X8gLgSDbqg5sY,4874
-utils/error_utils.py,sha256=iAUoTvA993zZnpKqKAIrX2vKEcp4YyQa3s6ubBakyW0,11625
+utils/error_utils.py,sha256=UJOH7Bp-rPV2JDY_XN38I2pSkqqPdnQoviKa4s4nK_A,12501
 utils/mock_objects.py,sha256=QxU-UXyHSY27IZYN8Sb8ei0JtNkpGSdMXoErrRLHXvE,6437
-utils/text_utils.py,sha256=8v20Fb_ziVZVnYhLWY4qHYAsC8ND3a2qNmOWr4cNr2c,211565
+utils/text_utils.py,sha256=g2r0QT6RGNi_8K5MD_EE-GT3cbffhk8cQyQaL6HSYtA,211955
 utils/unicode_utils.py,sha256=-WBKarXO756p7fd7gCeNsMag4ztDNURwFX5IVniOtwY,10366
 utils/url_utils.py,sha256=HdxIO8QvciP6Jp8Wd4sTSrS8JQrOMwgM7pxdUC8RJb4,9176
-academic_refchecker-1.2.48.dist-info/METADATA,sha256=h7fNPQOcbttBm4LimiqNXwAybBGV_-L8CNwHexS5-fs,22576
-academic_refchecker-1.2.48.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-academic_refchecker-1.2.48.dist-info/entry_points.txt,sha256=WdI89tYkIfz-M628PiboOfOLzTBWZAqvlF29qCVCkek,61
-academic_refchecker-1.2.48.dist-info/top_level.txt,sha256=6RlcQEA0kHb7-ndbKMFMZnYnJQVohgsU6BBkbEvJvEs,69
-academic_refchecker-1.2.48.dist-info/RECORD,,
+academic_refchecker-1.2.50.dist-info/METADATA,sha256=uOjNDL9zgwSxgFmaScRw635mQn0K4r2UfEYeQodcOy8,22576
+academic_refchecker-1.2.50.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+academic_refchecker-1.2.50.dist-info/entry_points.txt,sha256=WdI89tYkIfz-M628PiboOfOLzTBWZAqvlF29qCVCkek,61
+academic_refchecker-1.2.50.dist-info/top_level.txt,sha256=6RlcQEA0kHb7-ndbKMFMZnYnJQVohgsU6BBkbEvJvEs,69
+academic_refchecker-1.2.50.dist-info/RECORD,,

checkers/enhanced_hybrid_checker.py

@@ -312,12 +312,36 @@ class EnhancedHybridReferenceChecker:
         if (self.openreview and 
             hasattr(self.openreview, 'is_openreview_reference') and 
             self.openreview.is_openreview_reference(reference)):
+            logger.debug("Enhanced Hybrid: Trying OpenReview URL-based verification")
             verified_data, errors, url, success, failure_type = self._try_api('openreview', self.openreview, reference)
             if success:
                 return verified_data, errors, url
             if failure_type in ['throttled', 'timeout', 'server_error']:
                 failed_apis.append(('openreview', self.openreview, failure_type))
         
+        # Strategy 5b: Try OpenReview by search if venue suggests it might be there
+        elif (self.openreview and 
+              hasattr(self.openreview, 'verify_reference_by_search')):
+            # Check if venue suggests this might be on OpenReview
+            venue = reference.get('venue', reference.get('journal', '')).lower()
+            openreview_venues = [
+                'iclr', 'icml', 'neurips', 'nips', 'aaai', 'ijcai', 
+                'international conference on learning representations',
+                'international conference on machine learning',
+                'neural information processing systems'
+            ]
+            
+            venue_suggests_openreview = any(or_venue in venue for or_venue in openreview_venues)
+            logger.debug(f"Enhanced Hybrid: OpenReview venue check - venue: '{venue}', suggests: {venue_suggests_openreview}")
+            
+            if venue_suggests_openreview:
+                logger.debug("Enhanced Hybrid: Trying OpenReview search-based verification")
+                verified_data, errors, url, success, failure_type = self._try_openreview_search(reference)
+                if success:
+                    return verified_data, errors, url
+                if failure_type in ['throttled', 'timeout', 'server_error']:
+                    failed_apis.append(('openreview_search', self.openreview, failure_type))
+        
         # Strategy 6: Try CrossRef if we haven't already (for non-DOI references)
         if not self._should_try_doi_apis_first(reference) and self.crossref:
             verified_data, errors, url, success, failure_type = self._try_api('crossref', self.crossref, reference)
@@ -399,6 +423,66 @@ class EnhancedHybridReferenceChecker:
             'error_details': 'Could not verify reference using any available API'
         }], None
     
+    def _try_openreview_search(self, reference: Dict[str, Any]) -> Tuple[Optional[Dict[str, Any]], List[Dict[str, Any]], Optional[str], bool, str]:
+        """
+        Try to verify reference using OpenReview search
+        
+        Returns:
+            Tuple of (verified_data, errors, url, success, failure_type)
+        """
+        if not self.openreview:
+            return None, [], None, False, 'none'
+        
+        start_time = time.time()
+        failure_type = 'none'
+        
+        try:
+            verified_data, errors, url = self.openreview.verify_reference_by_search(reference)
+            duration = time.time() - start_time
+            
+            # Consider it successful if we found data or verification errors
+            success = verified_data is not None or len(errors) > 0
+            self._update_api_stats('openreview', success, duration)
+            
+            if success:
+                logger.debug(f"Enhanced Hybrid: OpenReview search successful in {duration:.2f}s, URL: {url}")
+                return verified_data, errors, url, True, 'none'
+            else:
+                logger.debug(f"Enhanced Hybrid: OpenReview search found no results in {duration:.2f}s")
+                return None, [], None, False, 'not_found'
+                
+        except requests.exceptions.Timeout as e:
+            duration = time.time() - start_time
+            self._update_api_stats('openreview', False, duration)
+            failure_type = 'timeout'
+            logger.debug(f"Enhanced Hybrid: OpenReview search timed out in {duration:.2f}s: {e}")
+            return None, [], None, False, failure_type
+            
+        except requests.exceptions.RequestException as e:
+            duration = time.time() - start_time
+            self._update_api_stats('openreview', False, duration)
+            
+            # Check if it's a rate limiting error
+            if hasattr(e, 'response') and e.response is not None:
+                if e.response.status_code in [429, 503]:
+                    failure_type = 'throttled'
+                elif e.response.status_code >= 500:
+                    failure_type = 'server_error'
+                else:
+                    failure_type = 'other'
+            else:
+                failure_type = 'other'
+            
+            logger.debug(f"Enhanced Hybrid: OpenReview search failed in {duration:.2f}s: {type(e).__name__}: {e}")
+            return None, [], None, False, failure_type
+            
+        except Exception as e:
+            duration = time.time() - start_time
+            self._update_api_stats('openreview', False, duration)
+            failure_type = 'other'
+            logger.debug(f"Enhanced Hybrid: OpenReview search error in {duration:.2f}s: {type(e).__name__}: {e}")
+            return None, [], None, False, failure_type
+    
     def get_performance_stats(self) -> Dict[str, Any]:
         """
         Get performance statistics for all APIs

checkers/openreview_checker.py

@@ -498,6 +498,160 @@ class OpenReviewReferenceChecker:
         logger.debug(f"OpenReview verification completed for: {openreview_url}")
         return verified_data, errors, openreview_url
     
+    def verify_by_search(self, reference: Dict[str, Any]) -> Tuple[Optional[Dict[str, Any]], List[Dict[str, Any]], Optional[str]]:
+        """
+        Verify a reference by searching OpenReview (when no URL is provided)
+        
+        Args:
+            reference: Reference dictionary with title, authors, year, etc.
+            
+        Returns:
+            Tuple of (verified_data, errors, paper_url) where:
+            - verified_data: Dict with verified OpenReview paper data or None
+            - errors: List of error/warning dictionaries  
+            - paper_url: The OpenReview URL if found
+        """
+        logger.debug(f"Searching OpenReview for reference: {reference.get('title', 'Untitled')}")
+        
+        title = reference.get('title', '').strip()
+        authors = reference.get('authors', [])
+        year = reference.get('year')
+        venue = reference.get('venue', '').strip()
+        
+        if not title:
+            return None, [], None
+        
+        # Check if venue suggests this might be on OpenReview
+        if not self._is_likely_openreview_venue(venue):
+            logger.debug(f"Venue '{venue}' doesn't suggest OpenReview, skipping search")
+            return None, [], None
+        
+        # Search for matching papers
+        search_results = self.search_paper(title, authors, year)
+        
+        if not search_results:
+            logger.debug("No matching papers found on OpenReview")
+            return None, [], None
+        
+        # Use the best match (first result, as they're sorted by relevance)
+        best_match = search_results[0]
+        paper_url = best_match.get('forum_url')
+        
+        logger.debug(f"Found OpenReview match: {best_match.get('title', 'Untitled')}")
+        
+        # Verify the reference against the found paper
+        errors = []
+        
+        # Check title match
+        cited_title = reference.get('title', '').strip()
+        paper_title = best_match.get('title', '').strip()
+        
+        if cited_title and paper_title:
+            similarity = calculate_title_similarity(cited_title, paper_title)
+            if similarity < 0.8:  # Slightly higher threshold for search results
+                from utils.error_utils import format_title_mismatch
+                details = format_title_mismatch(cited_title, paper_title) + f" (similarity: {similarity:.2f})"
+                errors.append({
+                    "warning_type": "title",
+                    "warning_details": details
+                })
+        
+        # Check authors
+        cited_authors = reference.get('authors', [])
+        paper_authors = best_match.get('authors', [])
+        
+        if cited_authors and paper_authors:
+            # Convert to list format if needed
+            if isinstance(cited_authors, str):
+                cited_authors = [author.strip() for author in cited_authors.split(',')]
+            if isinstance(paper_authors, str):
+                paper_authors = [author.strip() for author in paper_authors.split(',')]
+            
+            # Use the existing author comparison function
+            match, error_msg = compare_authors(cited_authors, paper_authors)
+            if not match and error_msg:
+                errors.append({
+                    "warning_type": "author", 
+                    "warning_details": error_msg
+                })
+        
+        # Check year
+        cited_year = reference.get('year')
+        paper_year = best_match.get('year')
+        
+        if cited_year and paper_year:
+            try:
+                cited_year_int = int(cited_year)
+                paper_year_int = int(paper_year)
+                
+                is_different, year_message = is_year_substantially_different(cited_year_int, paper_year_int)
+                if is_different and year_message:
+                    from utils.error_utils import format_year_mismatch
+                    errors.append({
+                        "warning_type": "year",
+                        "warning_details": format_year_mismatch(cited_year_int, paper_year_int)
+                    })
+            except (ValueError, TypeError):
+                pass  # Skip year validation if conversion fails
+        
+        # Check venue if provided in reference
+        cited_venue = reference.get('venue', '').strip()
+        paper_venue = best_match.get('venue', '').strip()
+        
+        if cited_venue and paper_venue:
+            if are_venues_substantially_different(cited_venue, paper_venue):
+                from utils.error_utils import format_venue_mismatch
+                errors.append({
+                    "warning_type": "venue",
+                    "warning_details": format_venue_mismatch(cited_venue, paper_venue)
+                })
+        
+        # Create verified data structure
+        verified_data = {
+            'title': best_match.get('title', cited_title),
+            'authors': best_match.get('authors', cited_authors),
+            'year': best_match.get('year', cited_year),
+            'venue': best_match.get('venue', cited_venue),
+            'url': paper_url,
+            'abstract': best_match.get('abstract', ''),
+            'keywords': best_match.get('keywords', []),
+            'openreview_metadata': best_match,
+            'verification_source': 'OpenReview (search)'
+        }
+        
+        logger.debug(f"OpenReview search verification completed for: {paper_url}")
+        return verified_data, errors, paper_url
+    
+    def _is_likely_openreview_venue(self, venue: str) -> bool:
+        """
+        Check if a venue suggests the paper might be on OpenReview
+        
+        Args:
+            venue: Venue string from reference
+            
+        Returns:
+            True if venue suggests OpenReview
+        """
+        if not venue:
+            return False
+        
+        venue_lower = venue.lower()
+        
+        # Common venues that use OpenReview
+        openreview_venues = [
+            'iclr', 'international conference on learning representations',
+            'neurips', 'neural information processing systems', 'nips',
+            'icml', 'international conference on machine learning',
+            'iclr workshop', 'neurips workshop', 'icml workshop',
+            'aaai', 'ijcai', 'aistats'
+        ]
+        
+        for or_venue in openreview_venues:
+            if or_venue in venue_lower:
+                return True
+        
+        return False
+    
     def search_paper(self, title: str, authors: List[str] = None, year: int = None) -> List[Dict[str, Any]]:
         """
         Search for papers on OpenReview by title, authors, and/or year
@@ -510,7 +664,316 @@ class OpenReviewReferenceChecker:
         Returns:
             List of matching paper metadata dictionaries
         """
-        # This would implement search functionality if needed
-        # For now, OpenReview verification is primarily URL-based
-        logger.debug(f"Search functionality not yet implemented for OpenReview")
-        return []
+        if not title or not title.strip():
+            return []
+            
+        logger.debug(f"Searching OpenReview for: {title}")
+        
+        # Clean title for search
+        search_title = clean_title_for_search(title)
+        
+        # Try API search first
+        results = self._search_via_api(search_title, authors, year)
+        if results:
+            return results
+        
+        # If API search fails, try web search as fallback
+        return self._search_via_web(search_title, authors, year)
+    
+    def _search_via_api(self, title: str, authors: List[str] = None, year: int = None) -> List[Dict[str, Any]]:
+        """
+        Search using OpenReview API
+        
+        Args:
+            title: Clean title to search for
+            authors: List of author names (optional)
+            year: Publication year (optional)
+            
+        Returns:
+            List of matching paper dictionaries
+        """
+        try:
+            # The OpenReview API requires specific parameters
+            # We'll search by content.title or content.venue (for venue-based search)
+            search_params = {
+                'limit': 20,  # Limit results to avoid overwhelming the API
+                'details': 'directReplies'  # Get basic details
+            }
+            
+            # Try searching by venue first if year suggests recent conferences
+            if year and year >= 2017:  # OpenReview started around 2017
+                venues_by_year = {
+                    2025: ['ICLR 2025'],
+                    2024: ['ICLR 2024', 'NeurIPS 2024', 'ICML 2024'],
+                    2023: ['ICLR 2023', 'NeurIPS 2023', 'ICML 2023'],
+                    2022: ['ICLR 2022', 'NeurIPS 2022', 'ICML 2022'],
+                    2021: ['ICLR 2021', 'NeurIPS 2021', 'ICML 2021'],
+                    2020: ['ICLR 2020', 'NeurIPS 2020', 'ICML 2020'],
+                    2019: ['ICLR 2019', 'NeurIPS 2019', 'ICML 2019'],
+                    2018: ['ICLR 2018', 'NeurIPS 2018', 'ICML 2018'],
+                    2017: ['ICLR 2017']
+                }
+                
+                possible_venues = venues_by_year.get(year, [])
+                
+                results = []
+                for venue in possible_venues:
+                    # Search by venue and then filter by title
+                    venue_params = search_params.copy()
+                    venue_params['content.venue'] = venue
+                    
+                    api_url = f"{self.api_url}/notes"
+                    response = self._respectful_request(api_url, params=venue_params)
+                    
+                    if response and response.status_code == 200:
+                        try:
+                            data = response.json()
+                            if 'notes' in data and data['notes']:
+                                for note in data['notes']:
+                                    try:
+                                        metadata = self._parse_api_response(note)
+                                        if metadata and self._is_good_match(metadata, title, authors, year):
+                                            results.append(metadata)
+                                            if len(results) >= 5:  # Limit results
+                                                break
+                                    except Exception as e:
+                                        logger.debug(f"Error parsing note: {e}")
+                                        continue
+                                
+                                if results:
+                                    break  # Found results, no need to search other venues
+                                    
+                        except (json.JSONDecodeError, KeyError) as e:
+                            logger.debug(f"Failed to parse venue search response: {e}")
+                            continue
+                    else:
+                        logger.debug(f"Venue search failed for {venue}: {response.status_code if response else 'No response'}")
+                
+                if results:
+                    logger.debug(f"OpenReview API search found {len(results)} matches via venue search")
+                    return results
+            
+            # If venue search didn't work, try other approaches
+            # OpenReview API is quite restrictive, so we might need to fall back to web scraping
+            logger.debug("OpenReview API venue search returned no results, trying web search")
+            return []
+            
+        except Exception as e:
+            logger.debug(f"OpenReview API search error: {e}")
+            return []
+    
+    def _search_via_web(self, title: str, authors: List[str] = None, year: int = None) -> List[Dict[str, Any]]:
+        """
+        Search using OpenReview web interface (fallback)
+        
+        Args:
+            title: Clean title to search for
+            authors: List of author names (optional)
+            year: Publication year (optional)
+            
+        Returns:
+            List of matching paper dictionaries
+        """
+        try:
+            # Build search URL
+            search_query = title.replace(' ', '+')
+            search_url = f"{self.base_url}/search?term={search_query}"
+            
+            response = self._respectful_request(search_url)
+            if not response or response.status_code != 200:
+                return []
+            
+            # Parse search results page
+            soup = BeautifulSoup(response.text, 'html.parser')
+            
+            # Look for paper links in search results
+            # OpenReview search results typically contain links to forum pages
+            results = []
+            
+            # Find links that look like OpenReview paper URLs
+            for link in soup.find_all('a', href=True):
+                href = link.get('href', '')
+                if '/forum?id=' in href:
+                    paper_id = self.extract_paper_id(href)
+                    if paper_id:
+                        # Get full metadata for this paper
+                        metadata = self.get_paper_metadata(paper_id)
+                        if metadata and self._is_good_match(metadata, title, authors, year):
+                            results.append(metadata)
+                            if len(results) >= 5:  # Limit results
+                                break
+            
+            logger.debug(f"OpenReview web search found {len(results)} matches")
+            return results
+            
+        except Exception as e:
+            logger.debug(f"OpenReview web search error: {e}")
+            return []
+    
+    def _is_good_match(self, metadata: Dict[str, Any], search_title: str, authors: List[str] = None, year: int = None) -> bool:
+        """
+        Check if the found paper is a good match for the search criteria
+        
+        Args:
+            metadata: Paper metadata from OpenReview
+            search_title: Title we're searching for
+            authors: Authors we're looking for (optional)
+            year: Year we're looking for (optional)
+            
+        Returns:
+            True if it's a good match
+        """
+        paper_title = metadata.get('title', '')
+        if not paper_title:
+            return False
+        
+        # Check title similarity
+        title_similarity = calculate_title_similarity(search_title, paper_title)
+        if title_similarity < 0.7:  # Require at least 70% similarity
+            return False
+        
+        # Check year if provided
+        if year:
+            paper_year = metadata.get('year')
+            if paper_year and abs(int(paper_year) - year) > 1:  # Allow 1 year difference
+                return False
+        
+        # Check authors if provided
+        if authors and len(authors) > 0:
+            paper_authors = metadata.get('authors', [])
+            if paper_authors:
+                # Check if at least one author matches
+                author_match = False
+                for search_author in authors[:2]:  # Check first 2 authors
+                    for paper_author in paper_authors[:3]:  # Check first 3 paper authors
+                        if is_name_match(search_author, paper_author):
+                            author_match = True
+                            break
+                    if author_match:
+                        break
+                
+                if not author_match:
+                    return False
+        
+        return True
+
+    def search_by_title(self, title: str, max_results: int = 5) -> List[Dict[str, Any]]:
+        """
+        Search OpenReview for papers by title using the working search API.
+        
+        Args:
+            title: Paper title to search for
+            max_results: Maximum number of results to return
+            
+        Returns:
+            List of paper data dictionaries
+        """
+        try:
+            # Use OpenReview's search API with term parameter (this works!)
+            params = {
+                'term': title,
+                'limit': max_results
+            }
+            
+            response = self._respectful_request(f"{self.api_url}/notes/search", params=params)
+            if not response or response.status_code != 200:
+                logger.debug(f"OpenReview search API failed with status {response.status_code if response else 'None'}")
+                return []
+            
+            data = response.json()
+            papers = []
+            
+            for note in data.get('notes', []):
+                # Filter to exact or close title matches
+                note_title = note.get('content', {}).get('title', '')
+                if self._is_title_match(title, note_title):
+                    paper_data = self._parse_api_response(note)
+                    if paper_data:
+                        papers.append(paper_data)
+            
+            logger.debug(f"OpenReview search found {len(papers)} matching papers for '{title}'")
+            return papers
+            
+        except Exception as e:
+            logger.error(f"Error searching OpenReview by title '{title}': {e}")
+            return []
+
+    def _is_title_match(self, search_title: str, found_title: str, threshold: float = 0.8) -> bool:
+        """
+        Check if two titles match closely enough.
+        
+        Args:
+            search_title: Title we're searching for
+            found_title: Title found in search results
+            threshold: Similarity threshold (0.0 to 1.0)
+            
+        Returns:
+            True if titles match closely enough
+        """
+        if not search_title or not found_title:
+            return False
+        
+        # Exact match
+        if search_title.lower().strip() == found_title.lower().strip():
+            return True
+        
+        # Check if one contains the other (for cases where one is longer)
+        search_clean = search_title.lower().strip()
+        found_clean = found_title.lower().strip()
+        
+        if search_clean in found_clean or found_clean in search_clean:
+            return True
+        
+        # Use similarity calculation from text_utils
+        try:
+            from utils.text_utils import calculate_title_similarity
+            similarity = calculate_title_similarity(search_title, found_title)
+            return similarity >= threshold
+        except ImportError:
+            # Fallback to simple word matching
+            search_words = set(search_clean.split())
+            found_words = set(found_clean.split())
+            
+            if not search_words or not found_words:
+                return False
+            
+            intersection = search_words.intersection(found_words)
+            union = search_words.union(found_words)
+            
+            jaccard_similarity = len(intersection) / len(union) if union else 0
+            return jaccard_similarity >= threshold
+
+    def verify_reference_by_search(self, reference: Dict[str, Any]) -> Tuple[Optional[Dict[str, Any]], List[Dict[str, Any]], Optional[str]]:
+        """
+        Verify a reference by searching OpenReview (for papers without URLs).
+        
+        Args:
+            reference: Reference data dictionary
+            
+        Returns:
+            Tuple of (verified_data, errors_and_warnings, debug_info)
+        """
+        title = reference.get('title', '').strip()
+        if not title:
+            return None, [], "No title provided for search"
+        
+        # Search for the paper
+        search_results = self.search_by_title(title)
+        
+        if not search_results:
+            return None, [], f"No papers found on OpenReview for title: {title}"
+        
+        # Take the best match (first result, as search is already filtered)
+        best_match = search_results[0]
+        
+        # Use the existing verify_reference method with the found URL
+        forum_url = best_match.get('forum_url')
+        if forum_url:
+            # Create a reference with the OpenReview URL for verification
+            reference_with_url = reference.copy()
+            reference_with_url['url'] = forum_url
+            
+            return self.verify_reference(reference_with_url)
+        
+        # If no URL, return the metadata as verification
+        return best_match, [], f"Found on OpenReview: {best_match.get('title')}"

checkers/semantic_scholar.py

@@ -583,8 +583,8 @@ class NonArxivReferenceChecker:
             
             if not (has_arxiv_url or has_arxiv_doi):
                 errors.append({
-                    'warning_type': 'url',
-                    'warning_details': f"Reference could include arXiv URL: {arxiv_url}",
+                    'info_type': 'url',
+                    'info_details': f"Reference could include arXiv URL: {arxiv_url}",
                     'ref_url_correct': arxiv_url
                 })
 

core/parallel_processor.py

@@ -340,7 +340,7 @@ class ParallelReferenceProcessor:
         # Display errors and warnings
         if result.errors:
             # Check if there's an unverified error
-            has_unverified_error = any(e.get('error_type') == 'unverified' or e.get('warning_type') == 'unverified' for e in result.errors)
+            has_unverified_error = any(e.get('error_type') == 'unverified' or e.get('warning_type') == 'unverified' or e.get('info_type') == 'unverified' for e in result.errors)
             
             if has_unverified_error:
                 # Use the centralized unverified error display function from base checker
@@ -348,9 +348,9 @@ class ParallelReferenceProcessor:
             
             # Display all non-unverified errors and warnings
             for error in result.errors:
-                if error.get('error_type') != 'unverified' and error.get('warning_type') != 'unverified':
-                    error_type = error.get('error_type') or error.get('warning_type')
-                    error_details = error.get('error_details') or error.get('warning_details', 'Unknown error')
+                if error.get('error_type') != 'unverified' and error.get('warning_type') != 'unverified' and error.get('info_type') != 'unverified':
+                    error_type = error.get('error_type') or error.get('warning_type') or error.get('info_type')
+                    error_details = error.get('error_details') or error.get('warning_details') or error.get('info_details', 'Unknown error')
                     
                     from utils.error_utils import print_labeled_multiline
 
@@ -359,8 +359,10 @@ class ParallelReferenceProcessor:
                         print(f"      ❌ {error_details}")
                     elif 'error_type' in error:
                         print_labeled_multiline("❌ Error", error_details)
-                    else:
+                    elif 'warning_type' in error:
                         print_labeled_multiline("⚠️  Warning", error_details)
+                    else:
+                        print_labeled_multiline("ℹ️  Information", error_details)
         
         # Show timing info for slow references
         if result.processing_time > 5.0:

core/refchecker.py

@@ -2033,6 +2033,9 @@ class ArxivReferenceChecker:
                     elif 'warning_type' in error:
                         formatted_error['warning_type'] = error['warning_type']
                         formatted_error['warning_details'] = error['warning_details']
+                    elif 'info_type' in error:
+                        formatted_error['info_type'] = error['info_type']
+                        formatted_error['info_details'] = error['info_details']
                     
                     # Add correct information based on error type
                     if error.get('error_type') == 'author':
@@ -2042,6 +2045,8 @@ class ArxivReferenceChecker:
                     elif error.get('error_type') == 'doi':
                         from utils.doi_utils import construct_doi_url
                         formatted_error['ref_url_correct'] = construct_doi_url(error.get('ref_doi_correct', ''))
+                    elif error.get('info_type') == 'url':
+                        formatted_error['ref_url_correct'] = error.get('ref_url_correct', '')
                     
                     formatted_errors.append(formatted_error)
                 
@@ -2153,17 +2158,22 @@ class ArxivReferenceChecker:
             for error in errors:
                 formatted_error = {}
                 
-                # Handle error_type and warning_type properly
+                # Handle error_type, warning_type, and info_type properly
                 if 'error_type' in error:
                     formatted_error['error_type'] = error['error_type']
                     formatted_error['error_details'] = error['error_details']
                 elif 'warning_type' in error:
                     formatted_error['warning_type'] = error['warning_type']
                     formatted_error['warning_details'] = error['warning_details']
+                elif 'info_type' in error:
+                    formatted_error['info_type'] = error['info_type']
+                    formatted_error['info_details'] = error['info_details']
                 
                 # Add correct information based on error type
                 if error.get('warning_type') == 'year':
                     formatted_error['ref_year_correct'] = error.get('ref_year_correct', '')
+                elif error.get('info_type') == 'url':
+                    formatted_error['ref_url_correct'] = error.get('ref_url_correct', '')
                 
                 formatted_errors.append(formatted_error)
             
@@ -2214,13 +2224,16 @@ class ArxivReferenceChecker:
             for error in errors:
                 formatted_error = {}
                 
-                # Handle error_type and warning_type properly
+                # Handle error_type, warning_type, and info_type properly
                 if 'error_type' in error:
                     formatted_error['error_type'] = error['error_type']
                     formatted_error['error_details'] = error['error_details']
                 elif 'warning_type' in error:
                     formatted_error['warning_type'] = error['warning_type']
                     formatted_error['warning_details'] = error['warning_details']
+                elif 'info_type' in error:
+                    formatted_error['info_type'] = error['info_type']
+                    formatted_error['info_details'] = error['info_details']
                 
                 formatted_errors.append(formatted_error)
             
@@ -2335,13 +2348,16 @@ class ArxivReferenceChecker:
             logger.debug(f"DEBUG: Error {i}: {error}")
             formatted_error = {}
             
-            # Handle error_type and warning_type properly
+            # Handle error_type, warning_type, and info_type properly
             if 'error_type' in error:
                 formatted_error['error_type'] = error['error_type']
                 formatted_error['error_details'] = error['error_details']
             elif 'warning_type' in error:
                 formatted_error['warning_type'] = error['warning_type']
                 formatted_error['warning_details'] = error['warning_details']
+            elif 'info_type' in error:
+                formatted_error['info_type'] = error['info_type']
+                formatted_error['info_details'] = error['info_details']
             
             # Add correct information based on error type
             if error.get('error_type') == 'author':
@@ -2657,8 +2673,8 @@ class ArxivReferenceChecker:
         else:
             # Single error - handle as before
             error = errors[0]
-            error_type = error.get('error_type') or error.get('warning_type', 'unknown')
-            error_details = error.get('error_details') or error.get('warning_details', '')
+            error_type = error.get('error_type') or error.get('warning_type') or error.get('info_type', 'unknown')
+            error_details = error.get('error_details') or error.get('warning_details') or error.get('info_details', '')
             
             error_entry = {
                 # Source paper metadata
@@ -2775,7 +2791,9 @@ class ArxivReferenceChecker:
                         emoji = "❓"
                     elif error_type in ['year', 'venue']:  # Warning types
                         emoji = "⚠️"
-                    else:  # Error types (title, author, doi, url, multiple, etc.)
+                    elif error_type == 'url':  # Info type (ArXiv URL suggestion)
+                        emoji = "ℹ️"
+                    else:  # Error types (title, author, doi, multiple, etc.)
                         emoji = "❌"
                     
                     f.write(f"Type: {emoji} {error_entry['error_type']}\n")
@@ -2906,8 +2924,10 @@ class ArxivReferenceChecker:
         self.total_references_processed = 0
         self.papers_with_errors = 0
         self.papers_with_warnings = 0
+        self.papers_with_info = 0
         self.total_errors_found = 0
         self.total_warnings_found = 0
+        self.total_info_found = 0
         self.total_arxiv_refs = 0
         self.total_non_arxiv_refs = 0
         self.total_other_refs = 0
@@ -3066,18 +3086,21 @@ class ArxivReferenceChecker:
                         # Separate actual errors from warnings for paper classification
                         actual_errors = [e for e in paper_errors if 'error_type' in e and e['error_type'] != 'unverified']
                         warnings_only = [e for e in paper_errors if 'warning_type' in e]
+                        info_only = [e for e in paper_errors if 'info_type' in e]
                         
                         if self.single_paper_mode:
                             # Single paper mode - show simple summary
-                            if actual_errors or warnings_only:
+                            if actual_errors or warnings_only or info_only:
                                 summary_parts = []
                                 if actual_errors:
                                     summary_parts.append(f"{len(actual_errors)} errors")
                                 if warnings_only:
                                     summary_parts.append(f"{len(warnings_only)} warnings")
+                                if info_only:
+                                    summary_parts.append(f"{len(info_only)} information")
                         else:
                             # Multi-paper mode - track paper statistics
-                            if actual_errors or warnings_only:
+                            if actual_errors or warnings_only or info_only:
                                 summary_parts = []
                                 if actual_errors:
                                     summary_parts.append(f"{len(actual_errors)} errors")
@@ -3086,6 +3109,10 @@ class ArxivReferenceChecker:
                                     summary_parts.append(f"{len(warnings_only)} warnings")
                                     # Count as paper with warnings if it has warnings (regardless of errors)
                                     self.papers_with_warnings += 1
+                                if info_only:
+                                    summary_parts.append(f"{len(info_only)} information")
+                                    # Count as paper with info if it has info messages (regardless of errors/warnings)
+                                    self.papers_with_info += 1
 
                 except Exception as e:
                     logger.error(f"Error processing paper {paper_id}: {str(e)}")
@@ -3127,9 +3154,11 @@ class ArxivReferenceChecker:
                     print(f"❌ Total errors: {self.total_errors_found}")
                 if self.total_warnings_found > 0:
                     print(f"⚠️  Total warnings: {self.total_warnings_found}")
+                if self.total_info_found > 0:
+                    print(f"ℹ️  Total information: {self.total_info_found}")
                 if self.total_unverified_refs > 0:
                     print(f"❓ References that couldn't be verified: {self.total_unverified_refs}")
-                if self.total_errors_found == 0 and self.total_warnings_found == 0 and self.total_unverified_refs == 0:
+                if self.total_errors_found == 0 and self.total_warnings_found == 0 and self.total_info_found == 0 and self.total_unverified_refs == 0:
                     print(f"✅ All references verified successfully!")
                 
                 # Show warning if unreliable extraction was used and there are many errors
@@ -3149,6 +3178,8 @@ class ArxivReferenceChecker:
                 print(f"         Total errors:   {self.total_errors_found}")
                 print(f"⚠️  Papers with warnings: {self.papers_with_warnings}")
                 print(f"         Total warnings: {self.total_warnings_found}")
+                print(f"ℹ️  Papers with information: {self.papers_with_info}")
+                print(f"         Total information: {self.total_info_found}")
                 print(f"❓ References that couldn't be verified: {self.total_unverified_refs}")
                 
                 # Show warning if unreliable extraction was used and there are many errors
@@ -5348,7 +5379,7 @@ class ArxivReferenceChecker:
         # If errors found, add to dataset and optionally print details
         if errors:
             # Check if there's an unverified error among the errors
-            has_unverified_error = any(e.get('error_type') == 'unverified' or e.get('warning_type') == 'unverified' for e in errors)
+            has_unverified_error = any(e.get('error_type') == 'unverified' or e.get('warning_type') == 'unverified' or e.get('info_type') == 'unverified' for e in errors)
             
             if has_unverified_error:
                 self.total_unverified_refs += 1
@@ -5358,11 +5389,13 @@ class ArxivReferenceChecker:
             self.add_error_to_dataset(paper, reference, errors, reference_url, verified_data)
             paper_errors.extend(errors)
             
-            # Count errors vs warnings
+            # Count errors vs warnings vs info
             error_count = sum(1 for e in errors if 'error_type' in e and e['error_type'] != 'unverified')
             warning_count = sum(1 for e in errors if 'warning_type' in e)
+            info_count = sum(1 for e in errors if 'info_type' in e)
             self.total_errors_found += error_count
             self.total_warnings_found += warning_count
+            self.total_info_found += info_count
             
             # Display all non-unverified errors and warnings
             self._display_non_unverified_errors(errors, debug_mode, print_output)
@@ -5509,9 +5542,9 @@ class ArxivReferenceChecker:
         """Display all non-unverified errors and warnings"""
         if not debug_mode and print_output:
             for error in errors:
-                if error.get('error_type') != 'unverified' and error.get('warning_type') != 'unverified':
-                    error_type = error.get('error_type') or error.get('warning_type')
-                    error_details = error.get('error_details') or error.get('warning_details', 'Unknown error')
+                if error.get('error_type') != 'unverified' and error.get('warning_type') != 'unverified' and error.get('info_type') != 'unverified':
+                    error_type = error.get('error_type') or error.get('warning_type') or error.get('info_type')
+                    error_details = error.get('error_details') or error.get('warning_details') or error.get('info_details', 'Unknown error')
                     
                     from utils.error_utils import print_labeled_multiline
 
@@ -5519,8 +5552,10 @@ class ArxivReferenceChecker:
                         print(f"      ❌ {error_details}")
                     elif 'error_type' in error:
                         print_labeled_multiline("❌ Error", error_details)
-                    else:
+                    elif 'warning_type' in error:
                         print_labeled_multiline("⚠️  Warning", error_details)
+                    else:
+                        print_labeled_multiline("ℹ️  Information", error_details)
 
     def _output_reference_errors(self, reference, errors, url):
         """

utils/arxiv_utils.py

@@ -111,56 +111,8 @@ def download_arxiv_source(arxiv_id):
                     main_tex_content = largest_file[1]
                     logger.debug(f"Using largest tex file: {largest_file[0]}")
             
-            # Find which .bib files are actually referenced in the main tex file
-            bib_content = None
-            if bib_files and main_tex_content:
-                # Extract bibliography references from main tex file
-                referenced_bibs = []
-                bib_pattern = r'\\bibliography\{([^}]+)\}'
-                matches = re.findall(bib_pattern, main_tex_content)
-                
-                for match in matches:
-                    # Handle multiple bib files separated by commas
-                    bib_names = [name.strip() for name in match.split(',')]
-                    for bib_name in bib_names:
-                        # Add .bib extension if not present
-                        if not bib_name.endswith('.bib'):
-                            bib_name += '.bib'
-                        referenced_bibs.append(bib_name)
-                
-                # Use only referenced .bib files, or all if no references found
-                if referenced_bibs:
-                    used_bibs = []
-                    for bib_name in referenced_bibs:
-                        if bib_name in bib_files:
-                            used_bibs.append(bib_files[bib_name])
-                            logger.debug(f"Using referenced .bib file: {bib_name}")
-                        else:
-                            logger.debug(f"Referenced .bib file not found: {bib_name}")
-                    
-                    if used_bibs:
-                        raw_bib_content = '\n\n'.join(used_bibs)
-                        
-                        # Filter BibTeX to only include cited references
-                        bib_content = filter_bibtex_by_citations(raw_bib_content, tex_files, main_tex_content)
-                        
-                        logger.debug(f"Found {len(used_bibs)} referenced .bib files out of {len(bib_files)} total")
-                    else:
-                        # Fallback to all bib files if none of the referenced ones found
-                        raw_bib_content = '\n\n'.join(bib_files.values())
-                        bib_content = filter_bibtex_by_citations(raw_bib_content, tex_files, main_tex_content)
-                        logger.debug(f"No referenced .bib files found, using all {len(bib_files)} .bib files")
-                else:
-                    # No \bibliography command found, use all bib files
-                    raw_bib_content = '\n\n'.join(bib_files.values())
-                    bib_content = filter_bibtex_by_citations(raw_bib_content, tex_files, main_tex_content)
-                    logger.debug(f"No \\bibliography command found, using all {len(bib_files)} .bib files")
-            elif bib_files:
-                # No main tex file but have bib files
-                raw_bib_content = '\n\n'.join(bib_files.values())
-                # Can't filter without tex files, so use original content
-                bib_content = raw_bib_content
-                logger.debug(f"Found {len(bib_files)} .bib files (no main tex to filter)")
+            # Process .bib files using shared logic
+            bib_content = select_and_filter_bib_files(bib_files, main_tex_content, tex_files)
             
             # Combine all bbl file contents  
             bbl_content = None
@@ -219,6 +171,78 @@ def download_arxiv_bibtex(arxiv_id):
         return None
 
 
+def select_and_filter_bib_files(bib_files, main_tex_content, tex_files):
+    """
+    Select appropriate .bib files based on main TeX file references and filter by citations.
+    
+    Args:
+        bib_files: Dict of .bib files {filename: content}
+        main_tex_content: Content of main tex file
+        tex_files: Dict of all tex files {filename: content} (for filtering)
+        
+    Returns:
+        Filtered BibTeX content or None if no files available
+    """
+    import re
+    
+    if not bib_files:
+        return None
+        
+    if main_tex_content:
+        # Extract bibliography references from main tex file
+        referenced_bibs = []
+        bib_pattern = r'\\bibliography\{([^}]+)\}'
+        matches = re.findall(bib_pattern, main_tex_content)
+        
+        for match in matches:
+            # Handle multiple bib files separated by commas
+            bib_names = [name.strip() for name in match.split(',')]
+            for bib_name in bib_names:
+                # Add .bib extension if not present
+                if not bib_name.endswith('.bib'):
+                    bib_name += '.bib'
+                referenced_bibs.append(bib_name)
+        
+        # Use only referenced .bib files, or all if no references found
+        if referenced_bibs:
+            used_bibs = []
+            seen_bib_names = set()  # Track which bib files we've already added
+            for bib_name in referenced_bibs:
+                if bib_name in bib_files and bib_name not in seen_bib_names:
+                    used_bibs.append(bib_files[bib_name])
+                    seen_bib_names.add(bib_name)
+                    logger.debug(f"Using referenced .bib file: {bib_name}")
+                elif bib_name in seen_bib_names:
+                    logger.debug(f"Skipping duplicate .bib file: {bib_name}")
+                else:
+                    logger.debug(f"Referenced .bib file not found: {bib_name}")
+            
+            if used_bibs:
+                raw_bib_content = '\n\n'.join(used_bibs)
+                # Filter BibTeX to only include cited references
+                filtered_content = filter_bibtex_by_citations(raw_bib_content, tex_files, main_tex_content)
+                logger.debug(f"Found {len(used_bibs)} referenced .bib files out of {len(bib_files)} total")
+                return filtered_content
+            else:
+                # Fallback to all bib files if none of the referenced ones found
+                raw_bib_content = '\n\n'.join(bib_files.values())
+                filtered_content = filter_bibtex_by_citations(raw_bib_content, tex_files, main_tex_content)
+                logger.debug(f"No referenced .bib files found, using all {len(bib_files)} .bib files")
+                return filtered_content
+        else:
+            # No \bibliography command found, use all bib files
+            raw_bib_content = '\n\n'.join(bib_files.values())
+            filtered_content = filter_bibtex_by_citations(raw_bib_content, tex_files, main_tex_content)
+            logger.debug(f"No \\bibliography command found, using all {len(bib_files)} .bib files")
+            return filtered_content
+    else:
+        # No main tex file but have bib files
+        raw_bib_content = '\n\n'.join(bib_files.values())
+        # Can't filter without tex files, so use original content
+        logger.debug(f"Found {len(bib_files)} .bib files (no main tex to filter)")
+        return raw_bib_content
+
+
 def extract_cited_keys_from_tex(tex_files, main_tex_content):
     """
     Extract all citation keys from TeX files.
@@ -261,7 +285,11 @@ def is_reference_used(reference_key, cited_keys):
     Returns:
         True if the reference is cited, False otherwise
     """
-    return reference_key in cited_keys
+    result = reference_key in cited_keys
+    # Add debugging for the first few mismatches to understand the issue
+    if not result and len([k for k in cited_keys if k.startswith('a')]) < 3:  # Limit debug output
+        logger.debug(f"Key '{reference_key}' not found in cited_keys")
+    return result
 
 
 def filter_bibtex_by_citations(bib_content, tex_files, main_tex_content):
@@ -291,14 +319,30 @@ def filter_bibtex_by_citations(bib_content, tex_files, main_tex_content):
         from utils.bibtex_parser import parse_bibtex_entries
         entries = parse_bibtex_entries(bib_content)
         
-        # Filter entries to only cited ones
+        # Filter entries to only cited ones and remove duplicates
         cited_entries = []
+        seen_keys = set()
+        not_cited_count = 0
+        duplicate_count = 0
+        
         for entry in entries:
             entry_key = entry.get('key', '')
             if is_reference_used(entry_key, cited_keys):
-                cited_entries.append(entry)
+                if entry_key not in seen_keys:
+                    cited_entries.append(entry)
+                    seen_keys.add(entry_key)
+                else:
+                    duplicate_count += 1
+                    logger.debug(f"Skipping duplicate entry: '{entry_key}'")
+            else:
+                not_cited_count += 1
+                # Log first few entries that are NOT cited for debugging
+                if not_cited_count <= 5:
+                    logger.debug(f"Entry NOT cited: '{entry_key}'")
                 
-        logger.debug(f"Filtered BibTeX: {len(entries)} total -> {len(cited_entries)} cited")
+        logger.debug(f"Filtered BibTeX: {len(entries)} total -> {len(cited_entries)} cited (removed {duplicate_count} duplicates)")
+        logger.debug(f"Citation keys found: {len(cited_keys)} keys")
+        logger.debug(f"Sample cited keys: {list(cited_keys)[:10]}")
         
         # Reconstruct BibTeX content from cited entries
         if not cited_entries:
@@ -392,41 +436,43 @@ def get_bibtex_content(paper):
         logger.debug(f"Detected ArXiv paper {arxiv_id}, checking for structured bibliography")
         tex_content, bib_content, bbl_content = download_arxiv_source(arxiv_id)
         
-        # Choose between .bib and .bbl files - .bbl files take priority when they contain entries
-        # .bbl files are processed biblatex output that reflects exactly what was cited
+        # Choose between .bib and .bbl files based on what the main TeX file actually uses
+        # Check the main TeX file to see if it uses \bibliography{...} (BibTeX) or not (BBL)
+        uses_bibtex = False
+        if tex_content:
+            # Look for \bibliography{...} commands in the main TeX file
+            bib_pattern = r'\\bibliography\{([^}]+)\}'
+            bib_matches = re.findall(bib_pattern, tex_content)
+            if bib_matches:
+                uses_bibtex = True
+                referenced_bibs = []
+                for match in bib_matches:
+                    bib_names = [name.strip() for name in match.split(',')]
+                    referenced_bibs.extend(bib_names)
+                logger.debug(f"Main TeX file references BibTeX files: {referenced_bibs}")
+        
         if bib_content and bbl_content:
             # Count entries in both for logging
             bib_entry_count = len(re.findall(r'@\w+\s*\{', bib_content))
-            bbl_entry_count = len(re.findall(r'\\bibitem\[', bbl_content))
+            bbl_entry_count = len(re.findall(r'\\bibitem[\[\{]', bbl_content))
             
             logger.debug(f"Bibliography comparison: .bbl has {bbl_entry_count} entries, .bib has {bib_entry_count} entries")
             
-            # Only use .bbl if it actually contains bibliography entries
-            if bbl_entry_count > 0:
-                logger.info(f"Using .bbl files from ArXiv source (biblatex takes priority over bibtex)")
+            if uses_bibtex and bib_entry_count > 0:
+                logger.info(f"Using .bib files from ArXiv source (main TeX uses \\bibliography{{...}})")
+                return bib_content
+            elif bbl_entry_count > 0:
+                logger.info(f"Using .bbl files from ArXiv source (main TeX doesn't use \\bibliography or .bib is empty)")
                 return bbl_content
-            else:
+            elif bib_entry_count > 0:
                 logger.info(f"Using .bib files from ArXiv source (.bbl file is empty)")
-                # If we have LaTeX content, filter BibTeX by cited keys
-                if tex_content:
-                    cited_keys = extract_cited_keys_from_tex({}, tex_content)
-                    if cited_keys:
-                        logger.debug(f"Found {len(cited_keys)} cited keys, filtering BibTeX")
-                        filtered_content = filter_bibtex_by_citations(bib_content, {}, tex_content)
-                        return filtered_content
                 return bib_content
+            else:
+                logger.warning(f"Both .bib and .bbl files appear to be empty")
+                return bib_content  # Default to bib_content as fallback
                     
         elif bib_content:
             logger.info(f"Found .bib files in ArXiv source for {arxiv_id}")
-            
-            # If we have LaTeX content, filter BibTeX by cited keys
-            if tex_content:
-                cited_keys = extract_cited_keys_from_tex({}, tex_content)
-                if cited_keys:
-                    logger.debug(f"Found {len(cited_keys)} cited keys, filtering BibTeX")
-                    filtered_content = filter_bibtex_by_citations(bib_content, {}, tex_content)
-                    return filtered_content
-            
             return bib_content
             
         elif bbl_content:

utils/error_utils.py

@@ -294,6 +294,39 @@ def create_generic_warning(warning_type: str, warning_details: str, **kwargs) ->
     return warning_dict
 
 
+def create_generic_info(info_type: str, info_details: str, **kwargs) -> Dict[str, Any]:
+    """
+    Create a generic info dictionary with custom fields.
+    
+    Args:
+        info_type: Type of info (e.g., 'url')
+        info_details: Description of the information
+        **kwargs: Additional fields to include in the info dictionary
+        
+    Returns:
+        Standardized info dictionary
+    """
+    info_dict = {
+        'info_type': info_type,
+        'info_details': info_details
+    }
+    
+    info_dict.update(kwargs)
+    return info_dict
+
+
+def create_info_message(reference, reason, arxiv_url=None):
+    """Create a standardized info message structure."""
+    info_msg = {
+        'info_type': 'arxiv_url_available',
+        'reference': reference,
+        'reason': reason
+    }
+    if arxiv_url:
+        info_msg['arxiv_url'] = arxiv_url
+    return info_msg
+
+
 def format_author_mismatch(author_number: int, cited_author: str, correct_author: str) -> str:
     """
     Format a three-line author mismatch message.

utils/text_utils.py

@@ -2337,6 +2337,9 @@ def format_author_for_display(author_name):
     if not author_name:
         return author_name
     
+    # First clean the author name to remove asterisks and other unwanted characters
+    author_name = clean_author_name(author_name)
+    
     # Clean up any stray punctuation that might have been attached during parsing
     author_name = author_name.strip()
     # Remove trailing semicolons that sometimes get attached during bibliographic parsing
@@ -3899,6 +3902,9 @@ def are_venues_substantially_different(venue1: str, venue2: str) -> bool:
                 # Handle specific multi-word patterns and well-known acronyms
                 'proc. natl. acad. sci.': 'proceedings of the national academy of sciences',
                 'pnas': 'proceedings of the national academy of sciences',
+                # Special cases that don't follow standard acronym patterns
+                'neurips': 'neural information processing systems',  # Special case
+                'nips': 'neural information processing systems',     # old name for neurips
             }
             # Sort by length (longest first) to ensure longer matches take precedence
             for abbrev, expansion in sorted(common_abbrevs.items(), key=lambda x: len(x[0]), reverse=True):