scholarcli 1.23__tar.gz → 1.24__tar.gz

{scholarcli-1.23 → scholarcli-1.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scholarcli
-Version: 1.23
+Version: 1.24
 Summary: A tool for structured literature searches across bibliographic databases
 Author-email: Daniel Bosk <dbosk@kth.se>, Ric Glassey <glassey@kth.se>
 License-Expression: MIT

{scholarcli-1.23 → scholarcli-1.24}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scholarcli"
-version = "1.23"
+version = "1.24"
 description = "A tool for structured literature searches across bibliographic databases"
 authors = [{ name = "Daniel Bosk", email = "dbosk@kth.se" },
            { name = "Ric Glassey", email = "glassey@kth.se" }]

{scholarcli-1.23 → scholarcli-1.24}/src/scholar/providers.py

@@ -37,6 +37,9 @@ logger = logging.getLogger(__name__)
 # Provider constants
 DEFAULT_LIMIT = 100
 DBLP_API_URL = "https://dblp.org/search/publ/api"
+DBLP_USER_AGENT = "scholarcli (https://github.com/dbosk/scholar)"
+DBLP_MAX_RETRIES = 3  # extra attempts on HTTP 429 before giving up
+DBLP_RETRY_BACKOFF = 2.0  # base seconds for the exponential 429 back-off
 WOS_STARTER_API_URL = (
     "https://api.clarivate.com/apis/wos-starter/v1/documents"
 )
@@ -344,6 +347,7 @@ class SemanticScholarProvider:
         self._cache: dict = load_cache(self.name)
         register_cache(self.name, self._cache)
         self._last_request_time: float = 0.0
+        self._key_rejected: bool = False
         self._session = self._create_session()
 
     def is_available(self) -> bool:
@@ -353,7 +357,7 @@ class SemanticScholarProvider:
     @property
     def _min_request_interval(self) -> float:
         """Return minimum interval based on whether API key is set."""
-        if self.api_key:
+        if self.api_key and not self._key_rejected:
             return self.RATE_LIMIT_WITH_KEY
         return self.RATE_LIMIT_WITHOUT_KEY
 
@@ -377,14 +381,46 @@ class SemanticScholarProvider:
             time.sleep(self._min_request_interval - elapsed)
         self._last_request_time = time.time()
 
-    @cachedmethod(
-        lambda self: self._cache,
-        key=lambda self, query, limit=100, filters=None: (
-            query,
-            limit,
-            filters.cache_key() if filters else "",
-        ),
-    )
+    def _build_headers(self) -> dict:
+        """Return request headers, omitting the API key once it was rejected."""
+        headers = {}
+        if self.api_key and not self._key_rejected:
+            headers["x-api-key"] = self.api_key
+        return headers
+
+    def _request(self, url: str, params: dict) -> requests.Response:
+        """GET ``url`` with rate limiting and stale-key fallback.
+
+        On a 403 while authenticated, warn once, stop sending the key for the
+        rest of the session, and retry the same request anonymously. A 403 that
+        persists without a key is logged at error level; the response is returned
+        so the caller degrades to an empty result rather than raising.
+        """
+        self._wait_for_rate_limit()
+        headers = self._build_headers()
+        response = self._session.get(
+            url, params=params, headers=headers, timeout=30
+        )
+
+        if response.status_code == 403 and "x-api-key" in headers:
+            logger.warning(
+                "s2: API key rejected (403) -- check or replace S2_API_KEY; "
+                "falling back to anonymous access"
+            )
+            self._key_rejected = True
+            self._wait_for_rate_limit()
+            response = self._session.get(
+                url, params=params, headers=self._build_headers(), timeout=30
+            )
+
+        if response.status_code == 403:
+            logger.error(
+                "s2: Forbidden (403) even without an API key; "
+                "returning no results for this request"
+            )
+
+        return response
+
     def search(
         self,
         query: str,
@@ -397,19 +433,33 @@ class SemanticScholarProvider:
         Automatically fetches multiple pages if the requested limit exceeds
         the API's per-request maximum (100 results per page).
         """
+        cache_key = (query, limit, filters.cache_key() if filters else "")
+        if cache_key in self._cache:
+            return self._cache[cache_key]
+
+        papers, cacheable = self._search_uncached(query, limit, filters)
+        if cacheable:
+            self._cache[cache_key] = papers
+        return papers
+
+    def _search_uncached(
+        self,
+        query: str,
+        limit: int,
+        filters: SearchFilters | None,
+    ) -> tuple[list[Paper], bool]:
+        """Run the paged search, returning ``(papers, cacheable)``.
+
+        ``cacheable`` is False when a transient failure (rate limit, rejected
+        key, HTTP or parse error) truncated the search, so the caller can avoid
+        freezing a partial or empty result into the cache.
+        """
         all_papers: list[Paper] = []
         offset = 0
         page_size = min(limit, 100)  # S2 API returns max 100 per request
 
         try:
             while len(all_papers) < limit:
-                self._wait_for_rate_limit()
-
-                headers = {}
-                if self.api_key:
-                    headers["x-api-key"] = self.api_key
-                    logger.debug("semantic_scholar: Using API key")
-
                 # Calculate how many results we still need
                 remaining = limit - len(all_papers)
                 current_page_size = min(page_size, remaining)
@@ -467,12 +517,7 @@ class SemanticScholarProvider:
                         if s2_types:
                             params["publicationTypes"] = ",".join(s2_types)
 
-                response = self._session.get(
-                    self.API_URL,
-                    params=params,
-                    headers=headers,
-                    timeout=30,
-                )
+                response = self._request(self.API_URL, params)
 
                 if response.status_code == 429:
                     logger.warning(
@@ -481,7 +526,12 @@ class SemanticScholarProvider:
                         "https://www.semanticscholar.org/product/api#api-key-form "
                         "and set S2_API_KEY environment variable."
                     )
-                    break
+                    return all_papers, False
+
+                if response.status_code == 403:
+                    # Key rejected and anonymous access also forbidden; the
+                    # helper already logged the error. Don't cache the empty.
+                    return all_papers, False
 
                 response.raise_for_status()
                 data = response.json()
@@ -502,7 +552,7 @@ class SemanticScholarProvider:
             logger.debug(
                 f"semantic_scholar: Retrieved {len(all_papers)} papers"
             )
-            return all_papers
+            return all_papers, True
         except requests.exceptions.RetryError:
             logger.warning(
                 "s2: Rate limited after retries. "
@@ -510,13 +560,13 @@ class SemanticScholarProvider:
                 "https://www.semanticscholar.org/product/api#api-key-form "
                 "and set S2_API_KEY environment variable."
             )
-            return all_papers
+            return all_papers, False
         except requests.exceptions.HTTPError as e:
             logger.warning("s2: HTTP error: %s", e)
-            return all_papers
+            return all_papers, False
         except Exception as e:
             logger.warning("s2: %s", e)
-            return all_papers
+            return all_papers, False
 
     def get_paper_by_doi(self, doi: str | None) -> Paper | None:
         """
@@ -542,21 +592,13 @@ class SemanticScholarProvider:
         )
 
         try:
-            self._wait_for_rate_limit()
-
-            headers = {}
-            if self.api_key:
-                headers["x-api-key"] = self.api_key
-
             # Use the paper endpoint with DOI: prefix
             url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}"
-            response = self._session.get(
+            response = self._request(
                 url,
-                params={
+                {
                     "fields": "title,authors,year,abstract,venue,externalIds,url,openAccessPdf,fieldsOfStudy",
                 },
-                headers=headers,
-                timeout=30,
             )
 
             if response.status_code == 404:
@@ -568,6 +610,11 @@ class SemanticScholarProvider:
                 logger.warning("s2: Rate limited during DOI lookup")
                 return None
 
+            if response.status_code == 403:
+                # Key rejected and anonymous access also forbidden (logged by
+                # the helper). Do not cache; a later call may succeed.
+                return None
+
             response.raise_for_status()
             paper_data = response.json()
 
@@ -656,22 +703,14 @@ class SemanticScholarProvider:
 
         try:
             while len(all_papers) < limit:
-                self._wait_for_rate_limit()
-
-                headers = {}
-                if self.api_key:
-                    headers["x-api-key"] = self.api_key
-
                 url = f"https://api.semanticscholar.org/graph/v1/paper/{paper_id}/{relation}"
-                response = self._session.get(
+                response = self._request(
                     url,
-                    params={
+                    {
                         "fields": "title,authors,year,abstract,venue,externalIds,url,openAccessPdf,fieldsOfStudy",
                         "limit": page_size,
                         "offset": offset,
                     },
-                    headers=headers,
-                    timeout=30,
                 )
 
                 if response.status_code == 404:
@@ -684,6 +723,11 @@ class SemanticScholarProvider:
                     )
                     break
 
+                if response.status_code == 403:
+                    # Key rejected and anonymous access also forbidden (logged
+                    # by the helper). Stop with what we have.
+                    break
+
                 response.raise_for_status()
                 data = response.json()
 
@@ -1057,6 +1101,14 @@ class OpenAlexProvider:
 register_provider(OpenAlexProvider())
 
 
+class _DBLPUnavailable(Exception):
+    """Raised internally when a DBLP request fails transiently.
+
+    Used to distinguish a throttled or malformed response (do not cache, allow
+    a retry later) from a genuine zero-result answer (safe to cache).
+    """
+
+
 class DBLPProvider:
     """Search provider for DBLP."""
 
@@ -1072,14 +1124,6 @@ class DBLPProvider:
         """DBLP is always available (no API key needed)."""
         return True
 
-    @cachedmethod(
-        lambda self: self._cache,
-        key=lambda self, query, limit=100, filters=None: (
-            query,
-            limit,
-            filters.cache_key() if filters else "",
-        ),
-    )
     def search(
         self,
         query: str,
@@ -1092,109 +1136,179 @@ class DBLPProvider:
         Automatically fetches multiple pages if the requested limit exceeds
         the API's per-request maximum (1000 results per page).
         """
+        cache_key = (query, limit, filters.cache_key() if filters else "")
+        if cache_key in self._cache:
+            return self._cache[cache_key]
+
+        try:
+            papers = self._fetch_pages(query, limit, filters)
+        except Exception as e:
+            # Transient API failure (throttling, malformed body, HTTP error) or
+            # an unexpected parsing error: return empty but do NOT cache, so a
+            # later call can retry instead of replaying a frozen empty result.
+            logger.warning("dblp: %s", e)
+            return []
+
+        self._cache[cache_key] = papers
+        return papers
+
+    def _fetch_pages(
+        self,
+        query: str,
+        limit: int,
+        filters: SearchFilters | None,
+    ) -> list[Paper]:
+        """Fetch successive pages until the limit or the result set is exhausted.
+
+        Raises ``_DBLPUnavailable`` on a transient API failure so the caller can
+        avoid caching an incomplete result.
+        """
+        effective_query = query
+        if filters:
+            # Year filter: DBLP uses "year:YYYY" or "year:YYYY:YYYY" syntax
+            if filters.year:
+                start, end = filters.year_range()
+                if start and end:
+                    if start == end:
+                        effective_query += f" year:{start}"
+                    else:
+                        effective_query += f" year:{start}:{end}"
+                elif start:
+                    # Open end: DBLP doesn't support this well, use current year
+                    import datetime
+
+                    current_year = datetime.datetime.now().year
+                    effective_query += f" year:{start}:{current_year}"
+                elif end:
+                    # Open start: use 1900 as earliest
+                    effective_query += f" year:1900:{end}"
+
+            # Venue filter
+            if filters.venue:
+                effective_query += f" venue:{filters.venue}"
+
+            # Publication types: map to DBLP type syntax
+            if filters.pub_types:
+                dblp_types = []
+                type_mapping = {
+                    "article": "Journal_Articles",
+                    "conference": "Conference_and_Workshop_Papers",
+                    "book": "Books_and_Theses",
+                }
+                for pt in filters.pub_types:
+                    if pt.lower() in type_mapping:
+                        dblp_types.append(type_mapping[pt.lower()])
+                    else:
+                        logger.warning(
+                            "dblp: Publication type '%s' not supported, ignoring",
+                            pt,
+                        )
+                if dblp_types:
+                    effective_query += f" type:{dblp_types[0]}"  # DBLP only supports one type
+
+            # Warn about unsupported filters
+            if filters.open_access:
+                logger.warning(
+                    "dblp: Open access filter not supported, ignoring"
+                )
+            if filters.min_citations is not None:
+                logger.warning(
+                    "dblp: Citation count filter not supported, ignoring"
+                )
+
+        logger.debug(f"dblp: Searching for '{query}' with limit={limit}")
+
         all_papers: list[Paper] = []
         offset = 0
-        page_size = min(limit, 1000)  # DBLP API caps at 1000 per request
+        page_size = min(limit, self.MAX_LIMIT)
 
-        try:
-            # Build query with embedded filters
-            effective_query = query
-            if filters:
-                # Year filter: DBLP uses "year:YYYY" or "year:YYYY:YYYY" syntax
-                if filters.year:
-                    start, end = filters.year_range()
-                    if start and end:
-                        if start == end:
-                            effective_query += f" year:{start}"
-                        else:
-                            effective_query += f" year:{start}:{end}"
-                    elif start:
-                        # Open end: DBLP doesn't support this well, use current year
-                        import datetime
+        while len(all_papers) < limit:
+            remaining = limit - len(all_papers)
+            current_page_size = min(page_size, remaining)
 
-                        current_year = datetime.datetime.now().year
-                        effective_query += f" year:{start}:{current_year}"
-                    elif end:
-                        # Open start: use 1900 as earliest
-                        effective_query += f" year:1900:{end}"
+            hits, total = self._fetch_one_page(
+                effective_query, current_page_size, offset
+            )
+            if not hits:
+                # No more results available
+                break
 
-                # Venue filter
-                if filters.venue:
-                    effective_query += f" venue:{filters.venue}"
+            all_papers.extend(self._convert_hit(hit) for hit in hits)
+            offset += len(hits)
 
-                # Publication types: map to DBLP type syntax
-                if filters.pub_types:
-                    dblp_types = []
-                    type_mapping = {
-                        "article": "Journal_Articles",
-                        "conference": "Conference_and_Workshop_Papers",
-                        "book": "Books_and_Theses",
-                    }
-                    for pt in filters.pub_types:
-                        if pt.lower() in type_mapping:
-                            dblp_types.append(type_mapping[pt.lower()])
-                        else:
-                            logger.warning(
-                                "dblp: Publication type '%s' not supported, ignoring",
-                                pt,
-                            )
-                    if dblp_types:
-                        effective_query += f" type:{dblp_types[0]}"  # DBLP only supports one type
+            if offset >= total:
+                break
 
-                # Warn about unsupported filters
-                if filters.open_access:
-                    logger.warning(
-                        "dblp: Open access filter not supported, ignoring"
-                    )
-                if filters.min_citations is not None:
-                    logger.warning(
-                        "dblp: Citation count filter not supported, ignoring"
-                    )
+        logger.debug(f"dblp: Retrieved {len(all_papers)} papers")
+        return all_papers
 
-            logger.debug(f"dblp: Searching for '{query}' with limit={limit}")
+    def _fetch_one_page(
+        self,
+        effective_query: str,
+        page_size: int,
+        offset: int,
+    ) -> tuple[list, int]:
+        """Fetch one page of hits, returning ``(hits, total)``.
 
-            while len(all_papers) < limit:
-                # Calculate how many results we still need
-                remaining = limit - len(all_papers)
-                current_page_size = min(page_size, remaining)
+        ``hits`` is always a list, even when DBLP collapses a single result into a
+        bare object. Raises ``_DBLPUnavailable`` on a non-JSON body or after
+        exhausting HTTP 429 retries.
+        """
+        for attempt in range(DBLP_MAX_RETRIES + 1):
+            response = requests.get(
+                DBLP_API_URL,
+                params={
+                    "q": effective_query,
+                    "format": "json",
+                    "h": page_size,
+                    "f": offset,
+                },
+                headers={"User-Agent": DBLP_USER_AGENT},
+                timeout=30,
+            )
 
-                response = requests.get(
-                    DBLP_API_URL,
-                    params={
-                        "q": effective_query,
-                        "format": "json",
-                        "h": current_page_size,
-                        "f": offset,
-                    },
-                    timeout=30,
+            if response.status_code == 429:
+                if attempt >= DBLP_MAX_RETRIES:
+                    break
+                delay = DBLP_RETRY_BACKOFF * (2**attempt)
+                retry_after = response.headers.get("Retry-After")
+                if retry_after:
+                    try:
+                        delay = max(delay, float(retry_after))
+                    except ValueError:
+                        pass
+                logger.warning(
+                    "dblp: rate limited (429), retrying in %.1fs", delay
                 )
-                response.raise_for_status()
-                data = response.json()
+                time.sleep(delay)
+                continue
 
-                hits = data.get("result", {}).get("hits", {}).get("hit", [])
-                if not hits:
-                    # No more results available
-                    break
+            response.raise_for_status()
 
-                all_papers.extend(self._convert_hit(hit) for hit in hits)
-                offset += len(hits)
+            try:
+                data = response.json()
+            except ValueError as e:
+                raise _DBLPUnavailable(
+                    f"non-JSON response (HTTP {response.status_code})"
+                ) from e
 
-                # Check if we've fetched all available results
-                total_str = (
-                    data.get("result", {}).get("hits", {}).get("@total", "0")
-                )
-                try:
-                    total = int(total_str)
-                except (ValueError, TypeError):
-                    total = 0
-                if offset >= total:
-                    break
+            hits = ensure_list(
+                safe_get_nested(data, "result", "hits", "hit", default=[])
+            )
+            total = self._parse_total(data)
+            return hits, total
 
-            logger.debug(f"dblp: Retrieved {len(all_papers)} papers")
-            return all_papers
-        except Exception as e:
-            logger.warning("dblp: %s", e)
-            return all_papers
+        raise _DBLPUnavailable("rate limited (HTTP 429) after retries")
+
+    def _parse_total(self, data: dict) -> int:
+        """Return the ``@total`` hit count, or 0 when absent or unparseable."""
+        total_str = safe_get_nested(
+            data, "result", "hits", "@total", default="0"
+        )
+        try:
+            return int(total_str)
+        except (ValueError, TypeError):
+            return 0
 
     def _convert_hit(self, hit: dict) -> Paper:
         """Convert a DBLP API hit to our Paper type."""

{scholarcli-1.23 → scholarcli-1.24}/src/scholarcli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scholarcli
-Version: 1.23
+Version: 1.24
 Summary: A tool for structured literature searches across bibliographic databases
 Author-email: Daniel Bosk <dbosk@kth.se>, Ric Glassey <glassey@kth.se>
 License-Expression: MIT

{scholarcli-1.23 → scholarcli-1.24}/tests/test_providers.py

@@ -212,6 +212,57 @@ class TestSemanticScholarProvider:
             results = provider.search("test")
             assert results == []
 
+    def test_search_caches_successful_results(self):
+        """A successful search is cached; the second call skips the API."""
+        provider = SemanticScholarProvider()
+
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "total": 1,
+            "data": [{"title": "Cached Paper", "authors": []}],
+        }
+
+        with patch.object(
+            provider._session, "get", return_value=mock_response
+        ) as mock_get:
+            first = provider.search("test")
+            second = provider.search("test")
+
+            assert len(first) == 1
+            assert len(second) == 1
+            assert mock_get.call_count == 1  # second call served from cache
+
+    def test_rate_limited_result_is_not_cached(self):
+        """A 429 returns empty without caching, so a later call retries.
+
+        Unlike a successful search, a rate-limited (transient) result must
+        not be frozen into the persistent cache, or every later call with the
+        same query would replay the empty result until the cache is cleared.
+        """
+        provider = SemanticScholarProvider()
+
+        throttled = Mock()
+        throttled.status_code = 429
+
+        recovered = Mock()
+        recovered.status_code = 200
+        recovered.json.return_value = {
+            "total": 1,
+            "data": [{"title": "Recovered Paper", "authors": []}],
+        }
+
+        with patch.object(
+            provider._session, "get", side_effect=[throttled, recovered]
+        ) as mock_get:
+            first = provider.search("test")
+            assert first == []
+
+            second = provider.search("test")
+            assert len(second) == 1
+            assert second[0].title == "Recovered Paper"
+            assert mock_get.call_count == 2  # not served from cache
+
     def test_rate_limiting_with_api_key(self):
         """Provider uses faster rate limit when API key is set."""
         provider = SemanticScholarProvider(api_key="test-key")
@@ -395,20 +446,27 @@ class TestSemanticScholarIntegration:
     """Integration test for the SemanticScholar provider."""
 
     @pytest.mark.integration
-    @pytest.mark.skip(reason="Skipped by default due to API rate limits")
     def test_real_api_call(self):
-        """Make one real API call to verify the provider works."""
+        """Anonymous search returns real results.
+
+        We force the key off *after* construction (the constructor falls back
+        to the ``S2_API_KEY`` environment variable, so ``api_key=None`` alone
+        would still pick up a stale key). This exercises the pure anonymous
+        path in one request, unmasked by any environment key. An empty result
+        fails the test rather than skipping, so a regression is visible.
+        """
+        provider = SemanticScholarProvider(api_key=None)
+        provider.api_key = None  # ignore any S2_API_KEY in the environment
         try:
-            provider = SemanticScholarProvider()
             papers = provider.search("machine learning", limit=1)
-            assert len(papers) >= 1, "Provider returned no results"
-
-            paper = papers[0]
-            assert isinstance(paper, Paper)
-            assert paper.title
-            assert paper.sources == ["s2"]
         except Exception as e:
             pytest.skip(f"Semantic Scholar API unavailable: {e}")
+
+        assert len(papers) >= 1, "Provider returned no results"
+        paper = papers[0]
+        assert isinstance(paper, Paper)
+        assert paper.title
+        assert paper.sources == ["s2"]
 class TestOpenAlexProvider:
     """Tests for the OpenAlex provider."""
 
@@ -616,22 +674,20 @@ class TestOpenAlexIntegration:
     """Integration test for the OpenAlex provider."""
 
     @pytest.mark.integration
-    @pytest.mark.skip(reason="Skipped by default to avoid API calls")
     def test_real_api_call(self):
         """Make one real API call to verify the provider works."""
         provider = OpenAlexProvider()
 
         try:
             papers = provider.search("machine learning", limit=1)
-
-            assert len(papers) >= 1, "Provider returned no results"
-
-            paper = papers[0]
-            assert isinstance(paper, Paper)
-            assert paper.title
-            assert paper.sources == ["openalex"]
         except Exception as e:
             pytest.skip(f"OpenAlex API unavailable: {e}")
+
+        assert len(papers) >= 1, "Provider returned no results"
+        paper = papers[0]
+        assert isinstance(paper, Paper)
+        assert paper.title
+        assert paper.sources == ["openalex"]
 class TestDBLPProvider:
     """Tests for the DBLP provider."""
 
@@ -711,6 +767,39 @@ class TestDBLPProvider:
             assert len(results) == 1
             assert results[0].authors == ["Solo Author"]
 
+    def test_search_handles_single_hit_as_dict(self):
+        """search() handles a lone hit returned as a dict, not a list.
+
+        DBLP collapses ``result.hits.hit`` to a bare object when exactly one
+        paper matches. Iterating that object used to crash and silently cache
+        an empty result; ``ensure_list`` must wrap it into a one-element list.
+        """
+        provider = DBLPProvider()
+
+        mock_response = Mock()
+        mock_response.json.return_value = {
+            "result": {
+                "hits": {
+                    "@total": "1",
+                    "hit": {
+                        "info": {
+                            "title": "The Only Match",
+                            "year": "2024",
+                        }
+                    },
+                }
+            }
+        }
+        mock_response.raise_for_status = Mock()
+
+        with patch("requests.get") as mock_get:
+            mock_get.return_value = mock_response
+
+            results = provider.search("very specific query")
+
+            assert len(results) == 1
+            assert results[0].title == "The Only Match"
+
     def test_search_handles_empty_results(self):
         """search() returns empty list when no results."""
         provider = DBLPProvider()
@@ -737,6 +826,94 @@ class TestDBLPProvider:
             results = provider.search("test")
             assert results == []
 
+    def test_search_sends_user_agent(self):
+        """search() identifies itself with a descriptive User-Agent.
+
+        DBLP throttles anonymous default-agent traffic more readily, so we
+        always send our own User-Agent header.
+        """
+        provider = DBLPProvider()
+
+        mock_response = Mock()
+        mock_response.json.return_value = {
+            "result": {"hits": {"hit": [], "@total": "0"}}
+        }
+        mock_response.raise_for_status = Mock()
+
+        with patch("requests.get") as mock_get:
+            mock_get.return_value = mock_response
+
+            provider.search("test")
+
+            headers = mock_get.call_args[1]["headers"]
+            assert headers["User-Agent"] == DBLP_USER_AGENT
+
+    def test_transient_failure_is_not_cached(self):
+        """A non-JSON (throttled) body returns empty without caching it.
+
+        Caching a transient outage would freeze the query into a permanently
+        empty result. The second call must therefore reach the API again, and
+        once it succeeds the real results come through.
+        """
+        provider = DBLPProvider()
+
+        bad_response = Mock()
+        bad_response.status_code = 200
+        bad_response.raise_for_status = Mock()
+        bad_response.json.side_effect = ValueError("Expecting value")
+
+        good_response = Mock()
+        good_response.status_code = 200
+        good_response.raise_for_status = Mock()
+        good_response.json.return_value = {
+            "result": {
+                "hits": {
+                    "@total": "1",
+                    "hit": [{"info": {"title": "Recovered Paper"}}],
+                }
+            }
+        }
+
+        with patch(
+            "requests.get", side_effect=[bad_response, good_response]
+        ) as mock_get:
+            first = provider.search("test")
+            assert first == []
+
+            second = provider.search("test")
+            assert len(second) == 1
+            assert second[0].title == "Recovered Paper"
+            assert mock_get.call_count == 2
+
+    def test_retries_on_rate_limit(self):
+        """search() retries after an HTTP 429 before succeeding."""
+        provider = DBLPProvider()
+
+        throttled = Mock()
+        throttled.status_code = 429
+        throttled.headers = {}
+
+        ok = Mock()
+        ok.status_code = 200
+        ok.raise_for_status = Mock()
+        ok.json.return_value = {
+            "result": {
+                "hits": {
+                    "@total": "1",
+                    "hit": [{"info": {"title": "After Backoff"}}],
+                }
+            }
+        }
+
+        with patch("requests.get", side_effect=[throttled, ok]) as mock_get:
+            with patch("time.sleep") as mock_sleep:
+                results = provider.search("test")
+
+            assert len(results) == 1
+            assert results[0].title == "After Backoff"
+            assert mock_get.call_count == 2
+            mock_sleep.assert_called_once()
+
     def test_handles_missing_fields(self):
         """search() handles publications with missing optional fields."""
         provider = DBLPProvider()
@@ -847,22 +1024,44 @@ class TestDBLPIntegration:
     """Integration test for the DBLP provider."""
 
     @pytest.mark.integration
-    @pytest.mark.skip(reason="Skipped by default to avoid API calls")
     def test_real_api_call(self):
         """Make one real API call to verify the provider works."""
         provider = DBLPProvider()
 
         try:
             papers = provider.search("machine learning", limit=1)
+        except Exception as e:
+            pytest.skip(f"DBLP API unavailable: {e}")
 
-            assert len(papers) >= 1, "Provider returned no results"
+        assert len(papers) >= 1, "Provider returned no results"
+        paper = papers[0]
+        assert isinstance(paper, Paper)
+        assert paper.title
+        assert paper.sources == ["dblp"]
 
-            paper = papers[0]
-            assert isinstance(paper, Paper)
-            assert paper.title
-            assert paper.sources == ["dblp"]
+    @pytest.mark.integration
+    def test_real_specific_query(self):
+        """A long, specific multi-term query still returns results.
+
+        Narrow queries that match only a handful of papers are exactly the
+        case that used to break: DBLP returns a lone hit as a bare object, and
+        the swallowed crash cached an empty result. This guards that the
+        ``ensure_list`` normalisation survives a real near-single-hit response.
+        (The exact bare-object case is covered deterministically by
+        ``test_search_handles_single_hit_as_dict``.)
+        """
+        provider = DBLPProvider()
+
+        title = (
+            "Dynamic Searchable Encryption in Very-Large Databases: "
+            "Data Structures and Implementation"
+        )
+        try:
+            papers = provider.search(title, limit=30)
         except Exception as e:
             pytest.skip(f"DBLP API unavailable: {e}")
+
+        assert len(papers) >= 1, "specific query returned no results"
 class TestWebOfScienceProvider:
     """Tests for the Web of Science provider."""
 
@@ -1673,7 +1872,6 @@ class TestWebOfScienceIntegration:
     """Integration test for the Web of Science provider."""
 
     @pytest.mark.integration
-    @pytest.mark.skip(reason="Skipped by default; requires WOS API key")
     def test_real_api_call(self):
         """Make one real API call to verify the provider works."""
         import os
@@ -1693,15 +1891,14 @@ class TestWebOfScienceIntegration:
 
         try:
             papers = provider.search("machine learning", limit=1)
-
-            assert len(papers) >= 1, "Provider returned no results"
-
-            paper = papers[0]
-            assert isinstance(paper, Paper)
-            assert paper.title
-            assert paper.sources == ["wos"]
         except Exception as e:
             pytest.skip(f"Web of Science API unavailable: {e}")
+
+        assert len(papers) >= 1, "Provider returned no results"
+        paper = papers[0]
+        assert isinstance(paper, Paper)
+        assert paper.title
+        assert paper.sources == ["wos"]
 class TestIEEEXploreProvider:
     """Tests for the IEEE Xplore provider."""
 
@@ -2028,7 +2225,6 @@ class TestIEEEXploreIntegration:
     """Integration test for the IEEE Xplore provider."""
 
     @pytest.mark.integration
-    @pytest.mark.skip(reason="Skipped by default; requires IEEE_API_KEY")
     def test_real_api_call(self):
         """Make one real API call to verify the provider works."""
         import os
@@ -2041,15 +2237,14 @@ class TestIEEEXploreIntegration:
 
         try:
             papers = provider.search("machine learning", limit=1)
-
-            assert len(papers) >= 1, "Provider returned no results"
-
-            paper = papers[0]
-            assert isinstance(paper, Paper)
-            assert paper.title
-            assert paper.sources == ["ieee"]
         except Exception as e:
             pytest.skip(f"IEEE Xplore API unavailable: {e}")
+
+        assert len(papers) >= 1, "Provider returned no results"
+        paper = papers[0]
+        assert isinstance(paper, Paper)
+        assert paper.title
+        assert paper.sources == ["ieee"]
 class TestArxivProvider:
     """Tests for the arXiv provider."""
 
@@ -2275,22 +2470,20 @@ class TestArxivIntegration:
     """Integration test for the arXiv provider."""
 
     @pytest.mark.integration
-    @pytest.mark.skip(reason="Skipped by default to avoid API calls")
     def test_real_api_call(self):
         """Make one real API call to verify the provider works."""
         provider = ArxivProvider()
 
         try:
             papers = provider.search("machine learning", limit=1)
-
-            assert len(papers) >= 1, "Provider returned no results"
-
-            paper = papers[0]
-            assert isinstance(paper, Paper)
-            assert paper.title
-            assert paper.sources == ["arxiv"]
         except Exception as e:
             pytest.skip(f"arXiv API unavailable: {e}")
+
+        assert len(papers) >= 1, "Provider returned no results"
+        paper = papers[0]
+        assert isinstance(paper, Paper)
+        assert paper.title
+        assert paper.sources == ["arxiv"]
 class TestScopusProvider:
     """Tests for the Scopus provider."""
 
@@ -2520,23 +2713,26 @@ class TestScopusIntegration:
     """Integration test for the Scopus provider."""
 
     @pytest.mark.integration
-    @pytest.mark.skip(
-        reason="Skipped to avoid API calls",
-    )
     def test_real_api_call(self):
         """Make one real API call to verify."""
+        import os
+
+        if not os.environ.get("SCOPUS_API_KEY"):
+            pytest.skip("SCOPUS_API_KEY environment variable not set")
+
         provider = ScopusProvider()
 
         try:
             papers = provider.search(
                 "machine learning", limit=1,
             )
-            assert len(papers) >= 1
-            paper = papers[0]
-            assert isinstance(paper, Paper)
-            assert paper.title
-            assert paper.sources == ["scopus"]
         except Exception as e:
             pytest.skip(
                 f"Scopus API unavailable: {e}"
             )
+
+        assert len(papers) >= 1
+        paper = papers[0]
+        assert isinstance(paper, Paper)
+        assert paper.title
+        assert paper.sources == ["scopus"]