bdcdata 2.0.0__tar.gz → 2.1.0__tar.gz

{bdcdata-2.0.0 → bdcdata-2.1.0}/.gitignore

@@ -13,6 +13,7 @@ venv/
 .ruff_cache/
 .coverage
 htmlcov/
+.python-version
 
 # Credentials -- never commit these
 .env

{bdcdata-2.0.0 → bdcdata-2.1.0}/CHANGELOG.md

@@ -5,7 +5,33 @@ All notable changes to this project are documented here.
 The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/).
 
-## [2.0.0] — unreleased
+## [2.1.0]
+
+### Removed/Changed
+Switch rate limiting to requests-ratelimiter
+
+Replace the hand-rolled sliding-window _RateLimiter with a shared
+LimiterSession (10 calls/minute). The session now paces every request
+it sends, retries included, and backs off for the rest of the window
+after a 429.
+
+Breaking: reset_session() is removed from the public API.
+
+- Comment out _RateLimiter, _limiter, and reset_session in _client.py
+- _get_session() returns a LimiterSession, still guarded by a lock
+- Add requests-ratelimiter as a dependency
+- conftest: give each test a fresh high-limit session in place of
+  reset_session() and the _limiter.acquire patch
+- TestRateLimiter: check the session's configured rate and that the
+  11th call in a minute is held back
+
+Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
+
+### Added
+
+Added uv.lock to the repository.
+
+## [2.0.0] 
 
 A ground-up rewrite. The submodule layout (`bdcdata.availability.fixed()`) is
 unchanged, but arguments, credential handling, and return types all changed.

{bdcdata-2.0.0 → bdcdata-2.1.0}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.5
 Name: bdcdata
-Version: 2.0.0
+Version: 2.1.0
 Summary: Work with FCC Broadband Data Collection (BDC) data in pandas.
 Project-URL: Homepage, https://github.com/npappin/bdcdata
 Project-URL: Documentation, https://github.com/npappin/bdcdata/tree/main/docs
 Project-URL: Issues, https://github.com/npappin/bdcdata/issues
-Author-email: "W. Nick Pappin" <nick.pappin@wsu.edu>
+Author-email: "W. Nick Pappin" <npappin@gmail.com>
 Maintainer-email: "W. Nick Pappin" <npappin@gmail.com>
 License-Expression: MIT
 License-File: LICENSE.txt
@@ -25,6 +25,7 @@ Requires-Python: >=3.10
 Requires-Dist: pandas>=2
 Requires-Dist: pyarrow>=12
 Requires-Dist: python-dotenv
+Requires-Dist: requests-ratelimiter
 Requires-Dist: requests>=2.28
 Provides-Extra: all
 Requires-Dist: pyogrio>=0.7; extra == 'all'

{bdcdata-2.0.0 → bdcdata-2.1.0}/pyproject.toml

@@ -11,7 +11,7 @@ requires-python = ">=3.10"
 license = "MIT"
 license-files = ["LICEN[CS]E*"]
 authors = [
-  { name = "W. Nick Pappin", email = "nick.pappin@wsu.edu" },
+  { name = "W. Nick Pappin", email = "npappin@gmail.com" },
 ]
 maintainers = [
   { name = "W. Nick Pappin", email = "npappin@gmail.com" },
@@ -22,6 +22,7 @@ dependencies = [
   "pandas>=2",
   "pyarrow>=12",
   "python-dotenv",
+  "requests-ratelimiter",
 ]
 classifiers = [
   "Development Status :: 3 - Alpha",

{bdcdata-2.0.0 → bdcdata-2.1.0}/src/bdcdata/__init__.py

@@ -26,7 +26,7 @@ logging.getLogger(__name__).addHandler(logging.NullHandler())
 
 from . import availability, catalog, challenges, funding, lookups  # noqa: E402
 from ._cache import cache_info, clear_cache  # noqa: E402
-from ._client import check_credentials, reset_session  # noqa: E402
+from ._client import check_credentials  # noqa: E402
 from .config import get_cache_settings, set_base_url, set_cache, set_timeout  # noqa: E402
 from .credentials import (  # noqa: E402
     clear_credentials,
@@ -77,7 +77,7 @@ __all__ = [  # noqa: RUF022
     "clear_cache",
     "set_base_url",
     "set_timeout",
-    "reset_session",
+    # "reset_session",
     # exceptions
     "BdcError",
     "BdcAuthError",

{bdcdata-2.0.0 → bdcdata-2.1.0}/src/bdcdata/_client.py

@@ -13,11 +13,13 @@ import contextlib
 import logging
 import threading
 import time
-from collections import deque
+
+# from collections import deque  # only used by the old _RateLimiter
 from collections.abc import Mapping
 from typing import Any
 
 import requests
+from requests_ratelimiter import LimiterSession
 
 from ._cache import cache_key, read_cached, write_cached
 from .config import RATE_LIMIT_PER_MINUTE, get_base_url, options
@@ -32,7 +34,8 @@ from .exceptions import (
     BdcUnprocessableError,
 )
 
-__all__ = ["check_credentials", "get_bytes", "get_json", "reset_session"]
+# __all__ = ["check_credentials", "get_bytes", "get_json", "reset_session"]
+__all__ = ["check_credentials", "get_bytes", "get_json"]
 
 logger = logging.getLogger("bdcdata")
 
@@ -49,64 +52,64 @@ def _user_agent() -> str:
     return f"bdcdata/{pkg_version} (+https://github.com/npappin/bdcdata)"
 
 
-class _RateLimiter:
-    """Sliding-window limiter shared by every request bdcdata makes.
-
-    The FCC documents 10 calls per minute on each endpoint. Pacing here means
-    a 50-state pull is slow rather than a wall of 429s.
-    """
-
-    def __init__(self, calls: int, period: float) -> None:
-        self._calls = calls
-        self._period = period
-        self._times: deque[float] = deque()
-        self._lock = threading.Lock()
-
-    def acquire(self) -> None:
-        with self._lock:
-            while True:
-                now = time.monotonic()
-                while self._times and now - self._times[0] >= self._period:
-                    self._times.popleft()
-                if len(self._times) < self._calls:
-                    self._times.append(now)
-                    return
-                wait = self._period - (now - self._times[0])
-                if wait > 0:
-                    logger.debug("Rate limit reached; waiting %.1fs", wait)
-                    time.sleep(wait)
-
-    def reset(self) -> None:
-        with self._lock:
-            self._times.clear()
-
-
-_limiter = _RateLimiter(RATE_LIMIT_PER_MINUTE, 60.0)
-_session: requests.Session | None = None
+# class _RateLimiter:
+#     """Sliding-window limiter shared by every request bdcdata makes.
+
+#     The FCC documents 10 calls per minute on each endpoint. Pacing here means
+#     a 50-state pull is slow rather than a wall of 429s.
+#     """
+
+#     def __init__(self, calls: int, period: float) -> None:
+#         self._calls = calls
+#         self._period = period
+#         self._times: deque[float] = deque()
+#         self._lock = threading.Lock()
+
+#     def acquire(self) -> None:
+#         with self._lock:
+#             while True:
+#                 now = time.monotonic()
+#                 while self._times and now - self._times[0] >= self._period:
+#                     self._times.popleft()
+#                 if len(self._times) < self._calls:
+#                     self._times.append(now)
+#                     return
+#                 wait = self._period - (now - self._times[0])
+#                 if wait > 0:
+#                     logger.debug("Rate limit reached; waiting %.1fs", wait)
+#                     time.sleep(wait)
+
+#     def reset(self) -> None:
+#         with self._lock:
+#             self._times.clear()
+
+
+# _limiter = _RateLimiter(RATE_LIMIT_PER_MINUTE, 60.0)
+_session: LimiterSession | None = None
 _session_lock = threading.Lock()
 
 
-def _get_session() -> requests.Session:
+def _get_session() -> LimiterSession:
     global _session
     with _session_lock:
         if _session is None:
-            _session = requests.Session()
+            _session = LimiterSession(per_minute=RATE_LIMIT_PER_MINUTE)
             _session.headers.update({"User-Agent": _user_agent()})
             logger.debug("HTTP session created")
-        return _session
+    return _session
 
 
-def reset_session() -> None:
-    """Close the pooled HTTP session and clear the rate-limit window.
+# def reset_session() -> None:
+#     """Close the pooled HTTP session and clear the rate-limit window.
 
-    Mostly useful in tests.
-    """
-    global _session
-    with _session_lock:
-        if _session is not None:
-            _session.close()
-        _session = None
-    _limiter.reset()
+#     Mostly useful in tests.
+#     """
+#     global _session
+#     with _session_lock:
+#         if _session is not None:
+#             _session.close()
+#         _session = None
+#     _limiter.reset()
 
 
 def _raise_for_status(response: requests.Response, url: str) -> None:
@@ -168,7 +171,7 @@ def _request(
     last_error: Exception | None = None
 
     for attempt in range(1, attempts + 1):
-        _limiter.acquire()
+        # _limiter.acquire()
         logger.debug("GET %s params=%s (attempt %d/%d)", url, clean_params, attempt, attempts)
         try:
             response = session.get(

{bdcdata-2.0.0 → bdcdata-2.1.0}/tests/conftest.py

@@ -9,10 +9,10 @@ from __future__ import annotations
 
 import pytest
 import responses
+from requests_ratelimiter import LimiterSession
 
 import bdcdata
-from bdcdata import catalog, credentials
-from bdcdata._client import reset_session
+from bdcdata import _client, catalog, credentials
 
 
 @pytest.fixture(autouse=True)
@@ -30,18 +30,22 @@ def _isolate(monkeypatch, tmp_path):
     catalog.clear_release_cache()
     bdcdata.set_base_url(None)
     bdcdata.set_cache(False, path=tmp_path / "cache")
-    reset_session()
+    # reset_session()
 
-    # Neutralize the 10-calls-per-minute pacing. It works (TestRateLimiter
-    # builds its own _RateLimiter to prove it), but a test that downloads 60
-    # files would otherwise sleep for six real minutes.
-    monkeypatch.setattr("bdcdata._client._limiter.acquire", lambda: None)
+    # A fresh session per test with pacing effectively off. The real
+    # 10/minute limit is covered by TestRateLimiter; a test that downloads
+    # 60 files would otherwise sleep for six real minutes. limit_statuses=()
+    # stops a mocked 429 from locking the session for the rest of the minute.
+    fast = LimiterSession(per_minute=100_000, limit_statuses=())
+    fast.headers.update({"User-Agent": _client._user_agent()})
+    monkeypatch.setattr("bdcdata._client._session", fast)
 
     yield
 
     credentials.clear_credentials()
     catalog.clear_release_cache()
-    reset_session()
+    # reset_session()
+    fast.close()
 
 
 @pytest.fixture

{bdcdata-2.0.0 → bdcdata-2.1.0}/tests/test_client.py

@@ -211,27 +211,35 @@ class TestCheckCredentials:
 
 
 class TestRateLimiter:
-    def test_paces_calls_beyond_the_limit(self, monkeypatch):
-        from bdcdata._client import _RateLimiter
-
-        clock = {"now": 0.0}
-        slept: list[float] = []
-        monkeypatch.setattr("bdcdata._client.time.monotonic", lambda: clock["now"])
-
-        def fake_sleep(seconds):
-            slept.append(seconds)
-            clock["now"] += seconds
-
-        monkeypatch.setattr("bdcdata._client.time.sleep", fake_sleep)
-
-        limiter = _RateLimiter(calls=10, period=60.0)
-        for _ in range(10):
-            limiter.acquire()
-        assert slept == []
-
-        # The 11th call must wait for the window to roll over.
-        limiter.acquire()
-        assert slept and slept[0] == pytest.approx(60.0)
+    def test_session_is_rate_limited_to_the_published_rate(self, monkeypatch):
+        from requests_ratelimiter import LimiterSession
+        from requests_ratelimiter.buckets import HostBucketFactory
+
+        monkeypatch.setattr(_client, "_session", None)  # bypass conftest's fast session
+        session = _client._get_session()
+        try:
+            assert isinstance(session, LimiterSession)
+            factory = session.limiter.bucket_factory
+            assert isinstance(factory, HostBucketFactory)
+            [rate] = factory.rates
+            assert (rate.limit, rate.interval) == (10, 60_000)
+        finally:
+            session.close()
+
+    def test_eleventh_call_in_a_minute_is_held_back(self, monkeypatch, mock_api):
+        import requests
+
+        monkeypatch.setattr(_client, "_session", None)
+        session = _client._get_session()
+        session.max_delay = 0.5  # fail fast instead of waiting out the minute
+        mock_api.add(responses.GET, AS_OF_DATES, json={"data": []})
+        try:
+            for _ in range(10):
+                session.get(AS_OF_DATES)
+            with pytest.raises(requests.exceptions.Timeout):
+                session.get(AS_OF_DATES)
+        finally:
+            session.close()
 
     def test_limit_matches_the_published_rate(self):
         from bdcdata.config import RATE_LIMIT_PER_MINUTE