ebird-api-requests 4.2.0__tar.gz → 4.3.0__tar.gz

{ebird_api_requests-4.2.0/src/ebird_api_requests.egg-info → ebird_api_requests-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ebird-api-requests
-Version: 4.2.0
+Version: 4.3.0
 Summary: A python library for accessing the eBird API, version 2.0
 Author-email: Stuart MacKay <smackay@flagstonesoftware.com>
 License: MIT License
@@ -346,6 +346,30 @@ client.get_observations('MX-OAX')
 
 The client supports all the API functions.
 
+### Rate Limiting
+
+The eBird API limits how quickly requests can be made. If you exceed the
+limit the API responds with HTTP 429 Too Many Requests. If you are making
+lots of calls, for example downloading all the checklists for a region,
+then set a minimum interval, in seconds, between requests:
+
+```python
+from ebird.api.requests import set_rate_limit
+
+set_rate_limit(1.0)
+```
+
+The limit applies to all the requests made by the process. It is disabled
+by default. If the API still responds with HTTP 429, the interval is doubled,
+up to a maximum of 60 seconds, so the requests slow down to a rate the API
+accepts. Use get_rate_limit() to see the current interval.
+
+When an HTTP 429 response is returned, the request is retried after waiting
+for the time given in the Retry-After header. If the request still fails
+after all the retries are used, a RateLimitError is raised. It is a subclass
+of HTTPError, and the retry_after attribute contains the number of seconds
+the API asked you to wait, or None if no Retry-After header was returned.
+
 ## Formats
 
 Most of the eBird API calls return JSON. Some of the calls such as getting

{ebird_api_requests-4.2.0 → ebird_api_requests-4.3.0}/README.md

@@ -316,6 +316,30 @@ client.get_observations('MX-OAX')
 
 The client supports all the API functions.
 
+### Rate Limiting
+
+The eBird API limits how quickly requests can be made. If you exceed the
+limit the API responds with HTTP 429 Too Many Requests. If you are making
+lots of calls, for example downloading all the checklists for a region,
+then set a minimum interval, in seconds, between requests:
+
+```python
+from ebird.api.requests import set_rate_limit
+
+set_rate_limit(1.0)
+```
+
+The limit applies to all the requests made by the process. It is disabled
+by default. If the API still responds with HTTP 429, the interval is doubled,
+up to a maximum of 60 seconds, so the requests slow down to a rate the API
+accepts. Use get_rate_limit() to see the current interval.
+
+When an HTTP 429 response is returned, the request is retried after waiting
+for the time given in the Retry-After header. If the request still fails
+after all the retries are used, a RateLimitError is raised. It is a subclass
+of HTTPError, and the retry_after attribute contains the number of seconds
+the API asked you to wait, or None if no Retry-After header was returned.
+
 ## Formats
 
 Most of the eBird API calls return JSON. Some of the calls such as getting

{ebird_api_requests-4.2.0 → ebird_api_requests-4.3.0}/pyproject.toml

@@ -29,7 +29,7 @@ license = {text = "MIT License"}
 name = "ebird-api-requests"
 readme = "README.md"
 requires-python = ">= 3.10"
-version = "4.2.0"
+version = "4.3.0"
 
 [project.urls]
 Repository = "https://github.com/StuartMacKay/ebird-api-requests.git"
@@ -37,7 +37,7 @@ Issues = "https://github.com/StuartMacKay/ebird-api-requests/issues"
 Changelog = "https://github.com/StuartMacKay/ebird-api-requests/blob/master/CHANGELOG.md"
 
 [tool.bumpversion]
-current_version = "4.2.0"
+current_version = "4.3.0"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
 serialize = ["{major}.{minor}.{patch}"]
 ignore_missing_version = false

{ebird_api_requests-4.2.0 → ebird_api_requests-4.3.0}/src/ebird/api/requests/__init__.py

@@ -2,7 +2,7 @@
 
 """A set of wrapper functions for accessing the eBird API."""
 
-__version__ = "4.2.0"
+__version__ = "4.3.0"
 
 from ebird.api.requests.checklists import get_checklist, get_visits
 from ebird.api.requests.client import Client
@@ -37,3 +37,8 @@ from ebird.api.requests.taxonomy import (
     get_taxonomy_locales,
     get_taxonomy_versions,
 )
+from ebird.api.requests.utils import (
+    RateLimitError,
+    get_rate_limit,
+    set_rate_limit,
+)

{ebird_api_requests-4.2.0 → ebird_api_requests-4.3.0}/src/ebird/api/requests/utils.py

@@ -1,13 +1,122 @@
 """Various functions used in the API."""
 
+import datetime as dt
 import json
+import threading
 import time
+from email.utils import parsedate_to_datetime
 from urllib.error import HTTPError
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen
 
 from ebird.api.requests import constants
 
+# The maximum interval, in seconds, the rate limiter will slow down to
+# after the eBird API responds with HTTP 429 Too Many Requests.
+MAX_RATE_LIMIT_INTERVAL = 60.0
+
+
+class RateLimitError(HTTPError):
+    """Raised when the eBird API still responds with HTTP 429 Too Many
+    Requests after all the retries have been used.
+
+    It is a subclass of HTTPError so existing error handling still works.
+    The retry_after attribute contains the number of seconds the API asked
+    the caller to wait, or None if no Retry-After header was returned.
+
+    """
+
+    def __init__(self, exc, retry_after=None):
+        super().__init__(exc.url, exc.code, exc.msg, exc.headers, exc.fp)
+        self.retry_after = retry_after
+
+
+class RateLimiter:
+    """Enforce a minimum interval between the start of successive requests.
+
+    An interval of zero (the default) disables the limiter.
+
+    """
+
+    def __init__(self, interval=0.0):
+        self.interval = float(interval)
+        self.last_request = None
+        self.lock = threading.Lock()
+
+    def wait(self):
+        with self.lock:
+            if self.interval > 0 and self.last_request is not None:
+                delay = self.interval - (time.monotonic() - self.last_request)
+                if delay > 0:
+                    time.sleep(delay)
+            self.last_request = time.monotonic()
+
+    def slow_down(self):
+        with self.lock:
+            if self.interval > 0:
+                self.interval = min(self.interval * 2, MAX_RATE_LIMIT_INTERVAL)
+
+
+_rate_limiter = RateLimiter()
+
+
+def set_rate_limit(interval):
+    """Set the minimum interval, in seconds, between requests to the eBird API.
+
+    The limit applies to all requests made by the process. Setting it to
+    zero, the default, disables rate limiting. If the API responds with
+    HTTP 429 Too Many Requests, the interval is doubled, up to a maximum
+    of MAX_RATE_LIMIT_INTERVAL, for the rest of the process, or until the
+    rate limit is set again.
+
+    :param interval: the number of seconds between requests.
+    :type interval: float
+
+    """
+    if interval < 0:
+        raise ValueError("The rate limit interval cannot be negative")
+    with _rate_limiter.lock:
+        _rate_limiter.interval = float(interval)
+
+
+def get_rate_limit():
+    """Get the current minimum interval, in seconds, between requests.
+
+    :return: the interval, which may have increased since it was set, if the
+    eBird API responded with HTTP 429 Too Many Requests.
+    :rtype: float
+
+    """
+    return _rate_limiter.interval
+
+
+def get_retry_after(exc):
+    """Get the number of seconds to wait from a Retry-After header.
+
+    :param exc: the error raised for the HTTP 429 response.
+    :type exc: HTTPError
+
+    :return: the number of seconds to wait, or None if the header was not
+    present or could not be parsed.
+    :rtype: float | None
+
+    """
+    value = exc.headers.get("Retry-After") if exc.headers else None
+    if not value:
+        return None
+    try:
+        return max(float(value), 0.0)
+    except ValueError:
+        pass
+    try:
+        when = parsedate_to_datetime(value)
+    except (TypeError, ValueError):
+        return None
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=dt.timezone.utc)
+    return max((when - dt.datetime.now(dt.timezone.utc)).total_seconds(), 0.0)
+
+
 _parameter_defaults = {
     "back": constants.DEFAULT_BACK,
     "cat": constants.DEFAULT_SPECIES_CATEGORY,
@@ -101,6 +210,9 @@ def get_response(url, params=None, headers=None, max_retries=3, backoff_factor=1
     :raises URLError if there is an error with the connection to the
     eBird site.
 
+    :raises RateLimitError if the eBird API still returns HTTP 429 Too Many
+    Requests after all the retries have been used.
+
     :raises HTTPError if the eBird API returns an error.
 
     """
@@ -114,14 +226,21 @@ def get_response(url, params=None, headers=None, max_retries=3, backoff_factor=1
             request.add_header(name, value)
 
     for attempt in range(max_retries + 1):
+        _rate_limiter.wait()
         try:
             return urlopen(request).read()
         except HTTPError as exc:
-            if attempt == max_retries:
+            retry_after = None
+            if exc.code == 429:
+                retry_after = get_retry_after(exc)
+                _rate_limiter.slow_down()
+                if attempt == max_retries:
+                    raise RateLimitError(exc, retry_after) from exc
+            elif attempt == max_retries:
                 raise
-            retry_after = exc.headers.get("Retry-After") if exc.code == 429 else None
-            delay = float(retry_after) if retry_after else backoff_factor * (2**attempt)
-            time.sleep(delay)
+            if retry_after is None:
+                retry_after = backoff_factor * (2**attempt)
+            time.sleep(retry_after)
 
 
 def get_json(content):

{ebird_api_requests-4.2.0 → ebird_api_requests-4.3.0/src/ebird_api_requests.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ebird-api-requests
-Version: 4.2.0
+Version: 4.3.0
 Summary: A python library for accessing the eBird API, version 2.0
 Author-email: Stuart MacKay <smackay@flagstonesoftware.com>
 License: MIT License
@@ -346,6 +346,30 @@ client.get_observations('MX-OAX')
 
 The client supports all the API functions.
 
+### Rate Limiting
+
+The eBird API limits how quickly requests can be made. If you exceed the
+limit the API responds with HTTP 429 Too Many Requests. If you are making
+lots of calls, for example downloading all the checklists for a region,
+then set a minimum interval, in seconds, between requests:
+
+```python
+from ebird.api.requests import set_rate_limit
+
+set_rate_limit(1.0)
+```
+
+The limit applies to all the requests made by the process. It is disabled
+by default. If the API still responds with HTTP 429, the interval is doubled,
+up to a maximum of 60 seconds, so the requests slow down to a rate the API
+accepts. Use get_rate_limit() to see the current interval.
+
+When an HTTP 429 response is returned, the request is retried after waiting
+for the time given in the Retry-After header. If the request still fails
+after all the retries are used, a RateLimitError is raised. It is a subclass
+of HTTPError, and the retry_after attribute contains the number of seconds
+the API asked you to wait, or None if no Retry-After header was returned.
+
 ## Formats
 
 Most of the eBird API calls return JSON. Some of the calls such as getting