ebird-api-requests 4.1.1__tar.gz → 4.3.0__tar.gz

{ebird_api_requests-4.1.1/src/ebird_api_requests.egg-info → ebird_api_requests-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ebird-api-requests
-Version: 4.1.1
+Version: 4.3.0
 Summary: A python library for accessing the eBird API, version 2.0
 Author-email: Stuart MacKay <smackay@flagstonesoftware.com>
 License: MIT License
@@ -346,6 +346,30 @@ client.get_observations('MX-OAX')
 
 The client supports all the API functions.
 
+### Rate Limiting
+
+The eBird API limits how quickly requests can be made. If you exceed the
+limit the API responds with HTTP 429 Too Many Requests. If you are making
+lots of calls, for example downloading all the checklists for a region,
+then set a minimum interval, in seconds, between requests:
+
+```python
+from ebird.api.requests import set_rate_limit
+
+set_rate_limit(1.0)
+```
+
+The limit applies to all the requests made by the process. It is disabled
+by default. If the API still responds with HTTP 429, the interval is doubled,
+up to a maximum of 60 seconds, so the requests slow down to a rate the API
+accepts. Use get_rate_limit() to see the current interval.
+
+When an HTTP 429 response is returned, the request is retried after waiting
+for the time given in the Retry-After header. If the request still fails
+after all the retries are used, a RateLimitError is raised. It is a subclass
+of HTTPError, and the retry_after attribute contains the number of seconds
+the API asked you to wait, or None if no Retry-After header was returned.
+
 ## Formats
 
 Most of the eBird API calls return JSON. Some of the calls such as getting

{ebird_api_requests-4.1.1 → ebird_api_requests-4.3.0}/README.md

@@ -316,6 +316,30 @@ client.get_observations('MX-OAX')
 
 The client supports all the API functions.
 
+### Rate Limiting
+
+The eBird API limits how quickly requests can be made. If you exceed the
+limit the API responds with HTTP 429 Too Many Requests. If you are making
+lots of calls, for example downloading all the checklists for a region,
+then set a minimum interval, in seconds, between requests:
+
+```python
+from ebird.api.requests import set_rate_limit
+
+set_rate_limit(1.0)
+```
+
+The limit applies to all the requests made by the process. It is disabled
+by default. If the API still responds with HTTP 429, the interval is doubled,
+up to a maximum of 60 seconds, so the requests slow down to a rate the API
+accepts. Use get_rate_limit() to see the current interval.
+
+When an HTTP 429 response is returned, the request is retried after waiting
+for the time given in the Retry-After header. If the request still fails
+after all the retries are used, a RateLimitError is raised. It is a subclass
+of HTTPError, and the retry_after attribute contains the number of seconds
+the API asked you to wait, or None if no Retry-After header was returned.
+
 ## Formats
 
 Most of the eBird API calls return JSON. Some of the calls such as getting

{ebird_api_requests-4.1.1 → ebird_api_requests-4.3.0}/pyproject.toml

@@ -29,7 +29,7 @@ license = {text = "MIT License"}
 name = "ebird-api-requests"
 readme = "README.md"
 requires-python = ">= 3.10"
-version = "4.1.1"
+version = "4.3.0"
 
 [project.urls]
 Repository = "https://github.com/StuartMacKay/ebird-api-requests.git"
@@ -37,7 +37,7 @@ Issues = "https://github.com/StuartMacKay/ebird-api-requests/issues"
 Changelog = "https://github.com/StuartMacKay/ebird-api-requests/blob/master/CHANGELOG.md"
 
 [tool.bumpversion]
-current_version = "4.1.1"
+current_version = "4.3.0"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
 serialize = ["{major}.{minor}.{patch}"]
 ignore_missing_version = false

{ebird_api_requests-4.1.1 → ebird_api_requests-4.3.0}/src/ebird/api/requests/__init__.py

@@ -2,7 +2,7 @@
 
 """A set of wrapper functions for accessing the eBird API."""
 
-__version__ = "4.1.1"
+__version__ = "4.3.0"
 
 from ebird.api.requests.checklists import get_checklist, get_visits
 from ebird.api.requests.client import Client
@@ -37,3 +37,8 @@ from ebird.api.requests.taxonomy import (
     get_taxonomy_locales,
     get_taxonomy_versions,
 )
+from ebird.api.requests.utils import (
+    RateLimitError,
+    get_rate_limit,
+    set_rate_limit,
+)

{ebird_api_requests-4.1.1 → ebird_api_requests-4.3.0}/src/ebird/api/requests/client.py

@@ -109,7 +109,7 @@ class Client:
             category=self.category,
         )
 
-    def get_historic_observations(self, area, date):
+    def get_historic_observations(self, area, date, rank="mrec"):
         """Get recent observations for a region.
 
         :param area: a country, subnational1, subnational2 or location code
@@ -117,6 +117,9 @@ class Client:
 
         :param date: the date, since Jan 1st 1800.
 
+        :param rank: for each species, return the latest observation on the
+        date, 'mrec' (the default), or the first observation, 'create'.
+
         :return: the list of observations in simple format.
 
         """
@@ -130,6 +133,7 @@ class Client:
             hotspot=self.hotspot,
             detail=self.detail,
             category=self.category,
+            rank=rank,
         )
 
     def get_nearby_observations(self, lat, lng, dist=25):

{ebird_api_requests-4.1.1 → ebird_api_requests-4.3.0}/src/ebird/api/requests/constants.py

@@ -7,6 +7,7 @@ DEFAULT_HOTSPOTS_ONLY = "false"
 DEFAULT_PROVISIONAL = "false"
 DEFAULT_LOCALE = "en"
 DEFAULT_OBSERVATION_ORDER = "date"
+DEFAULT_OBSERVATION_RANK = "mrec"
 DEFAULT_SPECIES_CATEGORY = None
 DEFAULT_TAXONOMY_ORDER = "ebird"
 DEFAULT_MAX_OBSERVATIONS = None
@@ -102,6 +103,8 @@ REGION_TYPES = ["country", "subnational1", "subnational2"]
 
 TOP_100_RANK = ["spp", "cl"]
 
+OBSERVATION_RANK = ["mrec", "create"]
+
 # Just occasionally the connection to eBird freezes. The timeout
 # will raise an error which is preferable to the sitting there
 # waiting for something to happen, when it will not.

{ebird_api_requests-4.1.1 → ebird_api_requests-4.3.0}/src/ebird/api/requests/observations.py

@@ -14,6 +14,7 @@ from ebird.api.requests.validation import (
     clean_lng,
     clean_locale,
     clean_max_observations,
+    clean_observation_rank,
     clean_provisional,
     clean_sort,
 )
@@ -633,6 +634,7 @@ def get_historic_observations(
     hotspot=False,
     detail="simple",
     category=None,
+    rank="mrec",
 ):
     """Get recent observations for a region.
 
@@ -671,6 +673,9 @@ def get_historic_observations(
     More than one value can be given in a comma-separated string. The default
     value is None and records from all categories will be returned.
 
+    :param rank: for each species, return the latest observation on the
+    date, 'mrec' (the default), or the first observation, 'create'.
+
     :return: the list of observations in simple format.
 
     :raises ValueError: if any of the arguments fail the validation checks.
@@ -686,7 +691,7 @@ def get_historic_observations(
     url = HISTORIC_OBSERVATIONS_URL % (cleaned[0], date.strftime("%Y/%m/%d"))
 
     params = {
-        "rank": "mrec",
+        "rank": clean_observation_rank(rank),
         "detail": clean_detail(detail),
         "sppLocale": clean_locale(locale),
         "includeProvisional": clean_provisional(provisional),

{ebird_api_requests-4.1.1 → ebird_api_requests-4.3.0}/src/ebird/api/requests/statistics.py

@@ -5,7 +5,7 @@ from ebird.api.requests.validation import (
     clean_area,
     clean_date,
     clean_max_observers,
-    clean_rank,
+    clean_observer_rank,
     clean_region,
 )
 
@@ -46,7 +46,7 @@ def get_top_100(token, region, date, rank="spp", max_results=100):
 
     params = {
         "maxObservers": clean_max_observers(max_results),
-        "rankedBy": clean_rank(rank),
+        "rankedBy": clean_observer_rank(rank),
     }
 
     headers = {

{ebird_api_requests-4.1.1 → ebird_api_requests-4.3.0}/src/ebird/api/requests/utils.py

@@ -1,13 +1,122 @@
 """Various functions used in the API."""
 
+import datetime as dt
 import json
+import threading
 import time
+from email.utils import parsedate_to_datetime
 from urllib.error import HTTPError
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen
 
 from ebird.api.requests import constants
 
+# The maximum interval, in seconds, the rate limiter will slow down to
+# after the eBird API responds with HTTP 429 Too Many Requests.
+MAX_RATE_LIMIT_INTERVAL = 60.0
+
+
+class RateLimitError(HTTPError):
+    """Raised when the eBird API still responds with HTTP 429 Too Many
+    Requests after all the retries have been used.
+
+    It is a subclass of HTTPError so existing error handling still works.
+    The retry_after attribute contains the number of seconds the API asked
+    the caller to wait, or None if no Retry-After header was returned.
+
+    """
+
+    def __init__(self, exc, retry_after=None):
+        super().__init__(exc.url, exc.code, exc.msg, exc.headers, exc.fp)
+        self.retry_after = retry_after
+
+
+class RateLimiter:
+    """Enforce a minimum interval between the start of successive requests.
+
+    An interval of zero (the default) disables the limiter.
+
+    """
+
+    def __init__(self, interval=0.0):
+        self.interval = float(interval)
+        self.last_request = None
+        self.lock = threading.Lock()
+
+    def wait(self):
+        with self.lock:
+            if self.interval > 0 and self.last_request is not None:
+                delay = self.interval - (time.monotonic() - self.last_request)
+                if delay > 0:
+                    time.sleep(delay)
+            self.last_request = time.monotonic()
+
+    def slow_down(self):
+        with self.lock:
+            if self.interval > 0:
+                self.interval = min(self.interval * 2, MAX_RATE_LIMIT_INTERVAL)
+
+
+_rate_limiter = RateLimiter()
+
+
+def set_rate_limit(interval):
+    """Set the minimum interval, in seconds, between requests to the eBird API.
+
+    The limit applies to all requests made by the process. Setting it to
+    zero, the default, disables rate limiting. If the API responds with
+    HTTP 429 Too Many Requests, the interval is doubled, up to a maximum
+    of MAX_RATE_LIMIT_INTERVAL, for the rest of the process, or until the
+    rate limit is set again.
+
+    :param interval: the number of seconds between requests.
+    :type interval: float
+
+    """
+    if interval < 0:
+        raise ValueError("The rate limit interval cannot be negative")
+    with _rate_limiter.lock:
+        _rate_limiter.interval = float(interval)
+
+
+def get_rate_limit():
+    """Get the current minimum interval, in seconds, between requests.
+
+    :return: the interval, which may have increased since it was set, if the
+    eBird API responded with HTTP 429 Too Many Requests.
+    :rtype: float
+
+    """
+    return _rate_limiter.interval
+
+
+def get_retry_after(exc):
+    """Get the number of seconds to wait from a Retry-After header.
+
+    :param exc: the error raised for the HTTP 429 response.
+    :type exc: HTTPError
+
+    :return: the number of seconds to wait, or None if the header was not
+    present or could not be parsed.
+    :rtype: float | None
+
+    """
+    value = exc.headers.get("Retry-After") if exc.headers else None
+    if not value:
+        return None
+    try:
+        return max(float(value), 0.0)
+    except ValueError:
+        pass
+    try:
+        when = parsedate_to_datetime(value)
+    except (TypeError, ValueError):
+        return None
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=dt.timezone.utc)
+    return max((when - dt.datetime.now(dt.timezone.utc)).total_seconds(), 0.0)
+
+
 _parameter_defaults = {
     "back": constants.DEFAULT_BACK,
     "cat": constants.DEFAULT_SPECIES_CATEGORY,
@@ -101,6 +210,9 @@ def get_response(url, params=None, headers=None, max_retries=3, backoff_factor=1
     :raises URLError if there is an error with the connection to the
     eBird site.
 
+    :raises RateLimitError if the eBird API still returns HTTP 429 Too Many
+    Requests after all the retries have been used.
+
     :raises HTTPError if the eBird API returns an error.
 
     """
@@ -114,14 +226,21 @@ def get_response(url, params=None, headers=None, max_retries=3, backoff_factor=1
             request.add_header(name, value)
 
     for attempt in range(max_retries + 1):
+        _rate_limiter.wait()
         try:
             return urlopen(request).read()
         except HTTPError as exc:
-            if attempt == max_retries:
+            retry_after = None
+            if exc.code == 429:
+                retry_after = get_retry_after(exc)
+                _rate_limiter.slow_down()
+                if attempt == max_retries:
+                    raise RateLimitError(exc, retry_after) from exc
+            elif attempt == max_retries:
                 raise
-            retry_after = exc.headers.get("Retry-After") if exc.code == 429 else None
-            delay = float(retry_after) if retry_after else backoff_factor * (2**attempt)
-            time.sleep(delay)
+            if retry_after is None:
+                retry_after = backoff_factor * (2**attempt)
+            time.sleep(retry_after)
 
 
 def get_json(content):

{ebird_api_requests-4.1.1 → ebird_api_requests-4.3.0}/src/ebird/api/requests/validation.py

@@ -3,12 +3,14 @@
 """Functions for validation."""
 
 import re
+import warnings
 from datetime import date, datetime
 from enum import Enum
 
 from ebird.api.requests import constants
 
 _locales = constants.LOCALES.values()
+_observation_rank = ", ".join(constants.OBSERVATION_RANK)
 _region_types = ", ".join(constants.REGION_TYPES)
 _sort_list = ", ".join(constants.SPECIES_SORT)
 _species_categories = ", ".join(constants.SPECIES_CATEGORIES)
@@ -38,7 +40,7 @@ def is_region(value):
     return (
         re.match(r"^[A-Z]{2}$", value)
         or re.match(r"^[A-Z]{2}-[A-Z0-9]{1,3}$", value)
-        or re.match(r"^[A-Z]{2}-[A-Z0-9]{2,3}-[A-Z0-9]{2,3}$", value)
+        or re.match(r"^[A-Z]{2}-[A-Z0-9]{1,3}-[A-Z0-9]{2,3}$", value)
     )
 
 
@@ -200,6 +202,15 @@ def clean_detail(value):
     return cleaned
 
 
+def clean_observation_rank(value):
+    cleaned = clean_code(value, transform=Transform.LOWER)
+    if cleaned not in constants.OBSERVATION_RANK:
+        raise ValueError(
+            "Value for 'rank', %s, must be one of: %s" % (value, _observation_rank)
+        )
+    return cleaned
+
+
 def clean_provisional(value):
     return "true" if bool(value) else "false"
 
@@ -339,10 +350,21 @@ def clean_date(value):
     return cleaned.strftime("%Y/%m/%d")
 
 
-def clean_rank(value):
+def clean_observer_rank(value):
     cleaned = clean_code(value, transform=Transform.LOWER)
     if cleaned not in constants.TOP_100_RANK:
         raise ValueError(
             "Top 100 rank, %s, must be one or more of : %s" % (value, _top_100_rank)
         )
     return cleaned
+
+
+def clean_rank(value):
+    # Deprecated in favour of clean_observer_rank(). Remove in 5.0.0.
+    warnings.warn(
+        "clean_rank() is deprecated and will be removed in 5.0.0. "
+        "Use clean_observer_rank() instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return clean_observer_rank(value)

{ebird_api_requests-4.1.1 → ebird_api_requests-4.3.0/src/ebird_api_requests.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ebird-api-requests
-Version: 4.1.1
+Version: 4.3.0
 Summary: A python library for accessing the eBird API, version 2.0
 Author-email: Stuart MacKay <smackay@flagstonesoftware.com>
 License: MIT License
@@ -346,6 +346,30 @@ client.get_observations('MX-OAX')
 
 The client supports all the API functions.
 
+### Rate Limiting
+
+The eBird API limits how quickly requests can be made. If you exceed the
+limit the API responds with HTTP 429 Too Many Requests. If you are making
+lots of calls, for example downloading all the checklists for a region,
+then set a minimum interval, in seconds, between requests:
+
+```python
+from ebird.api.requests import set_rate_limit
+
+set_rate_limit(1.0)
+```
+
+The limit applies to all the requests made by the process. It is disabled
+by default. If the API still responds with HTTP 429, the interval is doubled,
+up to a maximum of 60 seconds, so the requests slow down to a rate the API
+accepts. Use get_rate_limit() to see the current interval.
+
+When an HTTP 429 response is returned, the request is retried after waiting
+for the time given in the Retry-After header. If the request still fails
+after all the retries are used, a RateLimitError is raised. It is a subclass
+of HTTPError, and the retry_after attribute contains the number of seconds
+the API asked you to wait, or None if no Retry-After header was returned.
+
 ## Formats
 
 Most of the eBird API calls return JSON. Some of the calls such as getting