ebird-api-data 0.5.2__tar.gz → 0.5.3__tar.gz

{ebird_api_data-0.5.2 → ebird_api_data-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ebird-api-data
-Version: 0.5.2
+Version: 0.5.3
 Summary: eBird API Data is a reusable Django app for loading data from eBird into a database.
 Author-email: Stuart MacKay <smackay@flagstonesoftware.com>
 License-Expression: MIT
@@ -21,10 +21,8 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Python: >=3.10
-Requires-Dist: beautifulsoup4>=4.13.3
 Requires-Dist: django>=4.2
-Requires-Dist: ebird-api-requests>=4.1.1
-Requires-Dist: lxml>=6.0.0
+Requires-Dist: ebird-api-requests>=4.3.1
 Description-Content-Type: text/markdown
 
 # eBird API Data
@@ -165,8 +163,12 @@ be assigned to them. In practice, most people ensure their name is unique by
 adding middle initial's etc. However it is a problem with "Anonymous eBirder".
 The solution is to add two Observer records with the name "Anonymous eBirder",
 and a fake identifier, e.g. ANON1, ANON2, etc. That way the loader will first 
-scrape the web page for the checklist where the identifier, e.g. USER2743448, 
-can be found, and so the checklist will be assigned to the correct Observer. 
+fetch the Top 100 observers for the checklist location, on the date of the
+checklist, where the identifier, e.g. USER2743448, can be found, and so the
+checklist will be assigned to the correct Observer. If the identifier cannot
+be found, e.g. two observers with the same name submitted checklists at the
+same location on the same day, the checklist is assigned to the first
+Observer added with that name.
 Use this workaround for any other observers you encounter that share the same 
 name. This won't work retroactively, but then you can delete the checklists, 
 and manually reload them.

{ebird_api_data-0.5.2 → ebird_api_data-0.5.3}/README.md

@@ -136,8 +136,12 @@ be assigned to them. In practice, most people ensure their name is unique by
 adding middle initial's etc. However it is a problem with "Anonymous eBirder".
 The solution is to add two Observer records with the name "Anonymous eBirder",
 and a fake identifier, e.g. ANON1, ANON2, etc. That way the loader will first 
-scrape the web page for the checklist where the identifier, e.g. USER2743448, 
-can be found, and so the checklist will be assigned to the correct Observer. 
+fetch the Top 100 observers for the checklist location, on the date of the
+checklist, where the identifier, e.g. USER2743448, can be found, and so the
+checklist will be assigned to the correct Observer. If the identifier cannot
+be found, e.g. two observers with the same name submitted checklists at the
+same location on the same day, the checklist is assigned to the first
+Observer added with that name.
 Use this workaround for any other observers you encounter that share the same 
 name. This won't work retroactively, but then you can delete the checklists, 
 and manually reload them.

{ebird_api_data-0.5.2 → ebird_api_data-0.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ebird-api-data"
-version = "0.5.2"
+version = "0.5.3"
 description = "eBird API Data is a reusable Django app for loading data from eBird into a database."
 keywords = ["checklists", "observations", "species", "Django", "eBird", "models"]
 license = "MIT"
@@ -11,10 +11,8 @@ authors = [
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
-    "beautifulsoup4>=4.13.3",
     "django>=4.2",
-    "ebird-api-requests>=4.1.1",
-    "lxml>=6.0.0",
+    "ebird-api-requests>=4.3.1",
 ]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -64,7 +62,7 @@ dev-dependencies = [
 ]
 
 [tool.bumpversion]
-current_version = "0.5.2"
+current_version = "0.5.3"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
 serialize = ["{major}.{minor}.{patch}"]
 ignore_missing_version = false

ebird_api_data-0.5.3/src/ebird/api/data/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.3"

{ebird_api_data-0.5.2 → ebird_api_data-0.5.3}/src/ebird/api/data/admin.py

@@ -303,7 +303,8 @@ class FetchSpeciesView(PermissionRequiredMixin, generic.FormView):
         species_code = form.cleaned_data["species_code"]
         key: str = getattr(settings, "EBIRD_API_KEY")
         locales: dict = getattr(settings, "EBIRD_LOCALES")
-        loader = APILoader(key, locales)
+        rate_limit: float | None = getattr(settings, "EBIRD_RATE_LIMIT", None)
+        loader = APILoader(key, locales, rate_limit=rate_limit)
         species = loader.add_species(species_code)
         messages.add_message(
             self.request,

{ebird_api_data-0.5.2 → ebird_api_data-0.5.3}/src/ebird/api/data/loaders.py

@@ -15,10 +15,15 @@ from urllib.error import HTTPError, URLError
 from django.db import transaction
 from django.utils.timezone import get_default_timezone
 
-import requests
-
-from bs4 import BeautifulSoup
-from ebird.api.requests import get_checklist, get_regions, get_taxonomy, get_visits
+from ebird.api.requests import (
+    RateLimitError,
+    get_checklist,
+    get_regions,
+    get_taxonomy,
+    get_top_100,
+    get_visits,
+    set_rate_limit,
+)
 from ebird.api.requests.constants import API_MAX_RESULTS
 
 from .models import (
@@ -42,6 +47,8 @@ RETRY_LIMIT: int = 10
 RETRY_WAIT: int = 2
 # Multiplier to apply to wait time after each failed attempt
 RETRY_MULTIPLIER: float = 2.0
+# Minimum time, in seconds, between requests to the eBird API
+RATE_LIMIT: float = 1.0
 
 
 def str2datetime(value: str) -> dt.datetime:
@@ -70,9 +77,9 @@ class APILoader:
                  common name, family name, etc. is displayed in the language
                  selected by the user.
 
-        limit: The total number of retries that can be attempted when calling
-               the API or scraping a web page. Defaults to RETRY_LIMIT if not
-               set.
+        limit: The total number of retries that can be attempted when the
+               connection to the eBird API fails. Defaults to RETRY_LIMIT if
+               not set.
 
         wait: The initial number of seconds to wait. Defaults to RETRY_WAIT
               if not set.
@@ -80,6 +87,10 @@ class APILoader:
         multiplier: The multiplier to apply to the wait time after each retry.
                     Defaults to RETRY_MULTIPLIER if not set.
 
+        rate_limit: The minimum time, in seconds, between requests to the
+                    eBird API. Defaults to RATE_LIMIT if not set. Zero disables
+                    rate limiting.
+
     The eBird API limits the number of records returned to 200. When downloading
     the visits for a given region if 200 hundred records are returned then it is
     assumed there are more and the loader will fetch the sub-regions and download
@@ -90,9 +101,9 @@ class APILoader:
     AT HOME. Even if you don't get banned, if you melt the eBird servers, then
     karma will ensure bad things happen to you.
 
-    The loader uses a budget for retry attempts when calling the eBird API or
-    fetching a web page. After each failure, a wait is applied, which increases
-    after each attempt. Once the total number of retries is reached, no
+    The loader uses a budget for retry attempts when the connection to the
+    eBird API fails, e.g. a timeout. After each failure, a wait is applied,
+    which increases after each attempt. Once the total number of retries is reached, no
     further attempts are made. You can set the number of retries, the initial
     wait, and the multiplier when creating an APILoader object, otherwise
     sensible defaults are used.
@@ -108,6 +119,19 @@ class APILoader:
     is wrong with the network, or the eBird servers are overloaded, and you
     should stop from making the situation any worse.
 
+    Errors returned by the eBird API, e.g. HTTP 500 Internal Server Error,
+    are not retried by the loader since ebird-api-requests already retries
+    the request, with exponential back-off, before raising an HTTPError.
+
+    The eBird API also limits how quickly requests can be made. The loader
+    sets a rate limit, using set_rate_limit() from ebird-api-requests, so
+    there is a minimum interval between requests. The limit applies to all
+    the requests made by the process. If the API responds with HTTP 429 Too
+    Many Requests, ebird-api-requests waits, retries the request and slows
+    down the rate of subsequent requests. If the request still fails, the
+    loader stops rather than adding to the load on the eBird servers. Any
+    missing checklists will be added the next time the loader is run.
+
     """
 
     def __init__(
@@ -117,6 +141,7 @@ class APILoader:
         limit: int = None,
         wait: int = None,
         multiplier: float = None,
+        rate_limit: float = None,
     ):
         self.api_key: str = api_key
         self.locales: dict = locales
@@ -124,13 +149,21 @@ class APILoader:
         self.retry_limit: int = limit if limit else RETRY_LIMIT
         self.retry_wait: int = wait if wait else RETRY_WAIT
         self.retry_multiplier: float = multiplier if multiplier else RETRY_MULTIPLIER
+        self.rate_limit: float = RATE_LIMIT if rate_limit is None else rate_limit
+        set_rate_limit(self.rate_limit)
 
     def call(self, func, *args, **kwargs):
         wait: float = float(self.retry_wait)
         while True:
             try:
                 return func(*args, **kwargs)
-            except (URLError, HTTPError) as err:
+            except RateLimitError:
+                logger.exception("Rate limit exceeded")
+                raise
+            except HTTPError:
+                # ebird-api-requests has already retried the request.
+                raise
+            except (URLError, TimeoutError) as err:
                 logger.exception("Failed call #%d", self.retries)
                 self.retries += 1
                 if self.retries > self.retry_limit:
@@ -359,27 +392,55 @@ class APILoader:
                 )
         return observation
 
-    def get_observer_identifier(self, data) -> str:
+    def get_observer_identifier(self, data: dict) -> str:
+        # The checklist only contains the observer's name. The only place the
+        # eBird API returns the observer's identifier, e.g. USER59111, is the
+        # Top 100 for a region or location. Fetching the Top 100 for the
+        # location, on the date of the checklist, returns a short list, so
+        # the name will almost always be unique. The userId field is not
+        # documented, so it might be removed at some point.
         checklist_identifier: str = data["subId"]
-        logger.info("Scraping checklist: %s", checklist_identifier)
-        response = self.call(
-            requests.get, "https://ebird.org/checklist/%s" % checklist_identifier
-        )
-        content = response.content
-        soup = BeautifulSoup(content, "lxml")
-        attribute = "data-participant-userid"
-        node = soup.find("span", attrs={attribute: True})
-        identifier = node[attribute] if node else ""
-        if not identifier:
-            logger.error("Observer Identifier: not found")
-        return identifier
+        name: str = data.get("userDisplayName", "Anonymous eBirder")
+        location: str = data["locId"]
+        date: dt.date = str2datetime(data["obsDt"]).date()
+
+        entries: list = self.call_api(get_top_100, location, date)
+        identifiers: set = {
+            entry["userId"]
+            for entry in entries
+            if entry.get("userDisplayName") == name and entry.get("userId")
+        }
+
+        if len(identifiers) == 1:
+            return identifiers.pop()
+
+        if identifiers:
+            logger.warning(
+                "Checklist %s: observer %s is not unique at %s on %s",
+                checklist_identifier,
+                name,
+                location,
+                date,
+            )
+        else:
+            logger.warning(
+                "Checklist %s: identifier for observer %s not found",
+                checklist_identifier,
+                name,
+            )
+        return ""
+
+    @staticmethod
+    def generate_observer_identifier() -> str:
+        # Use a prefix which will not clash with eBird's USER identifiers.
+        return "UNKNOWN-%s" % random_word(7).upper()
 
     def get_observer(self, data: dict) -> Observer:
         # In theory, there is a serious problem here, as observers can have
         # the same name. There is no way to resolve this except to prime the
         # database with multiple Observers sharing the same name. That will
-        # force the identifier to be scraped from the checklist web page and
-        # so the correct "John Smith" will be used. In practice, apart from
+        # force the identifier to be fetched from the eBird API and so the
+        # correct "John Smith" will be used. In practice, apart from
         # Anonymous eBirder, this is not a problem. Most observers want their
         # name to be unique (so they show up in the leader boards), and so they
         # will add middle initials, etc. so the names are generally unique.
@@ -388,21 +449,40 @@ class APILoader:
         number_of_observers = Observer.objects.filter(original=name).count()
 
         if number_of_observers == 0:
-            identifier = self.get_observer_identifier(data)
-            # The observer might have changed their name.
-            observer, created = Observer.objects.update_or_create(
-                identifier=identifier, defaults={"name": name, "original": name}
-            )
+            if identifier := self.get_observer_identifier(data):
+                # The observer might have changed their name.
+                observer, created = Observer.objects.update_or_create(
+                    identifier=identifier, defaults={"name": name, "original": name}
+                )
+            else:
+                identifier = self.generate_observer_identifier()
+                observer = Observer.objects.create(
+                    identifier=identifier, name=name, original=name
+                )
+                created = True
         elif number_of_observers == 1:
             observer = Observer.objects.get(original=name)
             identifier = observer.identifier
             created = False
-        else:
-            identifier = self.get_observer_identifier(data)
+        elif identifier := self.get_observer_identifier(data):
             observer, created = Observer.objects.get_or_create(
                 identifier=identifier,
                 defaults={"name": name, "original": name},
             )
+        else:
+            # There is no way to tell which observer submitted the checklist
+            # so use the first one added.
+            observer = (
+                Observer.objects.filter(original=name).order_by("created").first()
+            )
+            identifier = observer.identifier
+            created = False
+            logger.warning(
+                "Checklist %s: assigned to observer %s (%s)",
+                data["subId"],
+                name,
+                identifier,
+            )
 
         if created:
             logger.info("Observer added: %s (%s)", name, identifier)
@@ -419,9 +499,8 @@ class APILoader:
         """
         # Make sure loading a checklist is an all or nothing proposition.
         # All the data is available from the eBird API call but there can
-        # still be further calls to scrape the checklist web page to get
-        # identifier of the observer, or the eBird API when a new species
-        # is added.
+        # still be further calls to the eBird API to get the identifier of
+        # the observer, or when a new species is added.
 
         with transaction.atomic():
             data: dict = self.call_api(get_checklist, identifier)
@@ -552,8 +631,8 @@ class APILoader:
     def fetch_visits(self, region: str, date: dt.date):
         visits = []
 
-        results: list = get_visits(
-            self.api_key, region, date=date, max_results=API_MAX_RESULTS
+        results: list = self.call_api(
+            get_visits, region, date=date, max_results=API_MAX_RESULTS
         )
 
         if len(results) == API_MAX_RESULTS:

{ebird_api_data-0.5.2 → ebird_api_data-0.5.3}/src/ebird/api/data/management/commands/add_checklists.py

@@ -65,8 +65,8 @@ Notes:
     3. The API has limitations. Observers are only identified by name. So if there
        are two Juan Garcias birding in a region, then all the observations will
        appear to belong to one person. To work around this, when an observer is
-       added the web page for the checklist is scraped to extract their eBird
-       user id.
+       added, their eBird user id is found by fetching the Top 100 observers for
+       the location, on the date of the checklist.
 
        Also the observations will not have been reviewed by moderators, so there
        are likely to be records where the identification is incorrect.
@@ -121,7 +121,8 @@ class Command(BaseCommand):
     def get_loader() -> APILoader:
         key: str = getattr(settings, "EBIRD_API_KEY")
         locales: dict = getattr(settings, "EBIRD_LOCALES")
-        return APILoader(key, locales)
+        rate_limit: float | None = getattr(settings, "EBIRD_RATE_LIMIT", None)
+        return APILoader(key, locales, rate_limit=rate_limit)
 
     def handle(self, *args, **options):
         date = options["date"] or dt.date.today()

ebird_api_data-0.5.3/src/ebird/api/data/management/commands/repair_observers.py

@@ -0,0 +1,185 @@
+"""
+repair_observers.py
+
+A Django management command for repairing checklists assigned to an
+Observer by mistake.
+
+Usage:
+    python manage.py repair_observers [--identifier <identifier>] [--dry-run]
+
+Arguments:
+    --identifier <identifier> The identifier of the Observer to repair. If
+        not given then it defaults to the empty string.
+
+    --dry-run Show what would be changed, without changing the database.
+
+Examples:
+    # Repair the checklists assigned to the Observer with an empty identifier
+    python manage.py repair_observers
+
+    # Show which observers the checklists would be assigned to
+    python manage.py repair_observers --dry-run
+
+Notes:
+    1. Up to version 0.5.2, the loader scraped the checklist web page to get
+       the observer's eBird user id. Since eBird started blocking bots, the
+       identifier scraped was always empty, so every new observer was saved
+       to the same Observer record, with an empty identifier. The record was
+       renamed each time a new observer was added.
+
+    2. To repair the checklists, the name of each observer is fetched from the
+       list of visits for the checklist's state, on the date of the checklist.
+       That takes far fewer calls to the eBird API than fetching each checklist.
+       If the checklist cannot be found then it is fetched directly. The
+       observer is then found, or added, in the same way as when the checklist
+       is loaded, and the checklist and its observations are reassigned.
+
+    3. Once all the checklists have been reassigned, the Observer record is
+       deleted. If any checklists could not be repaired, e.g. they were deleted
+       from eBird, then they remain assigned to the Observer, and the command
+       can be run again.
+
+"""
+
+import datetime as dt
+import logging
+
+from urllib.error import HTTPError
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from ebird.api.requests import get_checklist
+
+from ebird.api.data.loaders import APILoader
+from ebird.api.data.models import Checklist, Observer
+
+logger = logging.getLogger(__name__)
+
+# The name given to the Observer being repaired, so the loader does not
+# assign checklists to it when looking up observers by name.
+REPAIR_NAME = "Observer being repaired"
+
+
+class Command(BaseCommand):
+    help = "Reassign checklists assigned to an Observer by mistake"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--identifier",
+            type=str,
+            default="",
+            help="The identifier of the Observer to repair. Defaults to ''.",
+        )
+
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Show what would be changed, without changing the database.",
+        )
+
+    @staticmethod
+    def get_loader() -> APILoader:
+        key: str = settings.EBIRD_API_KEY
+        locales: dict = settings.EBIRD_LOCALES
+        rate_limit: float | None = getattr(settings, "EBIRD_RATE_LIMIT", None)
+        return APILoader(key, locales, rate_limit=rate_limit)
+
+    @staticmethod
+    def get_names(loader: APILoader, region: str, date: dt.date) -> dict[str, str]:
+        visits: list[dict] = loader.fetch_visits(region, date)
+        return {
+            visit["subId"]: visit.get("userDisplayName", "Anonymous eBirder")
+            for visit in visits
+        }
+
+    @staticmethod
+    def get_name(loader: APILoader, identifier: str) -> str | None:
+        try:
+            data: dict = loader.call_api(get_checklist, identifier)
+        except HTTPError as err:
+            logger.warning("Checklist %s: could not be fetched, %s", identifier, err)
+            return None
+        return data.get("userDisplayName", "Anonymous eBirder")
+
+    def handle(self, *args, **options):
+        identifier: str = options["identifier"]
+        dry_run: bool = options["dry_run"]
+
+        merged: Observer | None = Observer.objects.filter(identifier=identifier).first()
+
+        if merged is None:
+            raise CommandError("Observer '%s' does not exist" % identifier)
+
+        checklists = (
+            Checklist.objects.filter(observer=merged)
+            .select_related("state", "location")
+            .order_by("date", "identifier")
+        )
+
+        logger.info(
+            "Observer '%s' (%s): %d checklists to repair",
+            identifier,
+            merged.name,
+            checklists.count(),
+        )
+
+        loader: APILoader = self.get_loader()
+        names: dict[tuple, dict[str, str]] = {}
+        repaired: int = 0
+        skipped: int = 0
+
+        with transaction.atomic():
+            # Rename the Observer so the loader does not match it by name.
+            merged.name = REPAIR_NAME
+            merged.original = REPAIR_NAME
+            merged.save()
+
+            for checklist in checklists:
+                key = (checklist.state.code, checklist.date)
+                if key not in names:
+                    names[key] = self.get_names(loader, *key)
+
+                name = names[key].get(checklist.identifier)
+                if name is None:
+                    name = self.get_name(loader, checklist.identifier)
+
+                if name is None:
+                    skipped += 1
+                    continue
+
+                data: dict = {
+                    "subId": checklist.identifier,
+                    "userDisplayName": name,
+                    "locId": checklist.location.identifier,
+                    "obsDt": checklist.started.strftime("%Y-%m-%d %H:%M"),
+                }
+
+                observer: Observer = loader.get_observer(data)
+
+                if observer.pk == merged.pk:
+                    skipped += 1
+                    continue
+
+                checklist.observer = observer
+                checklist.save()
+                checklist.observations.update(observer=observer)
+                repaired += 1
+
+                logger.info(
+                    "Checklist %s: assigned to %s (%s)",
+                    checklist.identifier,
+                    observer.name,
+                    observer.identifier,
+                )
+
+            if not merged.checklists.exists() and not merged.observations.exists():
+                merged.delete()
+                logger.info("Observer '%s': deleted", identifier)
+
+            logger.info("Repaired %d checklists, skipped %d", repaired, skipped)
+
+            if dry_run:
+                transaction.set_rollback(True)
+                logger.info("Dry run: no changes were saved")

ebird_api_data-0.5.2/src/ebird/api/data/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.5.2"