ebird-api-data 0.1.0__py3-none-any.whl

ebird/api/data/loaders.py

@@ -0,0 +1,485 @@
+import datetime as dt
+import json
+import logging
+import random
+import re
+import socket
+import string
+import time
+
+from decimal import Decimal
+from typing import List, Optional
+from urllib.error import HTTPError, URLError
+
+from django.db import transaction
+from django.utils.timezone import get_default_timezone
+
+import requests
+
+from bs4 import BeautifulSoup
+from ebird.api.requests import get_checklist, get_regions, get_taxonomy, get_visits
+from ebird.api.requests.constants import API_MAX_RESULTS
+
+from .models import (
+    Checklist,
+    Country,
+    County,
+    Filter,
+    Location,
+    Observation,
+    Observer,
+    Species,
+    State,
+)
+
+logger = logging.getLogger(__name__)
+
+# Set timeout, in seconds, for SSL socket connections
+socket.setdefaulttimeout(30)
+# Total number number of retries to attempt
+RETRY_LIMIT: int = 10
+# Time, in seconds, to wait after an API call fails
+RETRY_WAIT: int = 2
+# Multiplier to apply to wait time after each failed attempt
+RETRY_MULTIPLIER: float = 2.0
+
+
+def str2datetime(value: str) -> dt.datetime:
+    return dt.datetime.fromisoformat(value).replace(tzinfo=get_default_timezone())
+
+
+def random_word(length):
+    letters = string.ascii_lowercase
+    return "".join(random.choice(letters) for i in range(length))
+
+
+class APILoader:
+    """
+    The APILoader downloads checklists from the eBird API and saves
+    them to the database.
+
+    Arguments:
+
+        api_key: Your key to access the eBird API.
+            Your can request a key at https://ebird.org/data/download.
+            You will need an eBird account to do so.
+
+        locales: A map of Django language codes to eBird locales so the species
+                 common name, family name, etc. is displayed in the language
+                 selected by the user.
+
+        limit: The total number of retries that can be attempted when calling
+               the API or scraping a web page. Defaults to RETRY_LIMIT if not
+               set.
+
+        wait: The initial number of seconds to wait. Defaults to RETRY_WAIT
+              if not set.
+
+        multiplier: The multiplier to apply to the wait time after each retry.
+                    Defaults to RETRY_MULTIPLIER if not set.
+
+    The eBird API limits the number of records returned to 200. When downloading
+    the visits for a given region if 200 hundred records are returned then it is
+    assumed there are more and the loader will fetch the sub-regions and download
+    the visits for each, repeating the process if necessary. To give an extreme
+    example if you download the visits for the United States, "US", then the API
+    will always return 200 results and the loader then download the visits to
+    each of the 50 states and then each of the 3143 counties. DON'T TRY THIS
+    AT HOME. Even if you don't get banned, if you melt the eBird servers, then
+    karma will ensure bad things happen to you.
+
+    The loader uses a budget for retry attempts when calling the eBird API or
+    fetching a web page. After each failure, a wait is applied, which increases
+    after each attempt. Once the total number of retries is reached, no
+    further attempts are made. You can set the number of retries, the initial
+    wait, and the multiplier when creating an APILoader object, otherwise
+    sensible defaults are used.
+
+    The default limit for retries is 10, with an initial wait of 2 seconds and
+    a multiplier also of 2. This means the loader will wait, 2, 4, 8, 16, etc.
+    seconds after each attempt. Since the loader will be run periodically,
+    using a scheduler such as cron, the retry limit is probably too high. Most
+    of the errors seen to date are timeout error when creating an SSL socket
+    connection. The are relatively rare - only one or two a week - with a
+    loader that is scheduled to run every hour. You could easily reduce the
+    limit to 3, as then, successive errors will usually mean something serious
+    is wrong with the network, or the eBird servers are overloaded, and you
+    should stop from making the situation any worse.
+
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        locales: dict,
+        limit: int = None,
+        wait: int = None,
+        multiplier: float = None,
+    ):
+        self.api_key: str = api_key
+        self.locales: dict = locales
+        self.retries: int = 0
+        self.retry_limit: int = limit if limit else RETRY_LIMIT
+        self.retry_wait: int = wait if wait else RETRY_WAIT
+        self.retry_multiplier: float = multiplier if multiplier else RETRY_MULTIPLIER
+
+    def call(self, func, *args, **kwargs):
+        wait: float = float(self.retry_wait)
+        while True:
+            try:
+                return func(*args, **kwargs)
+            except (URLError, HTTPError) as err:
+                logger.exception("Failed call #%d", self.retries)
+                self.retries += 1
+                if self.retries > self.retry_limit:
+                    logger.exception("Retry limit reached")
+                    raise err
+                time.sleep(wait)
+                wait *= self.retry_multiplier
+
+    def call_api(self, func, *args, **kwargs) -> dict | list:
+        return self.call(func, self.api_key, *args, **kwargs)
+
+    @staticmethod
+    def get_country(data: dict) -> Country:
+        code: str = data["countryCode"]
+        values: dict = {
+            "name": data["countryName"],
+            "place": data["countryName"],
+        }
+        country, created = Country.objects.get_or_create(code=code, defaults=values)
+        if created:
+            logger.info("Added country: %s, %s", code, values["name"])
+        return country
+
+    @staticmethod
+    def get_state(data: dict) -> State:
+        code: str = data["subnational1Code"]
+        values: dict = {
+            "name": data["subnational1Name"],
+            "place": "%s, %s" % (data["subnational1Name"], data["countryName"]),
+        }
+        state, created = State.objects.get_or_create(code=code, defaults=values)
+        if created:
+            logger.info("Added state: %s, %s", code, values["name"])
+        return state
+
+    @staticmethod
+    def get_county(data) -> County:
+        code: str = data["subnational2Code"]
+        values: dict = {
+            "name": data["subnational2Name"],
+            "place": "%s, %s, %s"
+            % (data["subnational2Name"], data["subnational1Name"], data["countryName"]),
+        }
+        county, created = County.objects.get_or_create(code=code, defaults=values)
+        if created:
+            logger.info("Added county: %s, %s", code, values["name"])
+        return county
+
+    def add_location(self, data: dict) -> Location:
+        identifier: str = data["locId"]
+        location: Location
+
+        values: dict = {
+            "identifier": identifier,
+            "name": data["name"],
+            "original": data["name"],
+            "country": self.get_country(data),
+            "state": self.get_state(data),
+            "county": None,
+            "hotspot": data["isHotspot"],
+            "latitude": Decimal(data["latitude"]),
+            "longitude": Decimal(data["longitude"]),
+            "url": "https://ebird.org/region/%s" % identifier,
+            "data": {"api": data},
+        }
+
+        if "subnational2Code" in data:
+            values["county"] = self.get_county(data)
+
+        location = Location.objects.create(**values)
+        logger.info("Added location: %s, %s", identifier, location.name)
+        return location
+
+    def add_species(self, code: str) -> Species:
+        """
+        Add the species with the eBird code.
+
+        Arguments:
+            code: the eBird code for the species, e.g. 'horlar' (Horned Lark).
+
+        """
+        values: dict = {
+            "common_name": {},
+            "family_common_name": {},
+            "data": {"api": {}},
+        }
+
+        for language, locale in self.locales.items():
+            data = self.call_api(get_taxonomy, locale=locale, species=code)[0]
+            values["taxon_order"] = int(data["taxonOrder"])
+            values["order"] = data.get("order", "")
+            values["category"] = data["category"]
+            values["family_code"] = data["familyCode"]
+            values["common_name"][language] = data["comName"]
+            values["scientific_name"] = data["sciName"]
+            values["family_common_name"][language] = data["familyComName"]
+            values["family_scientific_name"] = data["familySciName"]
+            values["data"]["api"][locale] = data
+
+        values["common_name"] = json.dumps(values["common_name"])
+        values["family_common_name"] = json.dumps(values["family_common_name"])
+
+        species = Species.objects.create(species_code=code, **values)
+        logger.info("Added species: %s, %s", code, species.get_common_name())
+
+        return species
+
+    def get_species(self, data: dict) -> Species:
+        code: str = data["speciesCode"]
+        species = Species.objects.filter(species_code=code).first()
+        if species is None:
+            species = self.add_species(code)
+        return species
+
+    @staticmethod
+    def get_urn(project_id, row: dict) -> str:
+        return f"URN:CornellLabOfOrnithology:{project_id}:{row['obsId']}"
+
+    def add_observation(self, data: dict, checklist: Checklist) -> Observation:
+        identifier: str = data["obsId"]
+        observation: Observation
+        species: Species = self.get_species(data)
+
+        values: dict = {
+            "edited": checklist.edited,
+            "published": False,
+            "checklist": checklist,
+            "country": checklist.country,
+            "state": checklist.state,
+            "county": checklist.county,
+            "location": checklist.location,
+            "observer": checklist.observer,
+            "species": species,
+            "date": checklist.date,
+            "time": checklist.time,
+            "started": checklist.started,
+            "count": 0,
+            "audio": False,
+            "photo": False,
+            "video": False,
+            "comments": "",
+            "urn": self.get_urn(checklist.project_code, data),
+            "data": {"api": data},
+        }
+
+        if re.match(r"\d+", data["howManyStr"]):
+            values["count"] = int(data["howManyStr"])
+
+        if "mediaCounts" in data:
+            values["audio"] = "A" in data["mediaCounts"]
+            values["photo"] = "P" in data["mediaCounts"]
+            values["video"] = "V" in data["mediaCounts"]
+
+        if "comments" in data:
+            values["comments"] = data["comments"]
+
+        observation = Observation.objects.create(identifier=identifier, **values)
+
+        return observation
+
+    def get_observer_identifier(self, data) -> str:
+        identifier: str = data["subId"]
+        logger.info("Scraping checklist: %s", identifier)
+        response = self.call(
+            requests.get, "https://ebird.org/checklist/%s" % identifier
+        )
+        content = response.content
+        soup = BeautifulSoup(content, "lxml")
+        attribute = "data-participant-userid"
+        node = soup.find("span", attrs={attribute: True})
+        return node[attribute] if node else ""
+
+    def get_observer(self, data: dict) -> Observer:
+        name: str = data.get("userDisplayName", "Anonymous eBirder")
+        observer, created = Observer.objects.get_or_create(
+            original=name, defaults={"name": name}
+        )
+        if observer.multiple:
+            observer, created = Observer.objects.get_or_create(
+                identifier=self.get_observer_identifier(data),
+                defaults={"original": random_word(8), "name": name},
+            )
+        elif observer.identifier == "":
+            observer.identifier = self.get_observer_identifier(data)
+            observer.save()
+        if created:
+            logger.info("Added observer: %s", observer.name)
+        return observer
+
+    def add_checklist(self, identifier: str) -> Checklist | None:
+        """
+        Add the checklist with the given identifier.
+
+        Arguments:
+            identifier: the eBird identifier for the checklist, e.g. "S318722167"
+
+        """
+        logger.info("Adding checklist: %s", identifier)
+
+        # Make sure loading a checklist is an all or nothing proposition.
+        # All the data is available from the eBird API call but there can
+        # still be further calls to scrape the checklist web page to get
+        # identifier of the observer, or the eBird API when a new species
+        # is added.
+
+        with transaction.atomic():
+            data: dict = self.call_api(get_checklist, identifier)
+            identifier: str = data["subId"]
+            created: dt.datetime = str2datetime(data["creationDt"])
+            edited: dt.datetime = str2datetime(data["lastEditedDt"])
+            started: dt.datetime = str2datetime(data["obsDt"])
+            location: Location = Location.objects.get(identifier=data["locId"])
+            checklist: Checklist
+            observer: Observer = self.get_observer(data)
+            observations: list = data.pop("obs", [])
+
+            if not observer.enabled:
+                return None
+
+            values: dict = {
+                "created": created,
+                "edited": edited,
+                "published": False,
+                "country": location.country,
+                "state": location.state,
+                "county": location.county,
+                "location": location,
+                "observer": observer,
+                "observer_count": None,
+                "species_count": data["numSpecies"],
+                "date": started.date(),
+                "time": None,
+                "started": started,
+                "protocol_code": data["protocolId"],
+                "project_code": data["projId"],
+                "duration": None,
+                "complete": data["allObsReported"],
+                "comments": "",
+                "url": "https://ebird.org/checklist/%s" % identifier,
+                "data": {"api": data},
+            }
+
+            if data["obsTimeValid"]:
+                values["time"] = started.time()
+
+            if "numObservers" in data:
+                values["observer_count"] = int(data["numObservers"])
+
+            if duration := data.get("durationHrs"):
+                values["duration"] = int(duration * 60.0)
+
+            if data["protocolId"] == "P22":
+                # The distance travelled might not be reported.
+                if "effortDistanceKm" in data:
+                    dist: str = data["effortDistanceKm"]
+                    values["distance"] = round(Decimal(dist), 3)
+            elif data["protocolId"] == "P23":
+                area: str = data["effortAreaHa"]
+                values["area"] = round(Decimal(area), 3)
+
+            if "comments" in data:
+                values["comments"] = data["comments"]
+
+            checklist = Checklist.objects.create(identifier=identifier, **values)
+
+            for observation_data in observations:
+                self.add_observation(observation_data, checklist)
+
+        return checklist
+
+    def fetch_subregions(self, region: str) -> List[str]:
+        region_types: list = ["subnational1", "subnational2", None]
+        levels: int = len(region.split("-", 2))
+        region_type: Optional[str] = region_types[levels - 1]
+
+        if region_type:
+            items: list = self.call_api(get_regions, region_type, region)
+            sub_regions = [item["code"] for item in items]
+        else:
+            sub_regions = []
+
+        return sub_regions
+
+    def fetch_visits(self, region: str, date: dt.date):
+        visits = []
+
+        results: list = get_visits(
+            self.api_key, region, date=date, max_results=API_MAX_RESULTS
+        )
+
+        if len(results) == API_MAX_RESULTS:
+            logger.info("API result limit reached - fetching visits for subregions")
+            if sub_regions := self.fetch_subregions(region):
+                for sub_region in sub_regions:
+                    logger.info("Fetching visits for sub-region: %s", sub_region)
+                    visits.extend(self.fetch_visits(sub_region, date))
+            else:
+                # No more sub-regions, issue a warning and return the results
+                visits.extend(results)
+                logger.warning(
+                    "Fetching visits - API limit reached: %s, %s", region, date
+                )
+        else:
+            visits.extend(results)
+
+        return visits
+
+    def add_checklists(self, region: str, date: dt.date) -> None:
+        """
+        Add all the checklists submitted for a region for a given date.
+
+        Arguments:
+            region: The code for a national, subnational1, subnational2
+                 area or hotspot identifier. For example, US, US-NY,
+                 US-NY-109, or L1379126, respectively.
+
+            date: The date the observations were made.
+
+        """
+
+        logger.info("Adding checklists: %s, %s", region, date)
+
+        visits: list[dict] = self.fetch_visits(region, date)
+
+        logger.info("Visits made: %d ", len(visits))
+
+        for visit in visits:
+            data = visit["loc"]
+            identifier = data["locId"]
+            if not Location.objects.filter(identifier=identifier).exists():
+                self.add_location(data)
+
+        added: int = 0
+
+        for visit in visits:
+            identifier = visit["subId"]
+            if not Checklist.objects.filter(identifier=identifier).exists():
+                self.add_checklist(identifier)
+                added += 1
+
+        logger.info("Checklists added: %d ", added)
+        logger.info("Adding checklists completed")
+
+    @staticmethod
+    def run_filters():
+        for filter in Filter.objects.filter(enabled=True):
+            logger.info("Applying filter: %s", filter.name)
+            count = filter.apply()
+            logger.info("Observations updated: %d", count)
+
+    @staticmethod
+    def publish():
+        Checklist.objects.filter(published=False).update(published=True)
+        Observation.objects.filter(published=False).update(published=True)

ebird/api/data/management/commands/add_checklists.py

@@ -0,0 +1,124 @@
+"""
+load_api.py
+
+A Django management command for loading observations from the eBird API.
+
+Usage:
+    python manage.py add_checklists --date <date> <region>+
+    python manage.py add_checklists --days <n> <region>+
+    python manage.py add_checklists --date <date> --days <n> <region>+
+
+Arguments:
+    --date <date> The date of the checklists to add to the database. If
+        no date is given then it defaults to today.
+
+    --days <n> The number of days to include. If not given then the number
+        of days defaults to 1. That means only checklists for the date will
+        be added.
+
+    <region> Required. One or more national, subnational1, subnational2, or hotspot
+        codes used by eBird. For example, US, US-NY, US-NY-109, L1379126
+
+Examples:
+    # Load checklists added in the past week for New York state
+    python manage.py add_checklists --days 7 US-NY
+
+    # Load checklists all checklists added on the last Big Day
+    python manage.py add_checklists --date 2024-10-05 US-NY
+
+Notes:
+    1. The eBird API returns a maximum of 200 results. The APILoader works
+       around this by fetching checklists from sub-regions if necessary.
+       Downloading checklists once a day should be sufficient for all hotspots
+       or subnational2 areas. For large countries or places with lots of birders
+       downloads will have to be more frequent. For really large area, i.e. the
+       USA you shouldn't be using the API at all. Instead use the data from the
+       eBird Basic Dataset.
+
+    2. Why does the command only add checklists, not update existing ones?
+       The number of checklists that are updated is relatively small, typically
+       less than 1%. The problem with the eBird API is that you can only find
+       out whether a checklist has changed by downloading it. This app basically
+       mirrors the eBird database for a given region so there's a strong temptation
+       to repeatedly download everything to keep the checklists in sync. That
+       means repeatedly downloading all the checklists submitted in the past week
+       or month, or longer to pick up a few changes. That's a heavy load on the
+       eBird servers and a lot of bandwidth for relatively little gain.
+
+       The API is really a news service. For accuracy and completeness you should
+       really use the eBird Basic Dataset, which is published on the 15th of each
+       month.
+
+    3. The API has limitations. Observers are only identified by name. So if there
+       are two Juan Garcias birding in a region, then all the observations will
+       appear to belong to one person. To work around this, when an observer is
+       added the web page for the checklist is scraped to extract their eBird
+       user id.
+
+       Also the observations will not have been reviewed by moderators, so there
+       are likely to be records where the identification is incorrect.
+
+    4. You automate running the command using a scheduler such as cron. If you use
+       the absolute paths to python and the command, then you don't need to deal
+       with activating the virtual environment, for example:
+
+       # Just before midnight, add all checklists submitted for the past week
+       55 23 * * * /home/me/my-project/.venv/bin/python /home/me/my-project/manage.py add_checklists --days 7 US-NY
+
+"""
+
+import datetime as dt
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ebird.api.data.loaders import APILoader
+
+
+class Command(BaseCommand):
+    help = "Add checklists from the eBird API"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--date",
+            type=lambda s: dt.datetime.strptime(s, "%Y-%m-%d"),
+            help="The number of previous days to add",
+        )
+
+        parser.add_argument(
+            "--days",
+            type=int,
+            help="The number of previous days to add",
+        )
+
+        parser.add_argument(
+            "regions",
+            nargs="+",
+            type=str,
+            help="Codes for the eBird regions, e.g US-NY",
+        )
+
+    @staticmethod
+    def get_loader() -> APILoader:
+        key: str = getattr(settings, "EBIRD_API_KEY")
+        locales: dict = getattr(settings, "EBIRD_LOCALES")
+        return APILoader(key, locales)
+
+    @staticmethod
+    def get_dates(days) -> list[dt.date]:
+        today: dt.date = dt.date.today()
+        return [today - dt.timedelta(days=n) for n in range(days)]
+
+    def handle(self, *args, **options):
+        date = options["date"] or dt.date.today()
+        days = options["days"] or 1
+
+        dates = [date - dt.timedelta(days=n) for n in range(days)]
+        loader = self.get_loader()
+
+        for region in options["regions"]:
+            for date in dates:
+                loader.add_checklists(region, date)
+
+        loader.run_filters()
+        loader.publish()