ebird-api-data 0.4.0__tar.gz → 0.5.1__tar.gz

{ebird_api_data-0.4.0 → ebird_api_data-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ebird-api-data
-Version: 0.4.0
+Version: 0.5.1
 Summary: eBird API Data is a reusable Django app for loading data from eBird into a database.
 Author-email: Stuart MacKay <smackay@flagstonesoftware.com>
 License-Expression: MIT
@@ -23,7 +23,7 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Python: >=3.10
 Requires-Dist: beautifulsoup4>=4.13.3
 Requires-Dist: django>=4.2
-Requires-Dist: ebird-api-requests==4.0.2
+Requires-Dist: ebird-api-requests>=4.1.0
 Requires-Dist: lxml>=6.0.0
 Description-Content-Type: text/markdown
 
@@ -158,6 +158,19 @@ python manage.py runserver
 Now, either visit the site, http:localhost:8000/, or log into the Django Admin, 
 http:localhost:8000/admin to browse the tables.
 
+IMPORTANT: There is a serious flaw in the eBird API - observers are identified
+by name. That means if more than one observer shares the same name then only 
+one Observer record will be added to the database, and all the checklists will
+be assigned to them. In practice, most people ensure their name is unique by 
+adding middle initial's etc. However it is a problem with "Anonymous eBirder".
+The solution is to add two Observer records with the name "Anonymous eBirder",
+and a fake identifier, e.g. ANON1, ANON2, etc. That way the loader will first 
+scrape the web page for the checklist where the identifier, e.g. USER2743448, 
+can be found, and so the checklist will be assigned to the correct Observer. 
+Use this workaround for any other observers you encounter that share the same 
+name. This won't work retroactively, but then you can delete the checklists, 
+and manually reload them.
+
 ## Project Information
 
 * Documentation: https://ebird-api-data.readthedocs.io/en/latest/

{ebird_api_data-0.4.0 → ebird_api_data-0.5.1}/README.md

@@ -129,6 +129,19 @@ python manage.py runserver
 Now, either visit the site, http:localhost:8000/, or log into the Django Admin, 
 http:localhost:8000/admin to browse the tables.
 
+IMPORTANT: There is a serious flaw in the eBird API - observers are identified
+by name. That means if more than one observer shares the same name then only 
+one Observer record will be added to the database, and all the checklists will
+be assigned to them. In practice, most people ensure their name is unique by 
+adding middle initial's etc. However it is a problem with "Anonymous eBirder".
+The solution is to add two Observer records with the name "Anonymous eBirder",
+and a fake identifier, e.g. ANON1, ANON2, etc. That way the loader will first 
+scrape the web page for the checklist where the identifier, e.g. USER2743448, 
+can be found, and so the checklist will be assigned to the correct Observer. 
+Use this workaround for any other observers you encounter that share the same 
+name. This won't work retroactively, but then you can delete the checklists, 
+and manually reload them.
+
 ## Project Information
 
 * Documentation: https://ebird-api-data.readthedocs.io/en/latest/

{ebird_api_data-0.4.0 → ebird_api_data-0.5.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ebird-api-data"
-version = "0.4.0"
+version = "0.5.1"
 description = "eBird API Data is a reusable Django app for loading data from eBird into a database."
 keywords = ["checklists", "observations", "species", "Django", "eBird", "models"]
 license = "MIT"
@@ -13,7 +13,7 @@ requires-python = ">=3.10"
 dependencies = [
     "beautifulsoup4>=4.13.3",
     "django>=4.2",
-    "ebird-api-requests==4.0.2",
+    "ebird-api-requests>=4.1.0",
     "lxml>=6.0.0",
 ]
 classifiers = [
@@ -64,7 +64,7 @@ dev-dependencies = [
 ]
 
 [tool.bumpversion]
-current_version = "0.4.0"
+current_version = "0.5.1"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
 serialize = ["{major}.{minor}.{patch}"]
 ignore_missing_version = false

ebird_api_data-0.5.1/src/ebird/api/data/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.1"

{ebird_api_data-0.4.0 → ebird_api_data-0.5.1}/src/ebird/api/data/admin.py

@@ -266,10 +266,10 @@ class ObservationAdmin(admin.ModelAdmin):
 
 @admin.register(models.Observer)
 class ObserverAdmin(admin.ModelAdmin):
-    list_display = ("name", "identifier", "multiple", "enabled")
+    list_display = ("name", "identifier", "enabled")
     ordering = ("name",)
     search_fields = ("name", "identifier")
-    list_filter = ("multiple", "enabled")
+    list_filter = ("enabled",)
     form = ObservationForm
 
     def formfield_for_dbfield(self, db_field, request, **kwargs):
@@ -370,16 +370,3 @@ class SpeciesAdmin(admin.ModelAdmin):
         extra_context = extra_context or {}
         extra_context["fetch_species_url"] = reverse("admin:data_species_fetch")
         return super().changelist_view(request, extra_context=extra_context)
-
-
-@admin.register(models.Filter)
-class FilterAdmin(admin.ModelAdmin):
-    list_display = ("name", "enabled")
-    search_fields = ("name", "species__common_name")
-    list_filter = ("enabled",)
-    autocomplete_fields = ("species", "location", "update_species")
-    formfield_overrides = {
-        TextField: {
-            "widget": TextInput(attrs={"class": "vTextField"}),
-        }
-    }

{ebird_api_data-0.4.0 → ebird_api_data-0.5.1}/src/ebird/api/data/loaders.py

@@ -25,7 +25,6 @@ from .models import (
     Checklist,
     Country,
     County,
-    Filter,
     Location,
     Observation,
     Observer,
@@ -144,17 +143,16 @@ class APILoader:
         return self.call(func, self.api_key, *args, **kwargs)
 
     @staticmethod
-    def update(obj, values: dict) -> bool:
-        ignored = ["added", "edited"]  # Don't log timestamp changes
-        updated = False
+    def update(obj, values: dict) -> dict[str, tuple]:
+        changed: dict[str, tuple] = {}
         for key, value in values.items():
             current = getattr(obj, key)
             if current != value:
                 setattr(obj, key, value)
-                updated = True
-                if key not in ignored:
-                    logger.info("Attribute Changed: %s, %s -> %s", key, current, value)
-        return updated
+                changed[key] = (current, value)
+        if changed:
+            obj.save()
+        return changed
 
     def get_country(self, data: dict) -> Country:
         code: str = data["countryCode"]
@@ -164,10 +162,11 @@ class APILoader:
         }
         country, created = Country.objects.get_or_create(code=code, defaults=values)
         if created:
-            logger.info("Added country: %s, %s", code, values["name"])
-        elif self.update(country, values):
-            logger.info("Updated country: %s", code)
-            country.save()
+            logger.info("Country %s: added", code)
+        elif changed := self.update(country, values):
+            if "name" in changed:
+                old, new = changed["name"]
+                logger.info("Country %s: renamed, %s -> %s", code, old, new)
         return country
 
     def get_state(self, data: dict) -> State:
@@ -178,10 +177,11 @@ class APILoader:
         }
         state, created = State.objects.get_or_create(code=code, defaults=values)
         if created:
-            logger.info("Added state: %s, %s", code, values["name"])
-        elif self.update(state, values):
-            logger.info("Updated state: %s", code)
-            state.save()
+            logger.info("State: %s:  added", code)
+        elif changed := self.update(state, values):
+            if "name" in changed:
+                old, new = changed["name"]
+                logger.info("State %s: renamed, %s -> %s", code, old, new)
         return state
 
     def get_county(self, data) -> County:
@@ -193,10 +193,11 @@ class APILoader:
         }
         county, created = County.objects.get_or_create(code=code, defaults=values)
         if created:
-            logger.info("Added county: %s, %s", code, values["name"])
-        elif self.update(county, values):
-            logger.info("Updated county: %s", code)
-            county.save()
+            logger.info("County %s: added", code)
+        elif changed := self.update(county, values):
+            if "name" in changed:
+                old, new = changed["name"]
+                logger.info("County %s: renamed, %s -> %s", code, old, new)
         return county
 
     def add_location(self, data: dict) -> Location:
@@ -223,13 +224,17 @@ class APILoader:
         )
 
         if not created:
-            values.pop("name") # Don't overwrite the name
+            values.pop("name")  # Don't overwrite the name
 
         if created:
-            logger.info("Added location: %s, %s", identifier, location.name)
-        elif self.update(location, values):
-            logger.info("Updated location: %s", identifier)
-            location.save()
+            logger.info("Location %s: added %s", identifier, values["name"])
+        elif changed := self.update(location, values):
+            included = ["name", "hotspot", "county", "state", "country"]
+            filtered = {key: value for key, value in changed.items() if key in included}
+            for key, (old, new) in filtered.items():
+                logger.info(
+                    "Location %s: changed %s, %s -> %s", identifier, key, old, new
+                )
         return location
 
     def add_species(self, code: str) -> Species:
@@ -260,7 +265,7 @@ class APILoader:
         values["family_common_name"] = json.dumps(values["family_common_name"])
 
         species = Species.objects.create(species_code=code, **values)
-        logger.info("Added species: %s, %s", code, species.get_common_name())
+        logger.info("Species added: %s, %s", code, species.get_common_name())
 
         return species
 
@@ -282,7 +287,6 @@ class APILoader:
 
         values: dict = {
             "edited": checklist.edited,
-            "published": False,
             "checklist": checklist,
             "country": checklist.country,
             "state": checklist.state,
@@ -315,49 +319,93 @@ class APILoader:
         observation, created = Observation.objects.get_or_create(
             identifier=identifier, defaults=values
         )
-        # Don't record when an observation is added. It's too noisy.
-        if not created:
-            values.pop("published")
-        if self.update(observation, values):
-            logger.info("Updated observation: %s", identifier)
-            observation.save()
+
+        # Log observations that were added after the checklist was first loaded.
+        if created and checklist.created < checklist.edited:
+            logger.info(
+                "Checklist %s: added %s (%d)",
+                checklist.identifier,
+                species,
+                values["count"],
+            )
+
+        if changed := self.update(observation, values):
+            for field in ["audio", "photo", "video"]:
+                if field in changed:
+                    old, new = changed[field]
+                    logger.info(
+                        "Checklist %s: %s, %s %s",
+                        checklist.identifier,
+                        species,
+                        "added" if new else "deleted",
+                        field,
+                    )
+            if "count" in changed:
+                old, new = changed["count"]
+                logger.info(
+                    "Checklist %s: %s, changed count, %s -> %s",
+                    checklist.identifier,
+                    species,
+                    old,
+                    new,
+                )
+            if "species" in changed:
+                old, new = changed["species"]
+                logger.info(
+                    "Checklist %s: changed species, %s -> %s",
+                    checklist.identifier,
+                    old,
+                    new,
+                )
         return observation
 
     def get_observer_identifier(self, data) -> str:
-        identifier: str = data["subId"]
-        logger.info("Scraping checklist: %s", identifier)
+        checklist_identifier: str = data["subId"]
+        logger.info("Scraping checklist: %s", checklist_identifier)
         response = self.call(
-            requests.get, "https://ebird.org/checklist/%s" % identifier
+            requests.get, "https://ebird.org/checklist/%s" % checklist_identifier
         )
         content = response.content
         soup = BeautifulSoup(content, "lxml")
         attribute = "data-participant-userid"
         node = soup.find("span", attrs={attribute: True})
-        return node[attribute] if node else ""
+        identifier = node[attribute] if node else ""
+        if not identifier:
+            logger.error("Observer Identifier: not found")
+        return identifier
 
     def get_observer(self, data: dict) -> Observer:
+        # In theory, there is a serious problem here, as observers can have
+        # the same name. There is no way to resolve this except to prime the
+        # database with multiple Observers sharing the same name. That will
+        # force the identifier to be scraped from the checklist web page and
+        # so the correct "John Smith" will be used. In practice, apart from
+        # Anonymous eBirder, this is not a problem. Most observers want their
+        # name to be unique (so they show up in the leader boards), and so they
+        # will add middle initials, etc. so the names are generally unique.
+
         name: str = data.get("userDisplayName", "Anonymous eBirder")
-        is_multiple = Observer.objects.filter(original=name, multiple=True).exists()
-        if is_multiple:
+        number_of_observers = Observer.objects.filter(original=name).count()
+
+        if number_of_observers == 0:
+            identifier = self.get_observer_identifier(data)
+            # The observer might have changed their name.
+            observer, created = Observer.objects.update_or_create(
+                identifier=identifier, defaults={"name": name, "original": name}
+            )
+        elif number_of_observers == 1:
+            observer = Observer.objects.get(original=name)
+            identifier = observer.identifier
+            created = False
+        else:
             identifier = self.get_observer_identifier(data)
             observer, created = Observer.objects.get_or_create(
                 identifier=identifier,
-                defaults={"name": name, "original": random_word(8)},
+                defaults={"name": name, "original": name},
             )
-        else:
-            if observer := Observer.objects.filter(original=name).first():
-                created = False
-            else:
-                identifier = self.get_observer_identifier(data)
-                observer, created = Observer.objects.get_or_create(
-                    identifier=identifier,
-                    defaults={"name": name, "original": name},
-                )
 
         if created:
-            logger.info("Added observer: %s", name)
-            if Observer.objects.filter(original=name).count() > 1:
-                logger.error("Multiple observers exist with same name: %s", name)
+            logger.info("Observer added: %s (%s)", name, identifier)
 
         return observer
 
@@ -392,7 +440,6 @@ class APILoader:
             values: dict = {
                 "added": added,
                 "edited": edited,
-                "published": False,
                 "country": location.country,
                 "state": location.state,
                 "county": location.county,
@@ -413,40 +460,78 @@ class APILoader:
 
             if data["obsTimeValid"]:
                 values["time"] = started.time()
+            else:
+                values["time"] = None
 
             if "numObservers" in data:
                 values["observer_count"] = int(data["numObservers"])
+            else:
+                values["observer_count"] = None
 
             if duration := data.get("durationHrs"):
                 values["duration"] = int(duration * 60.0)
+            else:
+                values["duration"] = None
 
             if dist := data.get("effortDistanceKm"):
                 values["distance"] = round(Decimal(dist), 3)
+            else:
+                values["distance"] = None
 
             if area := data.get("effortAreaHa"):
                 values["area"] = round(Decimal(area), 3)
+            else:
+                values["area"] = None
 
             if "comments" in data:
                 values["comments"] = data["comments"]
+            else:
+                values["comments"] = ""
 
             checklist, created = Checklist.objects.get_or_create(
                 identifier=identifier, defaults=values
             )
-            if not created:
-                values.pop("published")
+
             if created:
-                logger.info("Added checklist: %s", identifier)
-            elif self.update(checklist, values):
-                logger.info("Updated checklist: %s", identifier)
-                checklist.save()
+                logger.info("Checklist %s: added", identifier)
+            elif changed := self.update(checklist, values):
+                ignored = ["added", "edited", "started", "url", "comments"]
+                filtered = {
+                    key: value for key, value in changed.items() if key not in ignored
+                }
+                if "observer" in filtered:
+                    old, new = filtered.pop("observer")
+                    logger.info(
+                        "Checklist %s: changed observer, %s (%s) -> %s (%s)",
+                        identifier,
+                        old.name,
+                        old.identifier,
+                        new.name,
+                        new.identifier
+                    )
+                for key, (old, new) in filtered.items():
+                    logger.info(
+                        "Checklist %s: changed %s, %s -> %s",
+                        identifier,
+                        key.replace("_", " "),
+                        old,
+                        new,
+                    )
+            else:
+                logger.debug("Checklist %s: unchanged", identifier)
 
             for observation_data in observations:
                 self.get_observation(observation_data, checklist)
 
-            for observation in checklist.observations.all():
-                if observation.edited != checklist.edited:
-                    logger.info("Deleted observation: %s", observation.identifier)
-                    observation.delete()
+            for obs in checklist.observations.all():
+                if obs.edited != checklist.edited:
+                    obs.delete()
+                    logger.info(
+                        "Checklist %s: deleted, %s (%s)",
+                        identifier,
+                        obs.species,
+                        obs.count,
+                    )
 
         return checklist
 
@@ -472,15 +557,19 @@ class APILoader:
         )
 
         if len(results) == API_MAX_RESULTS:
-            logger.info("API limit reached - fetching visits for subregions")
+            logger.info("Region %s: API limit reached - fetching subregions", region)
             if sub_regions := self.fetch_subregions(region):
                 for sub_region in sub_regions:
-                    visits.extend(self.fetch_visits(sub_region, date))
+                    sub_region_visits = self.fetch_visits(sub_region, date)
+                    visits.extend(sub_region_visits)
+                    logger.info(
+                        "Region %s: %d visits", sub_region, len(sub_region_visits)
+                    )
             else:
                 # No more sub-regions, issue a warning and return the results
                 visits.extend(results)
                 logger.warning(
-                    "API limit reached - No subregions available: %s, %s", region, date
+                    "Region %s: API limit reached - no subregions available", region
                 )
         else:
             visits.extend(results)
@@ -502,11 +591,11 @@ class APILoader:
 
         """
 
-        logger.info("Adding checklists: %s, %s", region, date)
+        logger.info("Adding checklists: %s, %s", region, date.strftime("%Y-%m-%d"))
 
         visits: list[dict] = self.fetch_visits(region, date)
 
-        logger.info("Visits made: %d ", len(visits))
+        logger.info("Region %s: %s visits", region, len(visits))
 
         for visit in visits:
             data = visit["loc"]
@@ -514,26 +603,8 @@ class APILoader:
 
         for visit in visits:
             identifier = visit["subId"]
-            if (
-                not update
-                and Checklist.objects.filter(identifier=identifier).exists()
-            ):
+            if not update and Checklist.objects.filter(identifier=identifier).exists():
                 continue
             self.add_checklist(identifier)
 
-        self.run_filters()
-        self.publish()
-
-        logger.info("Adding checklists completed")
-
-    @staticmethod
-    def run_filters():
-        for filter in Filter.objects.filter(enabled=True):
-            logger.info("Applying filter: %s", filter.name)
-            count = filter.apply()
-            logger.info("Observations updated: %d", count)
-
-    @staticmethod
-    def publish():
-        Checklist.objects.filter(published=False).update(published=True)
-        Observation.objects.filter(published=False).update(published=True)
+        logger.info("Added checklists: %s, %s", region, date.strftime("%Y-%m-%d"))

ebird_api_data-0.5.1/src/ebird/api/data/migrations/0003_remove_checklist_published_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.2.7 on 2025-11-05 08:59
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0002_expand_filter'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='checklist',
+            name='published',
+        ),
+        migrations.RemoveField(
+            model_name='observation',
+            name='published',
+        ),
+        migrations.DeleteModel(
+            name='Filter',
+        ),
+    ]

ebird_api_data-0.5.1/src/ebird/api/data/migrations/0004_remove_observer_multiple.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.2.7 on 2025-11-05 09:15
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0003_remove_checklist_published_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='observer',
+            name='multiple',
+        ),
+    ]

{ebird_api_data-0.4.0 → ebird_api_data-0.5.1}/src/ebird/api/data/models/__init__.py

@@ -1,7 +1,6 @@
 from .checklist import Checklist
 from .country import Country
 from .county import County
-from .filter import Filter
 from .location import Location
 from .observation import Observation
 from .observer import Observer
@@ -12,7 +11,6 @@ __all__ = (
     "Checklist",
     "Country",
     "County",
-    "Filter",
     "Location",
     "Observation",
     "Observer",

{ebird_api_data-0.4.0 → ebird_api_data-0.5.1}/src/ebird/api/data/models/checklist.py

@@ -33,11 +33,6 @@ class Checklist(models.Model):
         verbose_name=_("edited"),
     )
 
-    published = models.BooleanField(
-        help_text=_("Is the checklist published?"),
-        verbose_name=_("published"),
-    )
-
     identifier = models.CharField(
         max_length=15,
         primary_key=True,

{ebird_api_data-0.4.0 → ebird_api_data-0.5.1}/src/ebird/api/data/models/observation.py

@@ -23,11 +23,6 @@ class Observation(models.Model):
         verbose_name=_("edited"),
     )
 
-    published = models.BooleanField(
-        help_text=_("Is the observation published?"),
-        verbose_name=_("published"),
-    )
-
     identifier = models.CharField(
         max_length=15,
         primary_key=True,

{ebird_api_data-0.4.0 → ebird_api_data-0.5.1}/src/ebird/api/data/models/observer.py

@@ -24,12 +24,6 @@ class Observer(models.Model):
         help_text=_("The original name of the observer from eBird."),
     )
 
-    multiple = models.BooleanField(
-        default=False,
-        verbose_name=_("Multiple accounts"),
-        help_text=_("Is the name used by more than one eBird account."),
-    )
-
     enabled = models.BooleanField(
         default=True,
         verbose_name=_("Enabled"),

ebird_api_data-0.4.0/src/ebird/api/data/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.4.0"