hdx-python-scraper 2.3.1__tar.gz → 2.3.2__tar.gz

{hdx_python_scraper-2.3.1 → hdx_python_scraper-2.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hdx-python-scraper
-Version: 2.3.1
+Version: 2.3.2
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>

{hdx_python_scraper-2.3.1 → hdx_python_scraper-2.3.2}/requirements.txt

@@ -6,7 +6,7 @@
 #
 annotated-types==0.6.0
     # via pydantic
-attrs==23.1.0
+attrs==23.2.0
     # via
     #   frictionless
     #   jsonlines
@@ -29,7 +29,7 @@ click==8.1.7
     # via typer
 colorama==0.4.6
     # via typer
-coverage[toml]==7.3.2
+coverage[toml]==7.4.0
     # via pytest-cov
 cryptography==41.0.7
     # via pyopenssl
@@ -59,7 +59,7 @@ google-auth==2.25.2
     #   gspread
 google-auth-oauthlib==1.2.0
     # via gspread
-gspread==5.12.2
+gspread==5.12.4
     # via hdx-python-scraper (pyproject.toml)
 hdx-python-api==6.2.0
     # via hdx-python-scraper (pyproject.toml)
@@ -119,7 +119,7 @@ nodeenv==1.8.0
     # via pre-commit
 num2words==0.5.13
     # via quantulum3
-numpy==1.26.2
+numpy==1.26.3
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib
@@ -153,11 +153,11 @@ pyasn1-modules==0.3.0
     # via google-auth
 pycparser==2.21
     # via cffi
-pydantic==2.5.2
+pydantic==2.5.3
     # via
     #   frictionless
     #   inflect
-pydantic-core==2.14.5
+pydantic-core==2.14.6
     # via pydantic
 pygments==2.17.2
     # via rich
@@ -169,7 +169,7 @@ pyphonetics==0.5.3
     # via hdx-python-country
 pyrsistent==0.20.0
     # via jsonschema
-pytest==7.4.3
+pytest==7.4.4
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   pytest-cov
@@ -198,7 +198,7 @@ quantulum3==0.9.0
     # via hdx-python-api
 ratelimit==2.2.1
     # via hdx-python-utilities
-regex==2023.10.3
+regex==2023.12.25
     # via hdx-python-scraper (pyproject.toml)
 requests==2.31.0
     # via
@@ -238,7 +238,7 @@ sphinxcontrib-napoleon==0.7
     # via defopt
 stringcase==1.2.0
     # via frictionless
-structlog==23.2.0
+structlog==23.3.0
     # via libhxl
 tableschema-to-template==0.0.13
     # via hdx-python-utilities
@@ -255,7 +255,7 @@ typing-extensions==4.9.0
     #   pydantic
     #   pydantic-core
     #   typer
-tzdata==2023.3
+tzdata==2023.4
     # via pandas
 unidecode==1.3.7
     # via

{hdx_python_scraper-2.3.1 → hdx_python_scraper-2.3.2}/src/hdx/scraper/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.3.1'
-__version_tuple__ = version_tuple = (2, 3, 1)
+__version__ = version = '2.3.2'
+__version_tuple__ = version_tuple = (2, 3, 2)

{hdx_python_scraper-2.3.1 → hdx_python_scraper-2.3.2}/src/hdx/scraper/runner.py

@@ -1209,32 +1209,52 @@ class Runner:
         """
         if not names:
             names = self.scrapers.keys()
-        results = {}
+        hapi_results = {}
 
         def add_results(scraper_level, scrap, levels_used):
-            nonlocal results
+            nonlocal hapi_results
 
             if scraper_level in levels_used:
                 return
             headers = scrap.headers.get(scraper_level)
             if headers is None:
                 return
+            headings = headers[0]
+            hxltags = headers[1]
             values = scrap.get_values(scraper_level)
             hapi_dataset_metadata = scrap.get_hapi_dataset_metadata()
+            if not hapi_dataset_metadata:
+                return
             hapi_resource_metadata = scrap.get_hapi_resource_metadata()
+            if not hapi_resource_metadata:
+                return
             dataset_id = hapi_dataset_metadata["hdx_id"]
-            hapi_metadata = results.get(
+            hapi_metadata = hapi_results.get(
                 dataset_id, copy(hapi_dataset_metadata)
             )
-            level_results = hapi_metadata.get("results", {})
-            level_results[scraper_level] = {
-                "headers": headers,
-                "values": values,
-                "hapi_resource_metadata": hapi_resource_metadata,
-            }
-            hapi_metadata["results"] = level_results
+            results = hapi_metadata.get("results", {})
+            level_results = results.get(scraper_level)
+            if level_results is None:
+                level_results = {
+                    "headers": ([], []),
+                    "values": [],
+                    "hapi_resource_metadata": hapi_resource_metadata,
+                }
+                results[scraper_level] = level_results
+            lev_headings = level_results["headers"][0]
+            lev_hxltags = level_results["headers"][1]
+            lev_values = level_results["values"]
+            for i, hxltag in enumerate(hxltags):
+                if hxltag in lev_hxltags:
+                    index = lev_hxltags.index(hxltag)
+                    lev_values[index].update(values[i])
+                else:
+                    lev_headings.append(headings[i])
+                    lev_hxltags.append(hxltag)
+                    lev_values.append(values[i])
+            hapi_metadata["results"] = results
             levels_used.add(scraper_level)
-            results[dataset_id] = hapi_metadata
+            hapi_results[dataset_id] = hapi_metadata
 
         for name in names:
             if self.scrapers_to_run and not any(
@@ -1247,4 +1267,4 @@ class Runner:
             lvls_used = set()
             for scrap_level in scraper.headers:
                 add_results(scrap_level, scraper, lvls_used)
-        return results
+        return hapi_results

{hdx_python_scraper-2.3.1 → hdx_python_scraper-2.3.2}/tests/hdx/scraper/education_closures.py

@@ -38,7 +38,7 @@ class EducationClosures(BaseScraper):
 
     def run(self) -> None:
         closures_headers, closures_iterator = self.get_reader().read(
-            self.datasetinfo, file_prefix=self.name
+            self.datasetinfo, file_prefix="education_closures"
         )
         closures = self.get_values("national")[0]
         closed_countries = self.get_values("regional")[0]

{hdx_python_scraper-2.3.1 → hdx_python_scraper-2.3.2}/tests/hdx/scraper/test_scrapers_custom.py

@@ -1,4 +1,4 @@
-from copy import deepcopy
+from copy import copy, deepcopy
 from datetime import datetime, timezone
 
 from .affected_targeted_reached import AffectedTargetedReached
@@ -12,6 +12,54 @@ from hdx.utilities.dateparse import parse_date
 
 
 class TestScrapersCustom:
+    hapi_values = {
+        "hdx_id": "6a41be98-75b9-4365-9ea3-e33d0dd2668b",
+        "hdx_stub": "global-school-closures-covid19",
+        "hdx_provider_stub": "unesco",
+        "hdx_provider_name": "UNESCO",
+        "reference_period": {
+            "enddate": datetime(2022, 4, 30, 23, 59, 59, tzinfo=timezone.utc),
+            "enddate_str": "2022-04-30T23:59:59+00:00",
+            "ongoing": False,
+            "startdate": datetime(2020, 2, 16, 0, 0, tzinfo=timezone.utc),
+            "startdate_str": "2020-02-16T00:00:00+00:00",
+        },
+        "results": {
+            "national": {
+                "hapi_resource_metadata": {
+                    "download_url": "https://data.humdata.org/dataset/6a41be98-75b9-4365-9ea3-e33d0dd2668b/resource/3b5baa74-c928-4cbc-adba-bf543c5d3050/download/covid_impact_education.csv",
+                    "name": "School " "Closures",
+                    "format": "csv",
+                    "hdx_id": "3b5baa74-c928-4cbc-adba-bf543c5d3050",
+                    "is_hxl": False,
+                    "update_date": datetime(
+                        2022, 4, 4, 9, 56, 5, tzinfo=timezone.utc
+                    ),
+                },
+                "headers": (["School Closure"], ["#impact+type"]),
+                "values": [{"AFG": "Closed due to COVID-19"}],
+            },
+            "regional": {
+                "hapi_resource_metadata": {
+                    "download_url": "https://data.humdata.org/dataset/6a41be98-75b9-4365-9ea3-e33d0dd2668b/resource/3b5baa74-c928-4cbc-adba-bf543c5d3050/download/covid_impact_education.csv",
+                    "name": "School " "Closures",
+                    "format": "csv",
+                    "hdx_id": "3b5baa74-c928-4cbc-adba-bf543c5d3050",
+                    "is_hxl": False,
+                    "update_date": datetime(
+                        2022, 4, 4, 9, 56, 5, tzinfo=timezone.utc
+                    ),
+                },
+                "headers": (
+                    ["No. closed countries"],
+                    ["#status+country+closed"],
+                ),
+                "values": [{"ROAP": 1}],
+            },
+        },
+        "title": "Global School Closures COVID-19",
+    }
+
     def test_get_custom(self, configuration, fallbacks_json):
         BaseScraper.population_lookup = {}
         source_date = "Apr 30, 2022"
@@ -86,55 +134,7 @@ class TestScrapersCustom:
             }
         }
         hapi_results = runner.get_hapi_results()
-        assert next(iter(hapi_results.values())) == {
-            "hdx_id": "6a41be98-75b9-4365-9ea3-e33d0dd2668b",
-            "hdx_stub": "global-school-closures-covid19",
-            "hdx_provider_stub": "unesco",
-            "hdx_provider_name": "UNESCO",
-            "reference_period": {
-                "enddate": datetime(
-                    2022, 4, 30, 23, 59, 59, tzinfo=timezone.utc
-                ),
-                "enddate_str": "2022-04-30T23:59:59+00:00",
-                "ongoing": False,
-                "startdate": datetime(2020, 2, 16, 0, 0, tzinfo=timezone.utc),
-                "startdate_str": "2020-02-16T00:00:00+00:00",
-            },
-            "results": {
-                "national": {
-                    "hapi_resource_metadata": {
-                        "download_url": "https://data.humdata.org/dataset/6a41be98-75b9-4365-9ea3-e33d0dd2668b/resource/3b5baa74-c928-4cbc-adba-bf543c5d3050/download/covid_impact_education.csv",
-                        "name": "School " "Closures",
-                        "format": "csv",
-                        "hdx_id": "3b5baa74-c928-4cbc-adba-bf543c5d3050",
-                        "is_hxl": False,
-                        "update_date": datetime(
-                            2022, 4, 4, 9, 56, 5, tzinfo=timezone.utc
-                        ),
-                    },
-                    "headers": (("School Closure",), ("#impact+type",)),
-                    "values": ({"AFG": "Closed due to COVID-19"},),
-                },
-                "regional": {
-                    "hapi_resource_metadata": {
-                        "download_url": "https://data.humdata.org/dataset/6a41be98-75b9-4365-9ea3-e33d0dd2668b/resource/3b5baa74-c928-4cbc-adba-bf543c5d3050/download/covid_impact_education.csv",
-                        "name": "School " "Closures",
-                        "format": "csv",
-                        "hdx_id": "3b5baa74-c928-4cbc-adba-bf543c5d3050",
-                        "is_hxl": False,
-                        "update_date": datetime(
-                            2022, 4, 4, 9, 56, 5, tzinfo=timezone.utc
-                        ),
-                    },
-                    "headers": (
-                        ("No. closed countries",),
-                        ("#status+country+closed",),
-                    ),
-                    "values": ({"ROAP": 1},),
-                },
-            },
-            "title": "Global School Closures COVID-19",
-        }
+        assert next(iter(hapi_results.values())) == self.hapi_values
 
         headers = (["No. closed countries"], ["#status+country+closed"])
         values = [{"ROAP": 1}]
@@ -200,6 +200,50 @@ class TestScrapersCustom:
             "https://data.humdata.org/dataset/global-school-closures-covid19",
             "https://data.humdata.org/organization/world-bank-group",
         ]
+        hapi_results = runner.get_hapi_results()
+        assert next(iter(hapi_results.values())) == self.hapi_values
+        edu_dsinfo = runner.get_scraper(name).datasetinfo
+        pop_dsinfo = runner.get_scraper("population").datasetinfo
+        # pretend population uses same dataset as education for testing
+        pop_dsinfo["hapi_dataset_metadata"] = edu_dsinfo[
+            "hapi_dataset_metadata"
+        ]
+        hapi_results = runner.get_hapi_results()
+        assert next(iter(hapi_results.values())) == self.hapi_values
+        # pretend population uses same resource as education for testing
+        pop_dsinfo["hapi_resource_metadata"] = edu_dsinfo[
+            "hapi_resource_metadata"
+        ]
+        hapi_results = runner.get_hapi_results()
+        hapi_values = copy(self.hapi_values)
+        hapi_values["results"]["national"]["headers"] = (
+            ["School Closure", "Population"],
+            ["#impact+type", "#population"],
+        )
+        hapi_values["results"]["national"]["values"] = [
+            {"AFG": "Closed due to COVID-19"},
+            {"AFG": 38041754},
+        ]
+        assert next(iter(hapi_results.values())) == hapi_values
+
+        education_closures2 = EducationClosures(
+            datasetinfo, today, countries, region
+        )
+        # give it a unique name
+        education_closures2.name = f"{education_closures2.name}2"
+        runner.add_custom(education_closures2)
+        runner.run_one(education_closures2.name)
+        # give it a unique result
+        del education_closures2.values["national"][0]["AFG"]
+        education_closures2.values["national"][0][
+            "SDN"
+        ] = "Closed due to COVID-19"
+        hapi_results = runner.get_hapi_results()
+        hapi_values["results"]["national"]["values"] = [
+            {"AFG": "Closed due to COVID-19", "SDN": "Closed due to COVID-19"},
+            {"AFG": 38041754},
+        ]
+        assert next(iter(hapi_results.values())) == hapi_values
 
         runner = Runner(("AFG",), today)
         datasetinfo = deepcopy(datasetinfo)