hdx-python-scraper 2.5.0__tar.gz → 2.5.1__tar.gz

{hdx_python_scraper-2.5.0 → hdx_python_scraper-2.5.1}/.config/pre-commit-config.yaml

@@ -2,14 +2,14 @@ default_language_version:
     python: python3.12
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
         exclude: test_scraper_.*\.json
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.4
+    rev: v0.8.6
     hooks:
       # Run the linter.
       - id: ruff
@@ -18,7 +18,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.4.7
+    rev: 0.5.15
     hooks:
       # Run the pip compile
       - id: pip-compile

{hdx_python_scraper-2.5.0 → hdx_python_scraper-2.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: hdx-python-scraper
-Version: 2.5.0
+Version: 2.5.1
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -26,9 +26,9 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
-Requires-Dist: hdx-python-api>=6.3.4
-Requires-Dist: hdx-python-country>=3.7.8
-Requires-Dist: hdx-python-utilities>=3.7.3
+Requires-Dist: hdx-python-api>=6.3.6
+Requires-Dist: hdx-python-country>=3.8.6
+Requires-Dist: hdx-python-utilities>=3.8.0
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'

{hdx_python_scraper-2.5.0 → hdx_python_scraper-2.5.1}/pyproject.toml

@@ -34,9 +34,9 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-api>=6.3.4",
-    "hdx-python-country>=3.7.8",
-    "hdx-python-utilities>=3.7.3",
+    "hdx-python-api>=6.3.6",
+    "hdx-python-country>=3.8.6",
+    "hdx-python-utilities>=3.8.0",
     "gspread",
     "regex",
 ]
@@ -67,6 +67,9 @@ packages = ["src/hdx"]
 [tool.hatch.build.hooks.vcs]
 version-file = "src/hdx/scraper/framework/_version.py"
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 # Versioning
 
 [tool.hatch.version]
@@ -91,6 +94,9 @@ run = """
        --cov-report=lcov --cov-report=term-missing
        """
 
+[tool.hatch.envs.hatch-static-analysis]
+dependencies = ["ruff==0.8.6"]
+
 [tool.hatch.envs.hatch-static-analysis.scripts]
 format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]
 format-fix = ["ruff format --config .config/ruff.toml {args:.}",]

{hdx_python_scraper-2.5.0 → hdx_python_scraper-2.5.1}/requirements.txt

@@ -2,7 +2,7 @@
 #    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
 annotated-types==0.7.0
     # via pydantic
-attrs==24.2.0
+attrs==24.3.0
     # via
     #   frictionless
     #   jsonlines
@@ -10,25 +10,25 @@ attrs==24.2.0
     #   referencing
 cachetools==5.5.0
     # via google-auth
-certifi==2024.8.30
+certifi==2024.12.14
     # via requests
 cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.3.2
+charset-normalizer==3.4.1
     # via requests
 ckanapi==4.8
     # via hdx-python-api
-click==8.1.7
+click==8.1.8
     # via typer
-coverage==7.6.1
+coverage==7.6.10
     # via pytest-cov
 defopt==6.4.0
     # via hdx-python-api
-distlib==0.3.8
+distlib==0.3.9
     # via virtualenv
-dnspython==2.6.1
+dnspython==2.7.0
     # via email-validator
 docopt==0.6.2
     # via
@@ -38,34 +38,34 @@ docutils==0.21.2
     # via defopt
 email-validator==2.2.0
     # via hdx-python-api
-et-xmlfile==1.1.0
+et-xmlfile==2.0.0
     # via openpyxl
-filelock==3.16.0
+filelock==3.16.1
     # via virtualenv
-frictionless==5.17.1
+frictionless==5.18.0
     # via hdx-python-utilities
-google-auth==2.34.0
+google-auth==2.37.0
     # via
     #   google-auth-oauthlib
     #   gspread
 google-auth-oauthlib==1.2.1
     # via gspread
-gspread==6.1.2
+gspread==6.1.4
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-api==6.3.4
+hdx-python-api==6.3.6
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-country==3.7.8
+hdx-python-country==3.8.6
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
-hdx-python-utilities==3.7.3
+hdx-python-utilities==3.8.0
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
     #   hdx-python-country
-humanize==4.10.0
+humanize==4.11.0
     # via frictionless
-identify==2.6.1
+identify==2.6.5
     # via pre-commit
 idna==3.10
     # via
@@ -73,37 +73,37 @@ idna==3.10
     #   requests
 ijson==3.3.0
     # via hdx-python-utilities
-inflect==7.4.0
+inflect==7.5.0
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
-isodate==0.6.1
+isodate==0.7.2
     # via frictionless
-jinja2==3.1.4
+jinja2==3.1.5
     # via frictionless
 jsonlines==4.0.0
     # via hdx-python-utilities
-jsonpath-ng==1.6.1
+jsonpath-ng==1.7.0
     # via libhxl
 jsonschema==4.23.0
     # via
     #   frictionless
     #   tableschema-to-template
-jsonschema-specifications==2023.12.1
+jsonschema-specifications==2024.10.1
     # via jsonschema
-libhxl==5.2.1
+libhxl==5.2.2
     # via
     #   hdx-python-api
     #   hdx-python-country
-loguru==0.7.2
+loguru==0.7.3
     # via hdx-python-utilities
-makefun==1.15.4
+makefun==1.15.6
     # via hdx-python-api
 markdown-it-py==3.0.0
     # via rich
 marko==2.1.2
     # via frictionless
-markupsafe==2.1.5
+markupsafe==3.0.2
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
@@ -111,21 +111,21 @@ more-itertools==10.5.0
     # via inflect
 nodeenv==1.9.1
     # via pre-commit
-num2words==0.5.13
+num2words==0.5.14
     # via quantulum3
-numpy==2.1.1
+numpy==2.2.1
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib
 openpyxl==3.1.5
     # via hdx-python-utilities
-packaging==24.1
+packaging==24.2
     # via pytest
-pandas==2.2.2
+pandas==2.2.3
     # via hdx-python-scraper (pyproject.toml)
 petl==1.7.15
     # via frictionless
-platformdirs==4.3.3
+platformdirs==4.3.6
     # via virtualenv
 pluggy==1.5.0
     # via pytest
@@ -135,7 +135,7 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==3.8.0
+pre-commit==4.0.1
     # via hdx-python-scraper (pyproject.toml)
 pyasn1==0.6.1
     # via
@@ -143,19 +143,19 @@ pyasn1==0.6.1
     #   rsa
 pyasn1-modules==0.4.1
     # via google-auth
-pydantic==2.9.1
+pydantic==2.10.5
     # via frictionless
-pydantic-core==2.23.3
+pydantic-core==2.27.2
     # via pydantic
-pygments==2.18.0
+pygments==2.19.1
     # via rich
 pyphonetics==0.5.3
-    # via hdx-python-country
-pytest==8.3.3
+    # via hdx-python-utilities
+pytest==8.3.4
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   pytest-cov
-pytest-cov==5.0.0
+pytest-cov==6.0.0
     # via hdx-python-scraper (pyproject.toml)
 python-dateutil==2.9.0.post0
     # via
@@ -184,7 +184,7 @@ referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
-regex==2024.9.11
+regex==2024.11.6
     # via hdx-python-scraper (pyproject.toml)
 requests==2.32.3
     # via
@@ -200,30 +200,29 @@ requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
-rich==13.8.1
+rich==13.9.4
     # via typer
-rpds-py==0.20.0
+rpds-py==0.22.3
     # via
     #   jsonschema
     #   referencing
 rsa==4.9
     # via google-auth
-ruamel-yaml==0.18.6
+ruamel-yaml==0.18.10
     # via hdx-python-utilities
-ruamel-yaml-clib==0.2.8
+ruamel-yaml-clib==0.2.12
     # via ruamel-yaml
-setuptools==75.0.0
+setuptools==75.8.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
-simpleeval==0.9.13
+simpleeval==1.0.3
     # via frictionless
 simplejson==3.19.3
     # via ckanapi
-six==1.16.0
+six==1.17.0
     # via
     #   ckanapi
-    #   isodate
     #   pockets
     #   python-dateutil
     #   sphinxcontrib-napoleon
@@ -237,11 +236,13 @@ tableschema-to-template==0.0.13
     # via hdx-python-utilities
 tabulate==0.9.0
     # via frictionless
+tenacity==9.0.0
+    # via hdx-python-country
 text-unidecode==1.3
     # via python-slugify
-typeguard==4.3.0
+typeguard==4.4.1
     # via inflect
-typer==0.12.5
+typer==0.15.1
     # via frictionless
 typing-extensions==4.12.2
     # via
@@ -250,27 +251,27 @@ typing-extensions==4.12.2
     #   pydantic-core
     #   typeguard
     #   typer
-tzdata==2024.1
+tzdata==2024.2
     # via pandas
 unidecode==1.3.8
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.2.3
+urllib3==2.3.0
     # via
     #   libhxl
     #   requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.26.4
+virtualenv==20.28.1
     # via pre-commit
-wheel==0.44.0
+wheel==0.45.1
     # via libhxl
 xlrd==2.0.1
     # via hdx-python-utilities
 xlrd3==1.1.0
     # via libhxl
-xlsx2csv==0.8.3
+xlsx2csv==0.8.4
     # via hdx-python-utilities
 xlsxwriter==3.2.0
     # via tableschema-to-template

{hdx_python_scraper-2.5.0 → hdx_python_scraper-2.5.1}/src/hdx/scraper/framework/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.5.0'
-__version_tuple__ = version_tuple = (2, 5, 0)
+__version__ = version = '2.5.1'
+__version_tuple__ = version_tuple = (2, 5, 1)

{hdx_python_scraper-2.5.0 → hdx_python_scraper-2.5.1}/src/hdx/scraper/framework/base_scraper.py

@@ -36,7 +36,7 @@ class BaseScraper(ABC):
             self.reader = datasetinfo.get("reader", name)
         self.setup(headers, source_configuration)
         self.datasetinfo = deepcopy(datasetinfo)
-        self.errors_on_exit = None
+        self.error_handler = None
         self.can_fallback = True
 
     def setup(

{hdx_python_scraper-2.5.0 → hdx_python_scraper-2.5.1}/src/hdx/scraper/framework/runner.py

@@ -16,7 +16,7 @@ from .utilities.reader import Read
 from .utilities.sources import Sources
 from hdx.location.adminlevel import AdminLevel
 from hdx.utilities.dateparse import now_utc
-from hdx.utilities.errors_onexit import ErrorsOnExit
+from hdx.utilities.error_handler import ErrorHandler
 from hdx.utilities.typehint import ListTuple
 
 logger = logging.getLogger(__name__)
@@ -28,7 +28,7 @@ class Runner:
     Args:
         countryiso3s (ListTuple[str]): List of ISO3 country codes to process
         today (datetime): Value to use for today. Defaults to now_utc().
-        errors_on_exit (ErrorsOnExit): ErrorsOnExit object that logs errors on exit
+        error_handler (ErrorHandler): ErrorHandler object that logs errors on exit
         scrapers_to_run (Optional[ListTuple[str]]): Scrapers to run. Defaults to None (all scrapers).
     """
 
@@ -36,12 +36,12 @@ class Runner:
         self,
         countryiso3s: ListTuple[str],
         today: datetime = now_utc(),
-        errors_on_exit: Optional[ErrorsOnExit] = None,
+        error_handler: Optional[ErrorHandler] = None,
         scrapers_to_run: Optional[ListTuple[str]] = None,
     ):
         self.countryiso3s = countryiso3s
         self.today = today
-        self.errors_on_exit = errors_on_exit
+        self.error_handler = error_handler
         if isinstance(scrapers_to_run, tuple):
             scrapers_to_run = list(scrapers_to_run)
         self.scrapers_to_run: Optional[List[str]] = scrapers_to_run
@@ -73,7 +73,7 @@ class Runner:
             and scraper_name not in self.scrapers_to_run
         ):
             self.scrapers_to_run.append(scraper_name)
-        scraper.errors_on_exit = self.errors_on_exit
+        scraper.error_handler = self.error_handler
         return scraper_name
 
     def add_customs(
@@ -142,7 +142,7 @@ class Runner:
             level_name,
             source_configuration,
             self.today,
-            self.errors_on_exit,
+            self.error_handler,
         )
         if scraper_name not in self.scraper_names:
             self.scraper_names.append(scraper_name)
@@ -612,8 +612,8 @@ class Runner:
                 if not Fallbacks.exist() or scraper.can_fallback is False:
                     raise
                 logger.exception(f"Using fallbacks for {scraper.name}!")
-                if self.errors_on_exit:
-                    self.errors_on_exit.add(
+                if self.error_handler:
+                    self.error_handler.add(
                         f"Using fallbacks for {scraper.name}! Error: {format_exc()}"
                     )
                 for level in scraper.headers.keys():

{hdx_python_scraper-2.5.0 → hdx_python_scraper-2.5.1}/src/hdx/scraper/framework/scrapers/configurable_scraper.py

@@ -17,7 +17,7 @@ from hdx.utilities.dateparse import (
 )
 from hdx.utilities.dictandlist import dict_of_lists_add
 from hdx.utilities.downloader import DownloadError
-from hdx.utilities.errors_onexit import ErrorsOnExit
+from hdx.utilities.error_handler import ErrorHandler
 from hdx.utilities.text import (  # noqa: F401
     get_fraction_str,
     get_numeric_if_possible,
@@ -42,7 +42,7 @@ class ConfigurableScraper(BaseScraper):
         level_name (Optional[str]): Customised level_name name. Defaults to None (level).
         source_configuration (Dict): Configuration for sources. Defaults to empty dict (use defaults).
         today (datetime): Value to use for today. Defaults to now_utc().
-        errors_on_exit (Optional[ErrorsOnExit]): ErrorsOnExit object that logs errors on exit
+        error_handler (Optional[ErrorHandler]): ErrorHandler object that logs errors on exit
         **kwargs: Variables to use when evaluating template arguments in urls
     """
 
@@ -67,7 +67,7 @@ class ConfigurableScraper(BaseScraper):
         level_name: Optional[str] = None,
         source_configuration: Dict = {},
         today: datetime = now_utc(),
-        errors_on_exit: Optional[ErrorsOnExit] = None,
+        error_handler: Optional[ErrorHandler] = None,
         **kwargs: Any,
     ):
         self.name = name
@@ -83,10 +83,10 @@ class ConfigurableScraper(BaseScraper):
         else:
             self.level_name: str = level_name
         self.countryiso3s = countryiso3s
-        self.adminlevel = adminlevel
+        self.adminlevel: Optional[AdminLevel] = adminlevel
         self.today = today
         self.subsets = self.get_subsets_from_datasetinfo(datasetinfo)
-        self.errors_on_exit: Optional[ErrorsOnExit] = errors_on_exit
+        self.error_handler: Optional[ErrorHandler] = error_handler
         self.variables = kwargs
         self.rowparser = None
         self.datasetinfo = copy.deepcopy(datasetinfo)

{hdx_python_scraper-2.5.0 → hdx_python_scraper-2.5.1}/tests/hdx/scraper/framework/scrapers/test_national.py

@@ -6,7 +6,7 @@ from hdx.scraper.framework.runner import Runner
 from hdx.scraper.framework.utilities.reader import Read
 from hdx.scraper.framework.utilities.writer import Writer
 from hdx.utilities.dateparse import parse_date
-from hdx.utilities.errors_onexit import ErrorsOnExit
+from hdx.utilities.error_handler import ErrorHandler
 
 
 class TestNational:
@@ -508,11 +508,11 @@ class TestNational:
         run_check_scraper(name, runner, level, headers, values, sources)
 
         today = parse_date("2021-05-03")
-        errors_on_exit = ErrorsOnExit()
+        error_handler = ErrorHandler()
         runner = Runner(
             ("AFG", "PHL"),
             today,
-            errors_on_exit=errors_on_exit,
+            error_handler=error_handler,
         )
         runner.add_configurables(scraper_configuration, level)
         name = "ourworldindata"
@@ -567,7 +567,7 @@ class TestNational:
             sources,
             fallbacks_used=True,
         )
-        error = errors_on_exit.errors[0]
+        error = error_handler.shared_errors["error"][""].pop()
         assert "Using fallbacks for broken_owd_url!" in error
         assert (
             "No such file or directory: 'tests/fixtures/input/broken_owd_url_notexist.csv'"
@@ -577,7 +577,7 @@ class TestNational:
     def test_get_national_afg_mmr_phl(self, configuration):
         BaseScraper.population_lookup = {}
         today = parse_date("2021-05-03")
-        errors_on_exit = ErrorsOnExit()
+        error_handler = ErrorHandler()
         level = "national"
         countries = ("AFG", "MMR", "PHL")
         scraper_configuration = configuration[f"scraper_{level}"]
@@ -585,7 +585,7 @@ class TestNational:
         runner = Runner(
             countries,
             today,
-            errors_on_exit=errors_on_exit,
+            error_handler=error_handler,
         )
         scrapers = runner.add_configurables(scraper_configuration, level)
         name = "idps"
@@ -631,11 +631,11 @@ class TestNational:
                 "https://data.unhcr.org/population/?widget_id=264111&geo_id=693&population_group=5407,4999",
             ],
         )
-        assert errors_on_exit.errors == []
+        assert error_handler.shared_errors["error"] == {}
         runner = Runner(
             ("AFG", "MMR", "PHL"),
             today,
-            errors_on_exit=errors_on_exit,
+            error_handler=error_handler,
         )
         runner.add_configurables(scraper_configuration, level)
         runner.add_instance_variables(
@@ -654,9 +654,9 @@ class TestNational:
                 "https://data.humdata.org/dataset/idmc-internally-displaced-persons-idps"
             ],
         )
-        assert errors_on_exit.errors == [
+        assert error_handler.shared_errors["error"][""] == {
             "Not using UNHCR Myanmar IDPs override! Error: [Errno 2] No such file or directory: 'tests/fixtures/input/idps_override_not-exist.json'",
-        ]
+        }
         runner.run()
         jsonout = JsonFile(configuration["json"], [level])
         outputs = {"json": jsonout}

{hdx_python_scraper-2.5.0 → hdx_python_scraper-2.5.1}/tests/hdx/scraper/framework/scrapers/unhcr_myanmar_idps.py

@@ -24,5 +24,5 @@ def idps_post_run(self) -> None:
     except Exception as ex:
         msg = "Not using UNHCR Myanmar IDPs override!"
         logger.exception(msg)
-        if self.errors_on_exit:
-            self.errors_on_exit.add(f"{msg} Error: {ex}")
+        if self.error_handler:
+            self.error_handler.add(f"{msg} Error: {ex}")