hdx-python-scraper 2.6.2__tar.gz → 2.6.4__tar.gz

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.4}/.github/workflows/publish.yaml

@@ -8,6 +8,13 @@ jobs:
   publish:
     runs-on: ubuntu-latest
 
+    environment:
+      name: pypi
+      url: https://pypi.org/p/hdx-python-scraper
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
     steps:
     - uses: actions/checkout@v4
     - name: Get history and tags for versioning to work
@@ -26,9 +33,5 @@ jobs:
     - name: Build with hatch
       run: |
         hatch build
-    - name: Publish with hatch
-      env:
-        HATCH_INDEX_USER: ${{secrets.HATCH_INDEX_USER}}
-        HATCH_INDEX_AUTH: ${{secrets.HATCH_INDEX_AUTH}}
-      run: |
-        hatch publish
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.4}/.github/workflows/run-python-tests.yaml

@@ -17,6 +17,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    permissions:
+      checks: write
+      pull-requests: write
+
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python
@@ -30,7 +34,7 @@ jobs:
       uses: pypa/hatch@install
     - name: Test with hatch/pytest
       env:
-        GSHEET_AUTH: ${{ secrets.GSHEET_AUTH }}
+        GSHEET_AUTH: ${{ secrets.HDX_PIPELINE_GSHEET_AUTH }}
       run: |
         hatch test
     - name: Check styling

hdx_python_scraper-2.6.2/.config/pre-commit-config.yaml → hdx_python_scraper-2.6.4/.pre-commit-config.yaml

@@ -9,16 +9,15 @@ repos:
         exclude: test_scraper_.*\.json
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.6
+    rev: v0.9.10
     hooks:
       # Run the linter.
       - id: ruff
-        args: [--config, .config/ruff.toml, --fix]
+        args: [ --fix ]
       # Run the formatter.
       - id: ruff-format
-        args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.5.29
+    rev: 0.6.5
     hooks:
       # Run the pip compile
       - id: pip-compile

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hdx-python-scraper
-Version: 2.6.2
+Version: 2.6.4
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -26,15 +26,18 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
-Requires-Dist: hdx-python-api>=6.3.8
-Requires-Dist: hdx-python-country>=3.8.8
-Requires-Dist: hdx-python-utilities>=3.8.3
+Requires-Dist: hdx-python-api>=6.3.9
+Requires-Dist: hdx-python-country>=3.9.2
+Requires-Dist: hdx-python-utilities>=3.8.6
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkapi; extra == 'docs'
 Provides-Extra: pandas
-Requires-Dist: pandas>=2.2.2; extra == 'pandas'
+Requires-Dist: pandas>=2.2.3; extra == 'pandas'
 Provides-Extra: test
+Requires-Dist: pandas>=2.2.3; extra == 'test'
 Requires-Dist: pytest; extra == 'test'
 Requires-Dist: pytest-cov; extra == 'test'
 Description-Content-Type: text/markdown

hdx_python_scraper-2.6.4/documentation/.readthedocs.yaml

@@ -0,0 +1,14 @@
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.12"
+  jobs:
+    pre_build:
+      - pip install .[docs]
+
+mkdocs:
+  configuration: documentation/mkdocs.yaml

hdx_python_scraper-2.6.4/documentation/mkdocs.yaml

@@ -0,0 +1,19 @@
+site_name: HDX Python Scraper
+repo_url: https://github.com/OCHA-DAP/hdx-python-scraper/
+repo_name: OCHA-DAP/hdx-python-scraper
+docs_dir: .
+site_dir: ../site
+theme:
+  name: material
+  highlightjs: true
+plugins:
+  - search
+  - mkapi
+nav:
+  - Home: index.md
+  - API Documentation:
+    - Runner: $src/hdx.scraper.framework.runner.Runner.*
+    - Scraper Base Class: $src/hdx.scraper.framework.base_scraper.BaseScraper.*
+    - Scrapers: $src/hdx.scraper.framework.scrapers.*
+    - Outputs: $src/hdx.scraper.framework.outputs.*
+    - Utilities: $src/hdx.scraper.framework.utilities.*

hdx_python_scraper-2.6.4/hatch.toml

@@ -0,0 +1,37 @@
+# Build
+
+[build.targets.wheel]
+packages = ["src/hdx"]
+
+[build.hooks.vcs]
+version-file = "src/hdx/scraper/framework/_version.py"
+
+[metadata]
+allow-direct-references = true
+
+# Versioning
+
+[version]
+source = "vcs"
+
+[version.raw-options]
+local_scheme = "no-local-version"
+version_scheme = "python-simplified-semver"
+
+# Tests
+
+[envs.hatch-test]
+features = ["test"]
+
+[[envs.hatch-test.matrix]]
+python = ["3.12"]
+
+[envs.hatch-test.scripts]
+run = """
+       pytest --rootdir=. --junitxml=test-results.xml --cov --no-cov-on-fail \
+       --cov-report=lcov --cov-report=term-missing
+       """
+
+[envs.hatch-static-analysis]
+config-path = "none"
+dependencies = ["ruff==0.9.10"]

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.4}/pyproject.toml

@@ -34,9 +34,9 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-api>=6.3.8",
-    "hdx-python-country>=3.8.8",
-    "hdx-python-utilities>=3.8.3",
+    "hdx-python-api>=6.3.9",
+    "hdx-python-country>=3.9.2",
+    "hdx-python-utilities>=3.8.6",
     "gspread",
     "regex",
 ]
@@ -50,55 +50,7 @@ content-type = "text/markdown"
 Homepage = "https://github.com/OCHA-DAP/hdx-python-scraper"
 
 [project.optional-dependencies]
-pandas = ["pandas>=2.2.2"]
-test = ["pytest", "pytest-cov"]
+pandas = ["pandas>=2.2.3"]
+test = ["pandas>=2.2.3", "pytest", "pytest-cov"]
 dev = ["pre-commit"]
-
-
-#########
-# Hatch #
-#########
-
-# Build
-
-[tool.hatch.build.targets.wheel]
-packages = ["src/hdx"]
-
-[tool.hatch.build.hooks.vcs]
-version-file = "src/hdx/scraper/framework/_version.py"
-
-[tool.hatch.metadata]
-allow-direct-references = true
-
-# Versioning
-
-[tool.hatch.version]
-source = "vcs"
-
-[tool.hatch.version.raw-options]
-local_scheme = "no-local-version"
-version_scheme = "python-simplified-semver"
-
-# Tests
-
-[tool.hatch.envs.hatch-test]
-features = ["pandas", "test"]
-
-[[tool.hatch.envs.hatch-test.matrix]]
-python = ["3.12"]
-
-[tool.hatch.envs.hatch-test.scripts]
-run = """
-       pytest -c .config/pytest.ini --rootdir=. --junitxml=test-results.xml \
-       --cov --cov-config=.config/coveragerc --no-cov-on-fail \
-       --cov-report=lcov --cov-report=term-missing
-       """
-
-[tool.hatch.envs.hatch-static-analysis]
-dependencies = ["ruff==0.9.6"]
-
-[tool.hatch.envs.hatch-static-analysis.scripts]
-format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]
-format-fix = ["ruff format --config .config/ruff.toml {args:.}",]
-lint-check = ["ruff check --config .config/ruff.toml {args:.}",]
-lint-fix = ["ruff check --config .config/ruff.toml --fix {args:.}",]
+docs = ["mkapi"]

{hdx_python_scraper-2.6.2/.config → hdx_python_scraper-2.6.4}/pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
-pythonpath = ../src
+pythonpath = src
 addopts = "--color=yes"
 log_cli = 1

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.4}/requirements.txt

@@ -2,28 +2,40 @@
 #    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
 annotated-types==0.7.0
     # via pydantic
-attrs==25.1.0
+astdoc==1.2.1
+    # via mkapi
+attrs==25.3.0
     # via
     #   frictionless
     #   jsonlines
     #   jsonschema
     #   referencing
-cachetools==5.5.1
+babel==2.17.0
+    # via mkdocs-material
+backrefs==5.8
+    # via mkdocs-material
+cachetools==5.5.2
     # via google-auth
-certifi==2025.1.31
+certifi==2025.4.26
     # via requests
 cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.4.1
+charset-normalizer==3.4.2
     # via requests
 ckanapi==4.8
     # via hdx-python-api
 click==8.1.8
-    # via typer
-coverage==7.6.12
+    # via
+    #   mkdocs
+    #   typer
+colorama==0.4.6
+    # via mkdocs-material
+coverage==7.8.0
     # via pytest-cov
+cydifflib==1.2.0
+    # via hdx-python-utilities
 defopt==6.4.0
     # via hdx-python-api
 distlib==0.3.9
@@ -40,32 +52,34 @@ email-validator==2.2.0
     # via hdx-python-api
 et-xmlfile==2.0.0
     # via openpyxl
-filelock==3.17.0
+filelock==3.18.0
     # via virtualenv
-frictionless==5.18.0
+frictionless==5.18.1
     # via hdx-python-utilities
-google-auth==2.38.0
+ghp-import==2.1.0
+    # via mkdocs
+google-auth==2.40.1
     # via
     #   google-auth-oauthlib
     #   gspread
-google-auth-oauthlib==1.2.1
+google-auth-oauthlib==1.2.2
     # via gspread
-gspread==6.1.4
+gspread==6.2.0
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-api==6.3.8
+hdx-python-api==6.3.9
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-country==3.8.8
+hdx-python-country==3.9.2
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
-hdx-python-utilities==3.8.3
+hdx-python-utilities==3.8.6
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
     #   hdx-python-country
-humanize==4.12.0
+humanize==4.12.3
     # via frictionless
-identify==2.6.7
+identify==2.6.10
     # via pre-commit
 idna==3.10
     # via
@@ -75,12 +89,16 @@ ijson==3.3.0
     # via hdx-python-utilities
 inflect==7.5.0
     # via quantulum3
-iniconfig==2.0.0
+iniconfig==2.1.0
     # via pytest
 isodate==0.7.2
     # via frictionless
-jinja2==3.1.5
-    # via frictionless
+jinja2==3.1.6
+    # via
+    #   frictionless
+    #   mkapi
+    #   mkdocs
+    #   mkdocs-material
 jsonlines==4.0.0
     # via hdx-python-utilities
 jsonpath-ng==1.7.0
@@ -89,7 +107,7 @@ jsonschema==4.23.0
     # via
     #   frictionless
     #   tableschema-to-template
-jsonschema-specifications==2024.10.1
+jsonschema-specifications==2025.4.1
     # via jsonschema
 libhxl==5.2.2
     # via
@@ -99,34 +117,65 @@ loguru==0.7.3
     # via hdx-python-utilities
 makefun==1.15.6
     # via hdx-python-api
+markdown==3.8
+    # via
+    #   mkdocs
+    #   mkdocs-material
+    #   pymdown-extensions
 markdown-it-py==3.0.0
     # via rich
-marko==2.1.2
+marko==2.1.3
     # via frictionless
 markupsafe==3.0.2
-    # via jinja2
+    # via
+    #   jinja2
+    #   mkdocs
 mdurl==0.1.2
     # via markdown-it-py
-more-itertools==10.6.0
+mergedeep==1.3.4
+    # via
+    #   mkdocs
+    #   mkdocs-get-deps
+mkapi==4.3.2
+    # via hdx-python-scraper (pyproject.toml)
+mkdocs==1.6.1
+    # via
+    #   mkapi
+    #   mkdocs-material
+mkdocs-get-deps==0.2.0
+    # via mkdocs
+mkdocs-material==9.6.12
+    # via mkapi
+mkdocs-material-extensions==1.3.1
+    # via mkdocs-material
+more-itertools==10.7.0
     # via inflect
 nodeenv==1.9.1
     # via pre-commit
 num2words==0.5.14
     # via quantulum3
-numpy==2.2.3
+numpy==2.2.5
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib
 openpyxl==3.1.5
     # via hdx-python-utilities
-packaging==24.2
-    # via pytest
+packaging==25.0
+    # via
+    #   mkdocs
+    #   pytest
+paginate==0.5.7
+    # via mkdocs-material
 pandas==2.2.3
     # via hdx-python-scraper (pyproject.toml)
-petl==1.7.15
+pathspec==0.12.1
+    # via mkdocs
+petl==1.7.16
     # via frictionless
-platformdirs==4.3.6
-    # via virtualenv
+platformdirs==4.3.7
+    # via
+    #   mkdocs-get-deps
+    #   virtualenv
 pluggy==1.5.0
     # via pytest
 ply==3.11
@@ -135,31 +184,36 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==4.1.0
+pre-commit==4.2.0
     # via hdx-python-scraper (pyproject.toml)
 pyasn1==0.6.1
     # via
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.4.1
+pyasn1-modules==0.4.2
     # via google-auth
-pydantic==2.10.6
+pydantic==2.11.4
     # via frictionless
-pydantic-core==2.27.2
+pydantic-core==2.33.2
     # via pydantic
 pygments==2.19.1
-    # via rich
+    # via
+    #   mkdocs-material
+    #   rich
+pymdown-extensions==10.15
+    # via mkdocs-material
 pyphonetics==0.5.3
     # via hdx-python-utilities
-pytest==8.3.4
+pytest==8.3.5
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   pytest-cov
-pytest-cov==6.0.0
+pytest-cov==6.1.1
     # via hdx-python-scraper (pyproject.toml)
 python-dateutil==2.9.0.post0
     # via
     #   frictionless
+    #   ghp-import
     #   hdx-python-utilities
     #   libhxl
     #   pandas
@@ -169,13 +223,19 @@ python-slugify==8.0.4
     # via
     #   ckanapi
     #   frictionless
-pytz==2025.1
+pytz==2025.2
     # via pandas
 pyyaml==6.0.2
     # via
     #   frictionless
+    #   mkdocs
+    #   mkdocs-get-deps
     #   pre-commit
+    #   pymdown-extensions
+    #   pyyaml-env-tag
     #   tableschema-to-template
+pyyaml-env-tag==0.1
+    # via mkdocs
 quantulum3==0.9.2
     # via hdx-python-api
 ratelimit==2.2.1
@@ -192,6 +252,7 @@ requests==2.32.3
     #   frictionless
     #   hdx-python-api
     #   libhxl
+    #   mkdocs-material
     #   requests-file
     #   requests-oauthlib
 requests-file==2.1.0
@@ -200,19 +261,19 @@ requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
-rich==13.9.4
+rich==14.0.0
     # via typer
-rpds-py==0.22.3
+rpds-py==0.24.0
     # via
     #   jsonschema
     #   referencing
-rsa==4.9
+rsa==4.9.1
     # via google-auth
 ruamel-yaml==0.18.10
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.12
     # via ruamel-yaml
-setuptools==75.8.0
+setuptools==80.3.1
     # via ckanapi
 shellingham==1.5.4
     # via typer
@@ -228,23 +289,21 @@ six==1.17.0
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
-stringcase==1.2.0
-    # via frictionless
-structlog==25.1.0
+structlog==25.3.0
     # via libhxl
 tableschema-to-template==0.0.13
     # via hdx-python-utilities
 tabulate==0.9.0
     # via frictionless
-tenacity==9.0.0
+tenacity==9.1.2
     # via hdx-python-country
 text-unidecode==1.3
     # via python-slugify
 typeguard==4.4.2
     # via inflect
-typer==0.15.1
+typer==0.15.3
     # via frictionless
-typing-extensions==4.12.2
+typing-extensions==4.13.2
     # via
     #   frictionless
     #   pydantic
@@ -252,20 +311,25 @@ typing-extensions==4.12.2
     #   referencing
     #   typeguard
     #   typer
-tzdata==2025.1
+    #   typing-inspection
+typing-inspection==0.4.0
+    # via pydantic
+tzdata==2025.2
     # via pandas
-unidecode==1.3.8
+unidecode==1.4.0
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.3.0
+urllib3==2.4.0
     # via
     #   libhxl
     #   requests
-validators==0.34.0
+validators==0.35.0
     # via frictionless
-virtualenv==20.29.2
+virtualenv==20.31.1
     # via pre-commit
+watchdog==6.0.0
+    # via mkdocs
 wheel==0.45.1
     # via libhxl
 xlrd==2.0.1
@@ -274,7 +338,7 @@ xlrd3==1.1.0
     # via libhxl
 xlsx2csv==0.8.4
     # via hdx-python-utilities
-xlsxwriter==3.2.2
+xlsxwriter==3.2.3
     # via tableschema-to-template
 xlwt==1.3.0
     # via hdx-python-utilities

{hdx_python_scraper-2.6.2/.config → hdx_python_scraper-2.6.4}/ruff.toml

@@ -1,4 +1,3 @@
-line-length = 79
 exclude = ["_version.py"]
 
 [lint]

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.4}/src/hdx/scraper/framework/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.6.2'
-__version_tuple__ = version_tuple = (2, 6, 2)
+__version__ = version = '2.6.4'
+__version_tuple__ = version_tuple = (2, 6, 4)

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.4}/src/hdx/scraper/framework/base_scraper.py

@@ -79,8 +79,7 @@ class BaseScraper(ABC):
              None
         """
         self.values: Dict[str, Tuple] = {
-            level: tuple({} for _ in value[0])
-            for level, value in self.headers.items()
+            level: tuple({} for _ in value[0]) for level, value in self.headers.items()
         }
         self.sources: Dict[str, List] = {level: [] for level in self.headers}
         self.source_configuration = deepcopy(source_configuration)
@@ -137,9 +136,7 @@ class BaseScraper(ABC):
             return
         if self.datasetinfo.get("no_sources", False):
             return
-        should_overwrite_sources = self.datasetinfo.get(
-            "should_overwrite_sources"
-        )
+        should_overwrite_sources = self.datasetinfo.get("should_overwrite_sources")
         if should_overwrite_sources is not None:
             self.source_configuration["should_overwrite_sources"] = (
                 should_overwrite_sources
@@ -209,16 +206,12 @@ class BaseScraper(ABC):
                 )
 
             for i, hxltag in enumerate(self.headers[level][1]):
-                suffix_attribute = self.source_configuration.get(
-                    "suffix_attribute"
-                )
+                suffix_attribute = self.source_configuration.get("suffix_attribute")
                 if suffix_attribute:
                     add_source(hxltag, suffix_attribute)
                     continue
                 values = self.get_values(level)[i]
-                admin_sources = self.source_configuration.get(
-                    "admin_sources", False
-                )
+                admin_sources = self.source_configuration.get("admin_sources", False)
                 if not admin_sources:
                     raise ValueError("Invalid source configuration!")
                 admin_mapping = self.source_configuration.get("admin_mapping")
@@ -260,9 +253,7 @@ class BaseScraper(ABC):
         """
         if datasetinfo is None:
             datasetinfo = self.datasetinfo
-        date = Sources.get_hxltag_source_date(
-            datasetinfo, hxltag, fallback=True
-        )
+        date = Sources.get_hxltag_source_date(datasetinfo, hxltag, fallback=True)
         if key is None:
             key = self.name
         dict_of_lists_add(