hdx-python-pipelineutils 0.0.1__tar.gz

hdx_python_pipelineutils-0.0.1/.github/workflows/publish.yaml

@@ -0,0 +1,34 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/hdx-python-pipelineutils
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Get history and tags for versioning to work
+      run: |
+        git fetch --prune --unshallow
+        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+
+    - name: Build with uv
+      run: uv build
+
+    - name: Publish distribution 📦 to PyPI
+      run: uv publish

hdx_python_pipelineutils-0.0.1/.github/workflows/run-python-tests.yaml

@@ -0,0 +1,52 @@
+name: Run tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches-ignore: [gh-pages, "dependabot/**"]
+  pull_request:
+    branches-ignore: [gh-pages]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      checks: write
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: uv sync --frozen
+
+      - name: Check styling
+        run: |
+          uv run ruff format --check
+          uv run ruff check
+
+      - name: Test with pytest
+        env:
+          HDX_KEY_TEST: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }}
+          GSHEET_AUTH: ${{ secrets.HDX_PIPELINE_GSHEET_AUTH }}
+        run: uv run pytest
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: test-results.xml
+
+      - name: Publish in Coveralls
+        uses: coverallsapp/github-action@v2
+        if: always()
+        with:
+          flag-name: tests
+          format: lcov

hdx_python_pipelineutils-0.0.1/.gitignore

@@ -0,0 +1,102 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.lcov
+coverage.xml
+test-results.xml
+*,cover
+.hypothesis/
+tests/*.log
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+documentation/_*/
+documentation/source/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# IntelliJ
+.idea/
+
+# hatch-vcs
+_version.py
+
+.vscode/settings.json

hdx_python_pipelineutils-0.0.1/.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+default_language_version:
+  python: python3.13
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+        exclude: test_scraper_.*\.json
+      - id: check-ast
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.14.14
+    hooks:
+      # Run the linter.
+      - id: ruff-check
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
+
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.9.25
+    hooks:
+      # Ensure the lockfile is up-to-date with pyproject.toml
+      - id: uv-lock

hdx_python_pipelineutils-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Michael Rans
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

hdx_python_pipelineutils-0.0.1/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.4
+Name: hdx-python-pipelineutils
+Version: 0.0.1
+Summary: HDX Python Pipeline Utilities
+Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-pipelineutils
+Author-email: Michael Rans <rans@email.com>
+License: MIT
+License-File: LICENSE
+Keywords: HDX,pipelines,scrapers,utilities
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: Unix
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: gspread
+Requires-Dist: hdx-python-api>=6.6.7
+Requires-Dist: hdx-python-country>=4.1.1
+Requires-Dist: hdx-python-utilities>=4.0.8
+Requires-Dist: libhxl
+Requires-Dist: regex
+Provides-Extra: docs
+Requires-Dist: mkapi; extra == 'docs'
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.3.3; extra == 'pandas'
+Description-Content-Type: text/markdown
+
+[![Build Status](https://github.com/OCHA-DAP/hdx-python-pipelineutils/actions/workflows/run-python-tests.yaml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-pipelineutils/actions/workflows/run-python-tests.yaml)
+[![Coverage Status](https://coveralls.io/repos/github/OCHA-DAP/hdx-python-pipelineutils/badge.svg?branch=main&ts=1)](https://coveralls.io/github/OCHA-DAP/hdx-python-pipelineutils?branch=main)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Downloads](https://img.shields.io/pypi/dm/hdx-python-pipelineutils.svg)](https://pypistats.org/packages/hdx-python-pipelineutils)
+
+
+The HDX Python PipelineUtils Library contains various utilities for use by pipelines.
+
+This library is part of the
+[Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have
+humanitarian related data, please upload your datasets to HDX.

hdx_python_pipelineutils-0.0.1/README.md

@@ -0,0 +1,11 @@
+[![Build Status](https://github.com/OCHA-DAP/hdx-python-pipelineutils/actions/workflows/run-python-tests.yaml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-pipelineutils/actions/workflows/run-python-tests.yaml)
+[![Coverage Status](https://coveralls.io/repos/github/OCHA-DAP/hdx-python-pipelineutils/badge.svg?branch=main&ts=1)](https://coveralls.io/github/OCHA-DAP/hdx-python-pipelineutils?branch=main)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Downloads](https://img.shields.io/pypi/dm/hdx-python-pipelineutils.svg)](https://pypistats.org/packages/hdx-python-pipelineutils)
+
+
+The HDX Python PipelineUtils Library contains various utilities for use by pipelines.
+
+This library is part of the
+[Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have
+humanitarian related data, please upload your datasets to HDX.

hdx_python_pipelineutils-0.0.1/pyproject.toml

@@ -0,0 +1,150 @@
+#########################
+# Project Configuration #
+#########################
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "hdx-python-pipelineutils"
+description = "HDX Python Pipeline Utilities"
+authors = [{name = "Michael Rans", email = "rans@email.com"}]
+license = {text = "MIT"}
+keywords = ["HDX", "scrapers", "pipelines", "utilities"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Natural Language :: English",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: Unix",
+    "Operating System :: MacOS",
+    "Operating System :: Microsoft :: Windows",
+]
+readme = "README.md"
+dynamic = ["version"]
+requires-python = ">=3.10"
+
+
+dependencies = [
+    "hdx-python-api>=6.6.7",
+    "hdx-python-country>=4.1.1",
+    "hdx-python-utilities>=4.0.8",
+    "libhxl",
+    "gspread",
+    "regex",
+]
+
+[project.optional-dependencies]
+pandas = ["pandas>=2.3.3"]
+docs = ["mkapi"]
+
+[dependency-groups]
+dev = [
+    "pandas>=2.3.3",
+    "pytest",
+    "pytest-cov",
+    "pre-commit",
+    "ruff==0.14.14",
+]
+
+[project.urls]
+Homepage = "https://github.com/OCHA-DAP/hdx-python-pipelineutils"
+
+# ----------------------------------------------------------------------------
+# Hatchling (Build & Versioning)
+# ----------------------------------------------------------------------------
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.version.raw-options]
+local_scheme = "no-local-version"
+version_scheme = "python-simplified-semver"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/hdx/python/pipelineutils/_version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/hdx"]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+# ----------------------------------------------------------------------------
+# Ruff (Linting & Formatting)
+# ----------------------------------------------------------------------------
+
+[tool.ruff]
+target-version = "py310"
+src = ["src"]
+exclude = ["_version.py"]
+
+[tool.ruff.lint]
+# Defaults are E (pycodestyle) and F (pyflakes). We extend them:
+extend-select = [
+  "I",   # isort
+  "UP",  # pyupgrade
+]
+ignore = [
+    "E501", # Line too long
+]
+
+[tool.ruff.lint.isort]
+known-local-folder = ["hdx.python.pipelineutils"]
+known-third-party = [
+    "hdx.api",
+    "hdx.data",
+    "hdx.facades",
+    "hdx.location",
+    "hdx.utilities",
+]
+
+# ----------------------------------------------------------------------------
+# Pytest (Testing)
+# ----------------------------------------------------------------------------
+
+[tool.pytest.ini_options]
+pythonpath = "src"
+log_cli = true
+addopts = """
+    --color=yes
+    --rootdir=.
+    --junitxml=test-results.xml
+    --cov
+    --no-cov-on-fail
+    --cov-report=lcov
+    --cov-report=term-missing
+"""
+
+# ----------------------------------------------------------------------------
+# Coverage (Reporting)
+# ----------------------------------------------------------------------------
+
+[tool.coverage.run]
+source = ["src"]
+omit = ["*/_version.py"]
+
+[tool.coverage.report]
+exclude_also = [
+    "from ._version",
+    "def __repr__",
+    "if self.debug:",
+    "if settings.DEBUG",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if 0:",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "@(abc\\.)?abstractmethod",
+]

hdx_python_pipelineutils-0.0.1/src/hdx/python/pipelineutils/__init__.py

@@ -0,0 +1,55 @@
+import re
+from datetime import datetime
+
+from hdx.data.dataset import Dataset
+
+template = re.compile("{{.*?}}")
+
+
+def string_params_to_dict(string: str) -> dict[str, str]:
+    params = {}
+    if not string:
+        return params
+    for name_par in string.split(","):
+        name, par = name_par.strip().split(":")
+        params[name] = par.strip()
+    return params
+
+
+def match_template(input: str) -> tuple[str | None, str | None]:
+    """Try to match {{XXX}} in input string
+
+    Args:
+        input: String in which to look for template
+
+    Returns:
+        (Matched string with brackets, matched string without brackets)
+    """
+    match = template.search(input)
+    if match:
+        template_string = match.group()
+        return template_string, template_string[2:-2]
+    return None, None
+
+
+def get_startend_dates_from_time_period(
+    dataset: Dataset, today: datetime | None = None
+) -> dict | None:
+    """Return the time period in form required for source_date
+
+    Args:
+        dataset: Dataset object
+        today: Date to use for today. Default is None (datetime.utcnow)
+
+    Returns:
+        Time period in form required for source_date
+    """
+    if today is None:
+        date_info = dataset.get_time_period()
+    else:
+        date_info = dataset.get_time_period(today=today)
+    startdate = date_info.get("startdate")
+    enddate = date_info.get("enddate")
+    if enddate is None:
+        return None
+    return {"start": startdate, "end": enddate}

hdx_python_pipelineutils-0.0.1/src/hdx/python/pipelineutils/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.0.1'
+__version_tuple__ = version_tuple = (0, 0, 1)
+
+__commit_id__ = commit_id = None

hdx_python_pipelineutils-0.0.1/src/hdx/python/pipelineutils/hapi_admins.py

@@ -0,0 +1,117 @@
+from hdx.location.adminlevel import AdminLevel
+
+
+def complete_admins(
+    admins: list[AdminLevel],
+    countryiso3: str,
+    provider_adm_names: list,
+    adm_codes: list,
+    adm_names: list,
+    fuzzy_match: bool = True,
+) -> tuple[int, list[str]]:
+    """Use information from adm_codes to populate adm_names and from
+    provider_adm_names to populate adm_codes with outptu of the admin level
+    and arnings for unknown and mismatched p-codes. All provided lists
+    should be of the same length.
+
+    Args:
+        admins: List of AdminLevel objects
+        countryiso3: Country ISO3 code
+        provider_adm_names: List of provider adm names
+        adm_codes: List of adm codes
+        adm_names: List of adm names
+        fuzzy_match: Whether to use fuzzy matching. Default is True.
+
+    Returns:
+        Admin level and warnings
+    """
+
+    warnings = []
+    child = None
+    adm_level = len(provider_adm_names)
+
+    def check_unknown_pcode(adm_code: str, pcode: str) -> str:
+        if pcode:
+            warnings.append(f"PCode unknown {adm_code}->{pcode} ({warntxt})")
+            return pcode
+        else:
+            warnings.append(f"PCode unknown {adm_code}->''")
+            return ""
+
+    for i, provider_adm_name in reversed(list(enumerate(provider_adm_names))):
+        adm_code = adm_codes[i]
+        parent = admins[i].pcode_to_parent.get(adm_code)
+        if not parent and i > 0:
+            parent = adm_codes[i - 1]
+        if not provider_adm_name:
+            provider_adm_name = ""
+            provider_adm_names[i] = ""
+        if child:
+            pcode = admins[i + 1].pcode_to_parent.get(child)
+            warntxt = "parent"
+        elif provider_adm_name:
+            pcode, _ = admins[i].get_pcode(
+                countryiso3,
+                provider_adm_name,
+                parent=parent,
+                fuzzy_match=fuzzy_match,
+            )
+            warntxt = f"provider_adm{i + 1}_name"
+        else:
+            pcode = None
+        if adm_code:
+            if adm_code not in admins[i].pcodes:
+                if admins[i].looks_like_pcode(adm_code):
+                    adj_adm_code = admins[i].convert_admin_pcode_length(
+                        countryiso3, adm_code, parent=parent
+                    )
+                    if adj_adm_code:
+                        warnings.append(f"PCode length {adm_code}->{adj_adm_code}")
+                        adm_code = adj_adm_code
+                    else:
+                        adm_code = check_unknown_pcode(adm_code, pcode)
+                else:
+                    adm_code = check_unknown_pcode(adm_code, pcode)
+            elif pcode and adm_code != pcode:
+                if child:
+                    warnings.append(f"PCode mismatch {adm_code}->{pcode} ({warntxt})")
+                    adm_code = pcode
+                else:
+                    warnings.append(f"PCode mismatch {adm_code} != {provider_adm_name}")
+        elif pcode:
+            adm_code = pcode
+        else:
+            adm_code = ""
+        adm_codes[i] = adm_code
+        if adm_code:
+            adm_names[i] = admins[i].pcode_to_name.get(adm_code, "")
+            child = adm_code
+        else:
+            adm_names[i] = ""
+            if provider_adm_name == "":
+                adm_level -= 1
+    return adm_level, warnings
+
+
+def pad_admins(
+    provider_adm_names: list[str],
+    adm_codes: list[str],
+    adm_names: list[str],
+    adm_level: int = 2,
+) -> None:
+    """Pad lists to size given in adm_level adding as many "" as needed.
+
+    Args:
+        provider_adm_names: List of provider adm names
+        adm_codes: List of adm codes
+        adm_names: List of adm names
+        adm_level: Admin level to which to pad. Default is 2.
+
+    Returns:
+        Admin level and warnings
+    """
+
+    for i in range(len(provider_adm_names), adm_level):
+        provider_adm_names.append("")
+        adm_codes.append("")
+        adm_names.append("")