hdx-python-api 6.6.3__tar.gz → 6.6.5__tar.gz

{hdx_python_api-6.6.3 → hdx_python_api-6.6.5}/.github/workflows/publish.yaml

@@ -13,25 +13,22 @@ jobs:
       url: https://pypi.org/p/hdx-python-api
 
     permissions:
-      id-token: write  # IMPORTANT: mandatory for trusted publishing
+      id-token: write
+      contents: read
 
     steps:
     - uses: actions/checkout@v6
+
     - name: Get history and tags for versioning to work
       run: |
         git fetch --prune --unshallow
         git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-    - name: Install Hatch
-      uses: pypa/hatch@install
-    - name: Build with hatch
-      run: |
-        hatch build
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+
+    - name: Build with uv
+      run: uv build
+
     - name: Publish distribution 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
+      run: uv publish

hdx_python_api-6.6.5/.github/workflows/run-python-tests.yaml

@@ -0,0 +1,52 @@
+name: Run tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches-ignore: [gh-pages, "dependabot/**"]
+  pull_request:
+    branches-ignore: [gh-pages]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      checks: write
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: uv sync --frozen
+
+      - name: Check styling
+        run: |
+          uv run ruff format --check
+          uv run ruff check
+
+      - name: Test with pytest
+        env:
+          HDX_KEY_TEST: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }}
+          GSHEET_AUTH: ${{ secrets.HDX_PIPELINE_GSHEET_AUTH }}
+        run: uv run pytest
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: test-results.xml
+
+      - name: Publish in Coveralls
+        uses: coverallsapp/github-action@v2
+        if: always()
+        with:
+          flag-name: tests
+          format: lcov

{hdx_python_api-6.6.3 → hdx_python_api-6.6.5}/.pre-commit-config.yaml

@@ -1,26 +1,27 @@
 default_language_version:
-    python: python3.13
+  python: python3.13
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
     hooks:
       - id: trailing-whitespace
+        exclude: (test_loader.py|pretty-false_sortkeys-false.yaml|pretty-false_sortkeys-true.yaml)
       - id: end-of-file-fixer
+        exclude: (test_csv_processing_blanks.csv|test.txt)
       - id: check-ast
+
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.10
+    rev: v0.14.14
     hooks:
       # Run the linter.
       - id: ruff-check
         args: [ --fix ]
       # Run the formatter.
       - id: ruff-format
+
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.9.22
+    rev: 0.9.25
     hooks:
-      # Run the pip compile
-      - id: pip-compile
-        name: pip-compile requirements.txt
-        files: pyproject.toml
-        args: [ pyproject.toml, --resolver=backtracking, --upgrade, -q,
-                -o, requirements.txt ]
+      # Ensure the lockfile is up-to-date with pyproject.toml
+      - id: uv-lock

{hdx_python_api-6.6.3 → hdx_python_api-6.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hdx-python-api
-Version: 6.6.3
+Version: 6.6.5
 Summary: HDX Python API for interacting with the Humanitarian Data Exchange
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-api
 Author: Michael Rans
@@ -18,31 +18,22 @@ Classifier: Operating System :: Unix
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.10
 Requires-Dist: ckanapi>=4.8
 Requires-Dist: defopt>=7.0.0
 Requires-Dist: email-validator
 Requires-Dist: hdx-python-country>=4.0.1
-Requires-Dist: hdx-python-utilities>=4.0.3
-Requires-Dist: libhxl>=5.2.2
+Requires-Dist: hdx-python-utilities>=4.0.4
 Requires-Dist: makefun
-Requires-Dist: quantulum3
 Requires-Dist: requests
-Provides-Extra: dev
-Requires-Dist: pre-commit; extra == 'dev'
 Provides-Extra: docs
 Requires-Dist: mkapi; extra == 'docs'
-Provides-Extra: test
-Requires-Dist: gspread; extra == 'test'
-Requires-Dist: pytest; extra == 'test'
-Requires-Dist: pytest-check; extra == 'test'
-Requires-Dist: pytest-cov; extra == 'test'
 Description-Content-Type: text/markdown
 
 [![Build Status](https://github.com/OCHA-DAP/hdx-python-api/actions/workflows/run-python-tests.yaml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-api/actions/workflows/run-python-tests.yaml)
@@ -61,3 +52,87 @@ For more information, please read the
 This library is part of the [Humanitarian Data Exchange](https://data.humdata.org/)
 (HDX) project. If you have humanitarian related data, please upload your datasets to
 HDX.
+
+# Development
+
+## Environment
+
+Development is currently done using Python 3.13. The environment can be created with:
+
+```shell
+    uv sync
+```
+
+This creates a .venv folder with the versions specified in the project's uv.lock file.
+
+### Pre-commit
+
+pre-commit will be installed when syncing uv. It is run every time you make a git
+commit if you call it like this:
+
+```shell
+    pre-commit install
+```
+
+With pre-commit, all code is formatted according to
+[ruff](https://docs.astral.sh/ruff/) guidelines.
+
+To check if your changes pass pre-commit without committing, run:
+
+```shell
+    pre-commit run --all-files
+```
+
+## Packages
+
+[uv](https://github.com/astral-sh/uv) is used for package management.  If
+you’ve introduced a new package to the source code (i.e. anywhere in `src/`),
+please add it to the `project.dependencies` section of `pyproject.toml` with
+any known version constraints.
+
+To add packages required only for testing, add them to the
+`[dependency-groups]`.
+
+Any changes to the dependencies will be automatically reflected in
+`uv.lock` with `pre-commit`, but you can re-generate the files without committing by
+executing:
+
+```shell
+    uv lock --upgrade
+```
+
+## Project
+
+[uv](https://github.com/astral-sh/uv) is used for project management. The project can be
+built using:
+
+```shell
+    uv build
+```
+
+Linting and syntax checking can be run with:
+
+```shell
+    uv run ruff check
+```
+
+To run the tests and view coverage, execute:
+
+```shell
+    uv run pytest
+```
+
+For the `test_ckan.py` test to run successfully, some configuration is required:
+
+1. The environment variable HDX_KEY_TEST needs to contain a valid key from the HDX demo
+server at https://demo.data-humdata-org.ahconu.org/
+1. Authentication details for Google Sheets need to be obtained and either saved in a
+file named `.gsheet_auth.json` in the home directory (~) or placed in an environment
+variable `GSHEET_AUTH`. The file is preferred for Windows systems since adding such a
+long text string to an environment variable in Windows is challenging.
+
+## Documentation
+
+The documentation, including API documentation, is generated using ReadtheDocs and
+MkDocs with Material. As you change the source code, remember to update the
+documentation at `documentation/index.md`.

hdx_python_api-6.6.5/README.md

@@ -0,0 +1,100 @@
+[![Build Status](https://github.com/OCHA-DAP/hdx-python-api/actions/workflows/run-python-tests.yaml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-api/actions/workflows/run-python-tests.yaml)
+[![Coverage Status](https://coveralls.io/repos/github/OCHA-DAP/hdx-python-api/badge.svg?branch=main&ts=1)](https://coveralls.io/github/OCHA-DAP/hdx-python-api?branch=main)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Downloads](https://img.shields.io/pypi/dm/hdx-python-api.svg)](https://pypistats.org/packages/hdx-python-api)
+
+The HDX Python API Library is designed to enable you to easily develop code that
+interacts with the [Humanitarian Data Exchange](https://data.humdata.org/) (HDX)
+platform. The major goal of the library is to make pushing and pulling data from HDX as
+simple as possible for the end user.
+
+For more information, please read the
+[documentation](https://hdx-python-api.readthedocs.io/en/latest/).
+
+This library is part of the [Humanitarian Data Exchange](https://data.humdata.org/)
+(HDX) project. If you have humanitarian related data, please upload your datasets to
+HDX.
+
+# Development
+
+## Environment
+
+Development is currently done using Python 3.13. The environment can be created with:
+
+```shell
+    uv sync
+```
+
+This creates a .venv folder with the versions specified in the project's uv.lock file.
+
+### Pre-commit
+
+pre-commit will be installed when syncing uv. It is run every time you make a git
+commit if you call it like this:
+
+```shell
+    pre-commit install
+```
+
+With pre-commit, all code is formatted according to
+[ruff](https://docs.astral.sh/ruff/) guidelines.
+
+To check if your changes pass pre-commit without committing, run:
+
+```shell
+    pre-commit run --all-files
+```
+
+## Packages
+
+[uv](https://github.com/astral-sh/uv) is used for package management.  If
+you’ve introduced a new package to the source code (i.e. anywhere in `src/`),
+please add it to the `project.dependencies` section of `pyproject.toml` with
+any known version constraints.
+
+To add packages required only for testing, add them to the
+`[dependency-groups]`.
+
+Any changes to the dependencies will be automatically reflected in
+`uv.lock` with `pre-commit`, but you can re-generate the files without committing by
+executing:
+
+```shell
+    uv lock --upgrade
+```
+
+## Project
+
+[uv](https://github.com/astral-sh/uv) is used for project management. The project can be
+built using:
+
+```shell
+    uv build
+```
+
+Linting and syntax checking can be run with:
+
+```shell
+    uv run ruff check
+```
+
+To run the tests and view coverage, execute:
+
+```shell
+    uv run pytest
+```
+
+For the `test_ckan.py` test to run successfully, some configuration is required:
+
+1. The environment variable HDX_KEY_TEST needs to contain a valid key from the HDX demo
+server at https://demo.data-humdata-org.ahconu.org/
+1. Authentication details for Google Sheets need to be obtained and either saved in a
+file named `.gsheet_auth.json` in the home directory (~) or placed in an environment
+variable `GSHEET_AUTH`. The file is preferred for Windows systems since adding such a
+long text string to an environment variable in Windows is challenging.
+
+## Documentation
+
+The documentation, including API documentation, is generated using ReadtheDocs and
+MkDocs with Material. As you change the source code, remember to update the
+documentation at `documentation/index.md`.

{hdx_python_api-6.6.3 → hdx_python_api-6.6.5}/documentation/index.md

@@ -54,6 +54,8 @@ The library has detailed API documentation which can be found in the menu at the
 
 
 ## Breaking Changes
+From 6.6.5, removed DatasetTitleHelper class and Dataset method remove_dates_from_title
+
 From 6.6.0, Python 3.10 or later is required
 
 From 6.5.7, get_size_and_hash moved to HDX Python Utilities

hdx_python_api-6.6.5/pyproject.toml

@@ -0,0 +1,148 @@
+#########################
+# Project Configuration #
+#########################
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "hdx-python-api"
+description = "HDX Python API for interacting with the Humanitarian Data Exchange"
+authors = [{name = "Michael Rans"}]
+license = {text = "MIT"}
+keywords = ["HDX", "API", "CKAN", "humanitarian data exchange"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Natural Language :: English",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: Unix",
+    "Operating System :: MacOS",
+    "Operating System :: Microsoft :: Windows",
+]
+readme = "README.md"
+dynamic = ["version"]
+requires-python = ">=3.10"
+
+dependencies = [
+    "ckanapi>=4.8",
+    "defopt>=7.0.0",
+    "email_validator",
+    "hdx-python-country>=4.0.1",
+    "hdx-python-utilities>=4.0.4",
+    "makefun",
+    "requests",
+]
+
+[project.optional-dependencies]
+docs = ["mkapi"]
+
+[dependency-groups]
+dev = [
+    "gspread",
+    "pytest",
+    "pytest-check",
+    "pytest-cov",
+    "pytest-mock",
+    "pre-commit",
+    "ruff==0.14.14",
+]
+
+[project.urls]
+Homepage = "https://github.com/OCHA-DAP/hdx-python-api"
+
+# ----------------------------------------------------------------------------
+# Hatchling (Build & Versioning)
+# ----------------------------------------------------------------------------
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.version.raw-options]
+local_scheme = "no-local-version"
+version_scheme = "python-simplified-semver"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/hdx/api/_version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/hdx"]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+# ----------------------------------------------------------------------------
+# Ruff (Linting & Formatting)
+# ----------------------------------------------------------------------------
+
+[tool.ruff]
+target-version = "py310"
+src = ["src"]
+exclude = ["_version.py"]
+
+[tool.ruff.lint]
+# Defaults are E (pycodestyle) and F (pyflakes). We extend them:
+extend-select = [
+  "I",   # isort
+  "UP",  # pyupgrade
+]
+ignore = [
+    "E501", # Line too long
+]
+
+[tool.ruff.lint.isort]
+known-local-folder = ["hdx.api", "hdx.data", "hdx.facades"]
+known-third-party = [
+    "hdx.location",
+    "hdx.utilities",
+]
+
+# ----------------------------------------------------------------------------
+# Pytest (Testing)
+# ----------------------------------------------------------------------------
+
+[tool.pytest.ini_options]
+pythonpath = "src"
+log_cli = true
+addopts = """
+    --color=yes
+    --rootdir=.
+    --junitxml=test-results.xml
+    --cov
+    --no-cov-on-fail
+    --cov-report=lcov
+    --cov-report=term-missing
+"""
+
+# ----------------------------------------------------------------------------
+# Coverage (Reporting)
+# ----------------------------------------------------------------------------
+
+[tool.coverage.run]
+source = ["src"]
+omit = ["*/_version.py"]
+
+[tool.coverage.report]
+exclude_also = [
+    "from ._version",
+    "def __repr__",
+    "if self.debug:",
+    "if settings.DEBUG",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if 0:",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "@(abc\\.)?abstractmethod",
+]

{hdx_python_api-6.6.3 → hdx_python_api-6.6.5}/src/hdx/api/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '6.6.3'
-__version_tuple__ = version_tuple = (6, 6, 3)
+__version__ = version = '6.6.5'
+__version_tuple__ = version_tuple = (6, 6, 5)
 
 __commit_id__ = commit_id = None

hdx_python_api-6.6.5/src/hdx/api/utilities/url_utils.py

@@ -0,0 +1,135 @@
+import re
+
+from hdx.utilities.session import get_session
+from requests import RequestException, Session
+
+import hdx.data.dataset
+import hdx.data.resource
+from hdx.api.configuration import Configuration
+
+GOOGLE_DRIVE_URL = re.compile(r"^https?://drive.google.com/open\?id=([0-9A-Za-z_-]+)$")
+GOOGLE_SHEETS_URL = re.compile(
+    r"^https?://[^/]+google.com/.*[^0-9A-Za-z_-]([0-9A-Za-z_-]{44})(?:.*gid=([0-9]+))?.*$"
+)
+GOOGLE_SHEETS_XLSX_URL = re.compile(
+    r"^https?://[^/]+google.com/.*[^0-9A-Za-z_-]([0-9A-Za-z_-]{33})(?:.*gid=([0-9]+))?.*$"
+)
+GOOGLE_FILE_URL = re.compile(r"https?://drive.google.com/file/d/([0-9A-Za-z_-]+)/.*$")
+DROPBOX_URL = re.compile(r"^https://www.dropbox.com/s/([0-9a-z]{15})/([^?]+)\?dl=[01]$")
+CKAN_URL = re.compile(
+    r"^(https?://[^/]+)/dataset/([^/]+)(?:/resource/([a-z0-9-]{36}))?$"
+)
+
+
+def get_ckan_ready_session(configuration: Configuration) -> Session:
+    """Get Session object setup to access CKAN
+
+    Args:
+        configuration: Configuration object
+
+    Returns:
+        Session object setup to access CKAN
+    """
+    apikey = configuration.get_api_key()
+    if apikey:
+        headers = {"Authorization": apikey}
+    else:
+        headers = None
+    session = get_session(
+        full_agent=configuration.get_user_agent(), use_env=False, headers=headers
+    )
+    return session
+
+
+def follow_url(
+    url: str, configuration: Configuration | None = None, session: Session | None = None
+) -> str:
+    """Follow a URL to get the direct-download URL after any redirects. Either a
+    Configuration object or Session object must be provided
+
+    Args:
+        url: URL to follow
+        configuration: Configuration object. Defaults to None.
+        session: Session object to use. Defaults to None.
+
+    Returns:
+        The direct-download URL
+
+    """
+    try:
+        if session is None:
+            if configuration is None:
+                raise ValueError(
+                    "Either Session or Configuration object must be provided!"
+                )
+            session = get_ckan_ready_session(configuration)
+
+        # Is it a CKAN resource? (Assumes the v.3 API for now)
+        result = CKAN_URL.match(url)
+        if result:
+            result = _get_ckan_urls(result.group(2), result.group(3))
+            if result:
+                return result
+
+        # Is it a Google Drive "open" URL?
+        result = GOOGLE_DRIVE_URL.match(url)
+        if result:
+            response = session.head(url)
+            if response.is_redirect:
+                return response.headers["Location"]
+
+        #
+        # Stage 2: rewrite URLs to get direct-download links
+        #
+
+        # Is it a Google *Sheet*?
+        result = GOOGLE_SHEETS_URL.match(url)
+        if result and not re.search(r"/pub", url):
+            if result.group(2):
+                return f"https://docs.google.com/spreadsheets/d/{result.group(1)}/export?format=csv&gid={result.group(2)}"
+            return f"https://docs.google.com/spreadsheets/d/{result.group(1)}/export?format=csv"
+
+        # Is it a Google Drive *file*?
+        result = GOOGLE_FILE_URL.match(url)
+        if not result:
+            result = GOOGLE_SHEETS_XLSX_URL.match(url)
+        if result and not re.search(r"/pub", url):
+            return f"https://drive.google.com/uc?export=download&id={result.group(1)}"
+
+        # Is it a Dropbox URL?
+        result = DROPBOX_URL.match(url)
+        if result:
+            return f"https://www.dropbox.com/s/{result.group(1)}/{result.group(2)}?dl=1"
+    except RequestException:
+        pass
+    # No changes
+    return url
+
+
+def _get_ckan_urls(dataset_id: str, resource_id: str) -> str | None:
+    """Look up a CKAN download URL starting from a dataset or resource page
+
+    If the link is to a dataset page, try the first resource. If it's
+    to a resource page, look up the resource's download link. Either
+    dataset_id or resource_id is required (will prefer resource_id
+    over dataset_id).
+
+    Args:
+        dataset_id: the CKAN dataset ID, or None if unavailable
+        resource_id: the CKAN resource ID, or None if unavailable
+
+    Returns:
+        The direct-download URL for the CKAN dataset
+    """
+    if resource_id:
+        resource = hdx.data.resource.Resource.read_from_hdx(resource_id)
+        if resource:
+            return resource["url"]
+
+    dataset = hdx.data.dataset.Dataset.read_from_hdx(dataset_id)
+    if not dataset:
+        return None
+    for resource in dataset.get_resources():
+        if url := resource.get("url"):
+            return url
+    return None