hdx-python-utilities 4.0.2__tar.gz → 4.0.4__tar.gz

hdx_python_utilities-4.0.4/.github/workflows/publish-test.yaml

@@ -0,0 +1,33 @@
+name: Publish to TestPyPI
+
+on:
+  workflow_dispatch: # add run button in github
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/hdx-python-utilities
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Get history and tags for versioning to work
+      run: |
+        git fetch --prune --unshallow
+        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+
+    - name: Build with uv
+      run: uv build
+
+    - name: Publish distribution 📦 to TestPyPI
+      run: uv publish --publish-url https://test.pypi.org/legacy/

hdx_python_utilities-4.0.4/.github/workflows/publish.yaml

@@ -0,0 +1,34 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/hdx-python-utilities
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Get history and tags for versioning to work
+      run: |
+        git fetch --prune --unshallow
+        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+
+    - name: Build with uv
+      run: uv build
+
+    - name: Publish distribution 📦 to PyPI
+      run: uv publish

hdx_python_utilities-4.0.4/.github/workflows/run-python-tests.yaml

@@ -0,0 +1,49 @@
+name: Run tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches-ignore: [gh-pages, "dependabot/**"]
+  pull_request:
+    branches-ignore: [gh-pages]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      checks: write
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: uv sync --frozen
+
+      - name: Check styling
+        run: |
+          uv run ruff format --check
+          uv run ruff check
+
+      - name: Test with pytest
+        run: uv run --extra html --extra diff --extra email pytest
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: test-results.xml
+
+      - name: Publish in Coveralls
+        uses: coverallsapp/github-action@v2
+        if: always()
+        with:
+          flag-name: tests
+          format: lcov

{hdx_python_utilities-4.0.2 → hdx_python_utilities-4.0.4}/.pre-commit-config.yaml

@@ -1,5 +1,6 @@
 default_language_version:
-    python: python3.13
+  python: python3.13
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
@@ -9,20 +10,18 @@ repos:
       - id: end-of-file-fixer
         exclude: (test_csv_processing_blanks.csv|test.txt)
       - id: check-ast
+
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.10
+    rev: v0.14.13
     hooks:
       # Run the linter.
       - id: ruff-check
         args: [ --fix ]
       # Run the formatter.
       - id: ruff-format
+
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.9.22
+    rev: 0.9.25
     hooks:
-      # Run the pip compile
-      - id: pip-compile
-        name: pip-compile requirements.txt
-        files: pyproject.toml
-        args: [ pyproject.toml, --resolver=backtracking, --upgrade, -q,
-                -o, requirements.txt ]
+      # Ensure the lockfile is up-to-date with pyproject.toml
+      - id: uv-lock

{hdx_python_utilities-4.0.2 → hdx_python_utilities-4.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hdx-python-utilities
-Version: 4.0.2
+Version: 4.0.4
 Summary: HDX Python Utilities for streaming tabular data, date and time handling and other helpful functions
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-utilities
 Author-email: Michael Rans <rans@email.com>
@@ -18,11 +18,11 @@ Classifier: Operating System :: Unix
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.10
 Requires-Dist: frictionless>=5.18.0
@@ -36,11 +36,10 @@ Requires-Dist: ratelimit
 Requires-Dist: requests-file
 Requires-Dist: ruamel-yaml
 Requires-Dist: tableschema-to-template>=0.0.13
+Requires-Dist: typing-extensions
 Requires-Dist: xlrd>=2.0.1
 Requires-Dist: xlsx2csv
 Requires-Dist: xlwt>=1.3.0
-Provides-Extra: dev
-Requires-Dist: pre-commit; extra == 'dev'
 Provides-Extra: diff
 Requires-Dist: cydifflib; extra == 'diff'
 Provides-Extra: docs
@@ -50,10 +49,6 @@ Requires-Dist: email-validator; extra == 'email'
 Provides-Extra: html
 Requires-Dist: beautifulsoup4; extra == 'html'
 Requires-Dist: html5lib; extra == 'html'
-Provides-Extra: test
-Requires-Dist: pytest; extra == 'test'
-Requires-Dist: pytest-cov; extra == 'test'
-Requires-Dist: pytest-loguru; extra == 'test'
 Description-Content-Type: text/markdown
 
 [![Build Status](https://github.com/OCHA-DAP/hdx-python-utilities/actions/workflows/run-python-tests.yaml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-utilities/actions/workflows/run-python-tests.yaml)
@@ -76,6 +71,7 @@ Note that these are not specific to HDX.
 1. Easy logging setup and error logging
 1. State utility
 1. Path utilities
+1. URL utilities
 1. Text processing
 1. Stable file hashing
 1. Matching utilities

{hdx_python_utilities-4.0.2 → hdx_python_utilities-4.0.4}/README.md

@@ -18,6 +18,7 @@ Note that these are not specific to HDX.
 1. Easy logging setup and error logging
 1. State utility
 1. Path utilities
+1. URL utilities
 1. Text processing
 1. Stable file hashing
 1. Matching utilities

{hdx_python_utilities-4.0.2 → hdx_python_utilities-4.0.4}/documentation/index.md

@@ -17,6 +17,7 @@ Python developers. Note that these are not specific to HDX.
 1. [Easy logging setup and error logging](#logging)
 1. [State utility](#state-utility)
 1. [Path utilities](#path-utilities)
+1. [URL utilities](#path-utilities)
 1. [Text processing](#text-processing)
 1. [Stable file hashing](#stable-file-hashing)
 1. [Matching utilities](#matching-utilities)
@@ -33,6 +34,9 @@ The code for the library is [here](https://github.com/OCHA-DAP/hdx-python-utilit
 The library has detailed API documentation which can be found in the menu at the top.
 
 ## Breaking Changes
+From 4.0.4, get_filename_from_url, get_filename_extension_from_url, get_path_for_url,
+get_url_for_get, get_url_params_for_post moved to module url
+
 From 4.0.1, Library is Path aware. The following methods return Path not str:
 download_file, get_path_for_url, stream_path, get_temp_dir, script_dir_plus_file,
 script_dir. The context managers in hdx.utilities.path where they yield a path
@@ -829,15 +833,6 @@ Get current directory of script with filename appended
 
     path = script_dir_plus_file("myfile.txt", ANY_PYTHON_OBJECT_IN_SCRIPT)
 
-Get filename or (filename, extension) from url
-
-    url = "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-utilities/master/tests/fixtures/test_data.csv"
-    filename = get_filename_from_url(fixtureurl)
-    assert filename == "test_data.csv"
-    filename, extension = get_filename_extension_from_url(fixtureurl)
-    assert filename == "test_data"
-    assert extension == ".csv"
-
 Gets temporary directory from environment variable `TEMP_DIR` and falls back to
 the temporary folder created by the os function `gettempdir`.
 
@@ -890,6 +885,34 @@ The batch code can be passed into `wheretostart_tempdir_batch` in the `batch`
 parameter. If not given, the batch code is generated. The folder to use will be
 a generated temporary folder unless `tempdir` is given.
 
+## URL utilities
+
+Examples:
+
+Get filename or (filename, extension) or path from url
+
+    url = "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-utilities/master/tests/fixtures/test_data.csv"
+    filename = get_filename_from_url(fixtureurl)
+    assert filename == "test_data.csv"
+    filename, extension = get_filename_extension_from_url(fixtureurl)
+    assert filename == "test_data"
+    assert extension == ".csv"
+    # Get unique filename from url and join to provided folder or temporary folder
+    # if no folder supplied
+    path = get_path_for_url(url, folder)
+
+Conversion between POST dictionary and GET parameters and the reverse:
+
+    # Build get url from url and dictionary of parameters
+    get_url_for_get("http://www.lala.com/hdfa?a=3&b=4",
+                             OrderedDict([("c", "e"), ("d", "f")]))
+    # == "http://www.lala.com/hdfa?a=3&b=4&c=e&d=f"
+
+    # Extract url and dictionary of parameters from get url
+    get_url_params_for_post("http://www.lala.com/hdfa?a=3&b=4",
+                                     OrderedDict([("c", "e"), ("d", "f")]))
+
+
 ## Text processing
 
 Examples:

hdx_python_utilities-4.0.4/pyproject.toml

@@ -0,0 +1,159 @@
+#########################
+# Project Configuration #
+#########################
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "hdx-python-utilities"
+description = "HDX Python Utilities for streaming tabular data, date and time handling and other helpful functions"
+authors = [{name = "Michael Rans", email = "rans@email.com"}]
+license = {text = "MIT"}
+keywords = ["HDX", "utilities", "library", "streaming", "tabular data", "datetime", "date", "time", "timezone", "dict",
+    "list", "json", "yaml"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Natural Language :: English",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: Unix",
+    "Operating System :: MacOS",
+    "Operating System :: Microsoft :: Windows",
+]
+readme = "README.md"
+dynamic = ["version"]
+requires-python = ">=3.10"
+
+# Extras for frictionless[excel,json] added explicitly
+# for conda-forge compatibility
+dependencies = [
+    "frictionless>=5.18.0",
+    # frictionless[excel]
+    "openpyxl>=3.1.2",
+    "tableschema-to-template>=0.0.13",
+    "xlrd>=2.0.1",
+    "xlwt>=1.3.0",
+    # frictionless[json]
+    "ijson>=3.2.3",
+    "jsonlines>=4.0.0",
+    # /end frictionless extras
+    "typing_extensions",
+    "loguru",
+    "pyphonetics",
+    "python-dateutil>=2.9.0, <2.9.1",
+    "ratelimit",
+    "requests-file",
+    "ruamel.yaml",
+    "xlsx2csv",
+]
+
+[project.optional-dependencies]
+html = ["beautifulsoup4", "html5lib"]
+diff = ["cydifflib"]
+email = ["email_validator"]
+docs = ["mkapi"]
+
+[dependency-groups]
+dev = [
+    "pytest",
+    "pytest-cov",
+    "pytest-loguru",
+    "pre-commit",
+    "ruff==0.14.13",
+]
+
+[project.urls]
+Homepage = "https://github.com/OCHA-DAP/hdx-python-utilities"
+
+# ----------------------------------------------------------------------------
+# Hatchling (Build & Versioning)
+# ----------------------------------------------------------------------------
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.version.raw-options]
+local_scheme = "no-local-version"
+version_scheme = "python-simplified-semver"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/hdx/utilities/_version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/hdx"]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+# ----------------------------------------------------------------------------
+# Ruff (Linting & Formatting)
+# ----------------------------------------------------------------------------
+
+[tool.ruff]
+target-version = "py310"
+src = ["src"]
+exclude = ["_version.py"]
+
+[tool.ruff.lint]
+# Defaults are E (pycodestyle) and F (pyflakes). We extend them:
+extend-select = [
+  "I",   # isort
+  "UP",  # pyupgrade
+]
+ignore = [
+    "E501", # Line too long
+]
+
+[tool.ruff.lint.isort]
+known-local-folder = ["hdx.utilities"]
+
+# ----------------------------------------------------------------------------
+# Pytest (Testing)
+# ----------------------------------------------------------------------------
+
+[tool.pytest.ini_options]
+pythonpath = "src"
+log_cli = true
+addopts = """
+    --color=yes
+    --rootdir=.
+    --junitxml=test-results.xml
+    --cov
+    --no-cov-on-fail
+    --cov-report=lcov
+    --cov-report=term-missing
+"""
+
+# ----------------------------------------------------------------------------
+# Coverage (Reporting)
+# ----------------------------------------------------------------------------
+
+[tool.coverage.run]
+source = ["src"]
+omit = ["*/_version.py"]
+
+[tool.coverage.report]
+exclude_also = [
+    "from ._version",
+    "def __repr__",
+    "if self.debug:",
+    "if settings.DEBUG",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if 0:",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "@(abc\\.)?abstractmethod",
+]

{hdx_python_utilities-4.0.2 → hdx_python_utilities-4.0.4}/src/hdx/utilities/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '4.0.2'
-__version_tuple__ = version_tuple = (4, 0, 2)
+__version__ = version = '4.0.4'
+__version_tuple__ = version_tuple = (4, 0, 4)
 
 __commit_id__ = commit_id = None

{hdx_python_utilities-4.0.2 → hdx_python_utilities-4.0.4}/src/hdx/utilities/base_downloader.py

@@ -3,6 +3,8 @@ from collections.abc import Iterator, Sequence
 from pathlib import Path
 from typing import Any
 
+from typing_extensions import Self
+
 
 class DownloadError(Exception):
     pass
@@ -12,7 +14,7 @@ class BaseDownload(ABC):
     """Base download class with various download operations that subclasses
     should implement."""
 
-    def __enter__(self) -> "BaseDownload":
+    def __enter__(self) -> Self:
         """Allow usage of with.
 
         Returns:

{hdx_python_utilities-4.0.2 → hdx_python_utilities-4.0.4}/src/hdx/utilities/downloader.py

@@ -4,11 +4,10 @@ import hashlib
 import logging
 from collections.abc import Callable, Iterator, Sequence
 from copy import deepcopy
-from os import remove
-from os.path import exists, isfile, split, splitext
+from os.path import exists, isfile
 from pathlib import Path
 from typing import Any
-from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+from urllib.parse import urlsplit, urlunsplit
 
 import requests
 from frictionless import FrictionlessException
@@ -21,8 +20,8 @@ from xlsx2csv import Xlsx2csv
 
 from hdx.utilities.base_downloader import BaseDownload, DownloadError
 from hdx.utilities.frictionless_wrapper import get_frictionless_tableresource
-from hdx.utilities.path import get_filename_from_url, get_temp_dir
 from hdx.utilities.session import get_session
+from hdx.utilities.url import get_path_for_url, get_url_for_get, get_url_params_for_post
 
 logger = logging.getLogger(__name__)
 
@@ -130,56 +129,8 @@ class Download(BaseDownload):
         """
         self.close()
 
-    @staticmethod
-    def get_path_for_url(
-        url: str,
-        folder: Path | str | None = None,
-        filename: str | None = None,
-        path: Path | str | None = None,
-        overwrite: bool = False,
-        keep: bool = False,
-    ) -> Path:
-        """Get filename from url and join to provided folder or temporary
-        folder if no folder supplied, ensuring uniqueness.
-
-        Args:
-            url: URL to download
-            folder: Folder to download it to. Defaults to None (temporary folder).
-            filename: Filename to use for downloaded file. Defaults to None (derive from the url).
-            path: Full path to use for downloaded file. Defaults to None (use folder and filename).
-            overwrite: Whether to overwrite existing file. Defaults to False.
-            keep: Whether to keep already downloaded file. Defaults to False.
-
-        Returns:
-            Path of downloaded file
-        """
-        if path:
-            if folder or filename:
-                raise DownloadError(
-                    "Cannot use folder or filename and path arguments together!"
-                )
-            folder, filename = split(path)
-        if not filename:
-            filename = get_filename_from_url(url)
-        filename, extension = splitext(filename)
-        if not folder:
-            folder = get_temp_dir()
-        folder = Path(folder)
-        path = folder / f"{filename}{extension}"
-        if overwrite:
-            try:
-                remove(path)
-            except OSError:
-                pass
-        elif not keep:
-            count = 0
-            while exists(path):
-                count += 1
-                path = folder / f"{filename}{count}{extension}"
-        return path
-
     def get_full_url(self, url: str) -> str:
-        """Get full url including any additional parameters.
+        """Get full url including any additional parameters added to the session.
 
         Args:
             url: URL for which to get full url
@@ -191,46 +142,6 @@ class Download(BaseDownload):
         preparedrequest = self.session.prepare_request(request)
         return preparedrequest.url
 
-    @staticmethod
-    def get_url_for_get(url: str, parameters: dict | None = None) -> str:
-        """Get full url for GET request including parameters.
-
-        Args:
-            url: URL to download
-            parameters: Parameters to pass. Defaults to None.
-
-        Returns:
-            Full url
-        """
-        spliturl = urlsplit(url)
-        getparams = dict(parse_qsl(spliturl.query))
-        if parameters is not None:
-            getparams.update(parameters)
-        spliturl = spliturl._replace(query=urlencode(getparams))
-        return urlunsplit(spliturl)
-
-    @staticmethod
-    def get_url_params_for_post(
-        url: str, parameters: dict | None = None
-    ) -> tuple[str, dict]:
-        """Get full url for POST request and all parameters including any in
-        the url.
-
-        Args:
-            url: URL to download
-            parameters: Parameters to pass. Defaults to None.
-
-        Returns:
-            (Full url, parameters)
-        """
-        spliturl = urlsplit(url)
-        getparams = dict(parse_qsl(spliturl.query))
-        if parameters is not None:
-            getparams.update(parameters)
-        spliturl = spliturl._replace(query="")
-        full_url = urlunsplit(spliturl)
-        return full_url, getparams
-
     @staticmethod
     def hxl_row(
         headers: Sequence[str],
@@ -291,7 +202,7 @@ class Download(BaseDownload):
                     spliturl = spliturl._replace(scheme="https")
                     url = urlunsplit(spliturl)
             if post:
-                full_url, parameters = self.get_url_params_for_post(url, parameters)
+                full_url, parameters = get_url_params_for_post(url, parameters)
                 if json_string:
                     self.response = self.session.post(
                         full_url,
@@ -310,7 +221,7 @@ class Download(BaseDownload):
                     )
             else:
                 self.response = self.session.get(
-                    self.get_url_for_get(url, parameters),
+                    get_url_for_get(url, parameters),
                     stream=stream,
                     timeout=timeout,
                     headers=headers,
@@ -408,7 +319,10 @@ class Download(BaseDownload):
         Returns:
             Path of downloaded file
         """
-        path = self.get_path_for_url(url, folder, filename, path, overwrite, keep)
+        try:
+            path = get_path_for_url(url, folder, filename, path, overwrite, keep)
+        except ValueError as ex:
+            raise DownloadError(ex) from ex
         if keep and exists(path):
             return path
         return self.stream_path(
@@ -446,7 +360,10 @@ class Download(BaseDownload):
         path = kwargs.get("path")
         overwrite = kwargs.get("overwrite", False)
         keep = kwargs.get("keep", False)
-        path = self.get_path_for_url(url, folder, filename, path, overwrite, keep)
+        try:
+            path = get_path_for_url(url, folder, filename, path, overwrite, keep)
+        except ValueError as ex:
+            raise DownloadError(ex) from ex
         if keep and exists(path):
             return path
         self.setup(