hdx-python-utilities 4.0.4__tar.gz → 4.0.6__tar.gz

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hdx-python-utilities
-Version: 4.0.4
+Version: 4.0.6
 Summary: HDX Python Utilities for streaming tabular data, date and time handling and other helpful functions
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-utilities
 Author-email: Michael Rans <rans@email.com>
@@ -83,3 +83,78 @@ For more information, please read the [documentation](https://hdx-python-utiliti
 
 This library is part of the [Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have
 humanitarian related data, please upload your datasets to HDX.
+
+# Development
+
+## Environment
+
+Development is currently done using Python 3.13. The environment can be created with:
+
+```shell
+    uv sync
+```
+
+This creates a .venv folder with the versions specified in the project's uv.lock file.
+
+### Pre-commit
+
+pre-commit will be installed when syncing uv. It is run every time you make a git
+commit if you call it like this:
+
+```shell
+    pre-commit install
+```
+
+With pre-commit, all code is formatted according to
+[ruff](https://docs.astral.sh/ruff/) guidelines.
+
+To check if your changes pass pre-commit without committing, run:
+
+```shell
+    pre-commit run --all-files
+```
+
+## Packages
+
+[uv](https://github.com/astral-sh/uv) is used for package management.  If
+you’ve introduced a new package to the source code (i.e. anywhere in `src/`),
+please add it to the `project.dependencies` section of `pyproject.toml` with
+any known version constraints.
+
+To add packages required only for testing, add them to the
+`[dependency-groups]`.
+
+Any changes to the dependencies will be automatically reflected in
+`uv.lock` with `pre-commit`, but you can re-generate the files without committing by
+executing:
+
+```shell
+    uv lock --upgrade
+```
+
+## Project
+
+[uv](https://github.com/astral-sh/uv) is used for project management. The project can be
+built using:
+
+```shell
+    uv build
+```
+
+Linting and syntax checking can be run with:
+
+```shell
+    uv run ruff check
+```
+
+To run the tests and view coverage, execute:
+
+```shell
+    uv run pytest
+```
+
+## Documentation
+
+The documentation, including API documentation, is generated using ReadtheDocs and
+MkDocs with Material. As you change the source code, remember to update the
+documentation at `documentation/index.md`.

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/README.md

@@ -30,3 +30,78 @@ For more information, please read the [documentation](https://hdx-python-utiliti
 
 This library is part of the [Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have
 humanitarian related data, please upload your datasets to HDX.
+
+# Development
+
+## Environment
+
+Development is currently done using Python 3.13. The environment can be created with:
+
+```shell
+    uv sync
+```
+
+This creates a .venv folder with the versions specified in the project's uv.lock file.
+
+### Pre-commit
+
+pre-commit will be installed when syncing uv. It is run every time you make a git
+commit if you call it like this:
+
+```shell
+    pre-commit install
+```
+
+With pre-commit, all code is formatted according to
+[ruff](https://docs.astral.sh/ruff/) guidelines.
+
+To check if your changes pass pre-commit without committing, run:
+
+```shell
+    pre-commit run --all-files
+```
+
+## Packages
+
+[uv](https://github.com/astral-sh/uv) is used for package management.  If
+you’ve introduced a new package to the source code (i.e. anywhere in `src/`),
+please add it to the `project.dependencies` section of `pyproject.toml` with
+any known version constraints.
+
+To add packages required only for testing, add them to the
+`[dependency-groups]`.
+
+Any changes to the dependencies will be automatically reflected in
+`uv.lock` with `pre-commit`, but you can re-generate the files without committing by
+executing:
+
+```shell
+    uv lock --upgrade
+```
+
+## Project
+
+[uv](https://github.com/astral-sh/uv) is used for project management. The project can be
+built using:
+
+```shell
+    uv build
+```
+
+Linting and syntax checking can be run with:
+
+```shell
+    uv run ruff check
+```
+
+To run the tests and view coverage, execute:
+
+```shell
+    uv run pytest
+```
+
+## Documentation
+
+The documentation, including API documentation, is generated using ReadtheDocs and
+MkDocs with Material. As you change the source code, remember to update the
+documentation at `documentation/index.md`.

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/src/hdx/utilities/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '4.0.4'
-__version_tuple__ = version_tuple = (4, 0, 4)
+__version__ = version = '4.0.6'
+__version_tuple__ = version_tuple = (4, 0, 6)
 
 __commit_id__ = commit_id = None

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/src/hdx/utilities/base_downloader.py

@@ -35,7 +35,7 @@ class BaseDownload(ABC):
         """
 
     @abstractmethod
-    def download_file(self, url: str, *args: Any, **kwargs: Any) -> Path:
+    def download_file(self, url: Path | str, *args: Any, **kwargs: Any) -> Path:
         """Download file from url.
 
         Args:
@@ -48,7 +48,7 @@ class BaseDownload(ABC):
         """
 
     @abstractmethod
-    def download_text(self, url: str, *args: Any, **kwargs: Any) -> str:
+    def download_text(self, url: Path | str, *args: Any, **kwargs: Any) -> str:
         """Download text from url.
 
         Args:
@@ -61,7 +61,7 @@ class BaseDownload(ABC):
         """
 
     @abstractmethod
-    def download_yaml(self, url: str, *args: Any, **kwargs: Any) -> Any:
+    def download_yaml(self, url: Path | str, *args: Any, **kwargs: Any) -> Any:
         """Download YAML from url.
 
         Args:
@@ -74,7 +74,7 @@ class BaseDownload(ABC):
         """
 
     @abstractmethod
-    def download_json(self, url: str, *args: Any, **kwargs: Any) -> Any:
+    def download_json(self, url: Path | str, *args: Any, **kwargs: Any) -> Any:
         """Download JSON from url.
 
         Args:
@@ -89,7 +89,7 @@ class BaseDownload(ABC):
     @abstractmethod
     def get_tabular_rows(
         self,
-        url: str | Sequence[str],
+        url: Path | str | Sequence[str],
         has_hxl: bool = False,
         headers: int | Sequence[int] | Sequence[str] = 1,
         dict_form: bool = False,

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/src/hdx/utilities/dictandlist.py

@@ -374,7 +374,7 @@ def avg_dicts(
 
 
 def read_list_from_csv(
-    url: str,
+    url: Path | str,
     headers: int | Sequence[int] | Sequence[str] | None = None,
     dict_form: bool = False,
     **kwargs: Any,

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/src/hdx/utilities/downloader.py

@@ -129,7 +129,7 @@ class Download(BaseDownload):
         """
         self.close()
 
-    def get_full_url(self, url: str) -> str:
+    def get_full_url(self, url: Path | str) -> str:
         """Get full url including any additional parameters added to the session.
 
         Args:
@@ -193,14 +193,16 @@ class Download(BaseDownload):
         self.close_response()
         self.response = None
         try:
-            url = str(url)
-            spliturl = urlsplit(url)
-            if not spliturl.scheme:
-                if isfile(url):
-                    url = Path(url).resolve().as_uri()
-                else:
-                    spliturl = spliturl._replace(scheme="https")
-                    url = urlunsplit(spliturl)
+            if isinstance(url, Path):
+                url = url.resolve().as_uri()
+            else:
+                spliturl = urlsplit(url)
+                if not spliturl.scheme:
+                    if isfile(url):
+                        url = Path(url).resolve().as_uri()
+                    else:
+                        spliturl = spliturl._replace(scheme="https")
+                        url = urlunsplit(spliturl)
             if post:
                 full_url, parameters = get_url_params_for_post(url, parameters)
                 if json_string:
@@ -671,7 +673,7 @@ class Download(BaseDownload):
 
     def get_tabular_rows(
         self,
-        url: str | Sequence[str],
+        url: Path | str | Sequence[str],
         has_hxl: bool = False,
         headers: int | Sequence[int] | Sequence[str] = 1,
         dict_form: bool = False,
@@ -778,7 +780,7 @@ class Download(BaseDownload):
 
     def get_tabular_rows_as_list(
         self,
-        url: str | Sequence[str],
+        url: Path | str | Sequence[str],
         has_hxl: bool = False,
         headers: int | Sequence[int] | Sequence[str] = 1,
         include_headers: bool = True,
@@ -854,7 +856,7 @@ class Download(BaseDownload):
 
     def get_tabular_rows_as_dict(
         self,
-        url: str | Sequence[str],
+        url: Path | str | Sequence[str],
         has_hxl: bool = False,
         headers: int | Sequence[int] | Sequence[str] = 1,
         ignore_blank_rows: bool = True,
@@ -928,7 +930,7 @@ class Download(BaseDownload):
 
     def download_tabular_key_value(
         self,
-        url: str | Sequence[str],
+        url: Path | str | Sequence[str],
         has_hxl: bool = False,
         headers: int | Sequence[int] | Sequence[str] = 1,
         include_headers: bool = True,
@@ -1008,7 +1010,7 @@ class Download(BaseDownload):
 
     def download_tabular_rows_as_dicts(
         self,
-        url: str | Sequence[str],
+        url: Path | str | Sequence[str],
         has_hxl: bool = False,
         headers: int | Sequence[int] | Sequence[str] = 1,
         keycolumn: int = 1,
@@ -1092,7 +1094,7 @@ class Download(BaseDownload):
 
     def download_tabular_cols_as_dicts(
         self,
-        url: str | Sequence[str],
+        url: Path | str | Sequence[str],
         has_hxl: bool = False,
         headers: int | Sequence[int] | Sequence[str] = 1,
         keycolumn: int = 1,

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/src/hdx/utilities/frictionless_wrapper.py

@@ -1,5 +1,6 @@
 """Frictionless wrapper"""
 
+from pathlib import Path
 from typing import Any
 
 import requests
@@ -119,7 +120,7 @@ def get_frictionless_dialect(
 
 
 def get_frictionless_tableresource(
-    url: str | None = None,
+    url: Path | str | None = None,
     ignore_blank_rows: bool = True,
     infer_types: bool = False,
     session: requests.Session | None = None,

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/src/hdx/utilities/retriever.py

@@ -120,7 +120,7 @@ class Retrieve(BaseDownload):
 
     def get_filename(
         self,
-        url: str,
+        url: Path | str,
         filename: str | None = None,
         possible_extensions: tuple[str, ...] = tuple(),
         **kwargs: Any,
@@ -180,7 +180,7 @@ class Retrieve(BaseDownload):
 
     def download_file(
         self,
-        url: str,
+        url: Path | str,
         filename: str | None = None,
         logstr: str | None = None,
         fallback: bool = False,
@@ -231,7 +231,7 @@ class Retrieve(BaseDownload):
 
     def download_text(
         self,
-        url: str,
+        url: Path | str,
         filename: str | None = None,
         logstr: str | None = None,
         fallback: bool = False,
@@ -282,7 +282,7 @@ class Retrieve(BaseDownload):
 
     def download_yaml(
         self,
-        url: str,
+        url: Path | str,
         filename: str | None = None,
         logstr: str | None = None,
         fallback: bool = False,
@@ -333,7 +333,7 @@ class Retrieve(BaseDownload):
 
     def download_json(
         self,
-        url: str,
+        url: Path | str,
         filename: str | None = None,
         logstr: str | None = None,
         fallback: bool = False,
@@ -384,7 +384,7 @@ class Retrieve(BaseDownload):
 
     def get_tabular_rows(
         self,
-        url: str | Sequence[str],
+        url: Path | str | Sequence[str],
         has_hxl: bool = False,
         headers: int | Sequence[int] | Sequence[str] = 1,
         dict_form: bool = False,

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/src/hdx/utilities/saver.py

@@ -281,10 +281,13 @@ def save_iterable(
     row_function: Callable[[dict], dict | None] | None = None,
     no_empty: bool = True,
 ) -> list:
-    """Save an iterable of rows in dict or list form to a csv. (The headers
+    """Save an iterable of rows in dict or list form to a csv. The headers
     argument is either a row number (rows start counting at 1), or the actual
-    headers defined as a list of strings. If not set, all rows will be treated
-    as containing values.)
+    headers defined as a list of strings, the order of which defines the column order.
+    Columns not named in that list of strings will be dropped. If headers is not set,
+    all rows will be treated as containing values. The columns argument defines which
+    columns will be output. It can be used in conjunction with an integer headers with
+    columns selecting the columns and hedders selecting the starting row.
 
     Args:
         filepath: Path to write to
@@ -327,32 +330,23 @@ def save_iterable(
     if isinstance(row, dict):
         has_header = True
         row = row_function(row)
+        while row is None:
+            row = next(rows)
+            row = row_function(row)
         if columns:
-            if row is not None:
-                newrow = {}
-                for column in columns:
-                    if column in row:
-                        newrow[column] = row[column]
-                newrows.append(newrow)
-            for row in rows:
-                row = row_function(row)
-                if row is None:
-                    continue
-                newrow = {}
-                for column in columns:
-                    if column in row:
-                        newrow[column] = row[column]
-                newrows.append(newrow)
+            row = {k: row[k] for k in columns if k in row}
+            newrows.append(row)
             if headers is None:
                 headers = columns
         else:
-            if row is not None:
-                newrows.append(row)
-            for row in rows:
-                row = row_function(row)
-                if row is None:
-                    continue
-                newrows.append(row)
+            if headers and isinstance(headers, list):
+                row = {k: row.get(k) for k in headers}
+            newrows.append(row)
+        for row in rows:
+            row = row_function(row)
+            if row is None:
+                continue
+            newrows.append(row)
     else:
         if headers is None:
             headers = 1

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/src/hdx/utilities/url.py

@@ -62,7 +62,7 @@ def get_filename_from_url(
 
 
 def get_path_for_url(
-    url: str,
+    url: Path | str,
     folder: Path | str | None = None,
     filename: str | None = None,
     path: Path | str | None = None,

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/tests/hdx/utilities/test_downloader.py

@@ -48,6 +48,10 @@ class TestDownloader:
     def fixturefile(self, downloaderfolder):
         return downloaderfolder / "extra_params_tree.yaml"
 
+    @pytest.fixture(scope="class")
+    def fixturepath(self, downloaderfolder):
+        return downloaderfolder / "test_csv_processing.csv"
+
     @pytest.fixture(scope="class")
     def fixtureurlexcel(self):
         return "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-utilities/main/tests/fixtures/downloader/test_data.xlsx"
@@ -595,7 +599,9 @@ class TestDownloader:
                 {"la1": "gas", "ha1": "2", "ba1": "6.5", "ma1": "'n/a'"},
             ]
 
-    def test_get_tabular_rows(self, fixtureprocessurl, fixtureprocessurlblank):
+    def test_get_tabular_rows(
+        self, fixturepath, fixtureprocessurl, fixtureprocessurlblank
+    ):
         with Download() as downloader:
             expected = [
                 ["la1", "ha1", "ba1", "ma1"],
@@ -604,6 +610,9 @@ class TestDownloader:
                 ["gas", "2", "6.5", "'n/a'"],
             ]
             expected_headers = expected[0]
+            headers, iterator = downloader.get_tabular_rows(fixturepath)
+            assert headers == expected_headers
+            assert list(iterator) == expected[1:]
             headers, iterator = downloader.get_tabular_rows(fixtureprocessurl)
             assert headers == expected_headers
             assert list(iterator) == expected[1:]
@@ -1271,11 +1280,9 @@ class TestDownloader:
                 ["AFG", "Afghanistan", 38041754],
             ]
 
-    def test_download_tabular_rows_as_dicts(self, fixtureprocessurl):
+    def test_download_tabular_rows_as_dicts(self, fixturepath, fixtureprocessurl):
         with Download() as downloader:
-            result = downloader.download_tabular_rows_as_dicts(
-                fixtureprocessurl, headers=2
-            )
+            result = downloader.download_tabular_rows_as_dicts(fixturepath, headers=2)
             assert result == {
                 "coal": {
                     "header2": "3",

{hdx_python_utilities-4.0.4 → hdx_python_utilities-4.0.6}/tests/hdx/utilities/test_saver.py

@@ -353,6 +353,12 @@ class TestLoader:
                 ["5", "6", "4"],
                 ["8", "9", "7"],
             ]
+            save_iterable(
+                str(filepath), list_of_dicts, headers=2, columns=["h2", "h3", "h1"]
+            )
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [["2", "3", "1"], ["5", "6", "4"], ["8", "9", "7"]]
 
             xlfilepath = filepath.with_suffix(".xlsx")
             rows = save_iterable(
@@ -465,6 +471,24 @@ class TestLoader:
                 ["4", "5", "6", "b"],
                 ["7", "8", "9", "c"],
             ]
+            save_iterable(filepath, list_of_dicts, headers=["h2", "h1", "h4", "h3"])
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [
+                ["h2", "h1", "h4", "h3"],
+                ["2", "1", "a", "3"],
+                ["5", "4", "b", "6"],
+                ["8", "7", "c", "9"],
+            ]
+            save_iterable(filepath, list_of_dicts, headers=["h2", "h1", "h4"])
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [
+                ["h2", "h1", "h4"],
+                ["2", "1", "a"],
+                ["5", "4", "b"],
+                ["8", "7", "c"],
+            ]
             save_iterable(filepath, list_of_dicts)
             newll = read_list_from_csv(filepath)
             remove(filepath)