hdx-python-utilities 3.9.3__tar.gz → 3.9.4__tar.gz

{hdx_python_utilities-3.9.3 → hdx_python_utilities-3.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hdx-python-utilities
-Version: 3.9.3
+Version: 3.9.4
 Summary: HDX Python Utilities for streaming tabular data, date and time handling and other helpful functions
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-utilities
 Author-email: Michael Rans <rans@email.com>

{hdx_python_utilities-3.9.3 → hdx_python_utilities-3.9.4}/documentation/index.md

@@ -386,6 +386,24 @@ Examples:
     # sorting the keys
     save_json(mydict, "mypath.json", pretty=False, sortkeys=False)
 
+## Saving iterable to file
+
+`save_iterable` is a utility to save an iterable of dict, list or tuple to a file such
+as a csv or Excel.
+
+    # Save iterable to file
+    l = [[1, 2, 3, "a"],
+         [4, 5, 6, "b"],
+         [7, 8, 9, "c"]]
+    save_iterable(filepath, l, headers=["h1", "h2", "h3", "h4"])
+    newll = read_list_from_csv(filepath)
+    newld = read_list_from_csv(filepath, headers=1, dict_form=True)
+    assert newll == [["h1", "h2", "h3", "h4"], ["1", "2", "3", "a"], ["4", "5", "6", "b"], ["7", "8", "9", "c"]]
+    assert newld == [{"h1": "1", "h2": "2", "h4": "a", "h3": "3"},
+                    {"h1": "4", "h2": "5", "h4": "b", "h3": "6"},
+                    {"h1": "7", "h2": "8", "h4": "c", "h3": "9"}]
+    save_iterable(xlfilepath, list_of_lists, headers=["h1", "h2", "h3", "h4"], format="xlsx")
+
 ## Loading and saving HXLated csv and/or JSON
 
 `save_hxlated_output` is a utility to save HXLated output (currently JSON and/or csv are
@@ -555,6 +573,10 @@ Examples:
     d2 = {1: 2, 2: 2.0, 3: 5, 4: 4, 7: 3}
     assert avg_dicts(d1, d2) == {1: 1.5, 2: 1.5, 3: 4, 4: 4}
 
+    ## Convert command line arguments to dictionary
+    args = "a=1,big=hello,1=3"
+    assert args_to_dict(args) == {"a": "1", "big": "hello", "1": "3"}
+
     # Read and write lists to csv
     l = [[1, 2, 3, "a"],
          [4, 5, 6, "b"],
@@ -567,9 +589,9 @@ Examples:
                     {"h1": "4", "h2": "5", "h4": "b", "h3": "6"},
                     {"h1": "7", "h2": "8", "h4": "c", "h3": "9"}]
 
-    ## Convert command line arguments to dictionary
-    args = "a=1,big=hello,1=3"
-    assert args_to_dict(args) == {"a": "1", "big": "hello", "1": "3"}
+Note that `get_tabular_rows` in the `Download` class is a more powerful and streaming
+version of `read_list_from_csv` and `save_iterable` is a generally more efficient
+version of`write_list_to_csv` as it uses iteration instead of list manipulation.
 
 ## HTML utilities
 

{hdx_python_utilities-3.9.3 → hdx_python_utilities-3.9.4}/requirements.txt

@@ -4,7 +4,7 @@ annotated-types==0.7.0
     # via pydantic
 astdoc==1.3.2
     # via mkapi
-attrs==25.3.0
+attrs==25.4.0
     # via
     #   frictionless
     #   jsonlines
@@ -14,9 +14,9 @@ babel==2.17.0
     # via mkdocs-material
 backrefs==5.9
     # via mkdocs-material
-beautifulsoup4==4.13.5
+beautifulsoup4==4.14.2
     # via hdx-python-utilities (pyproject.toml)
-certifi==2025.8.3
+certifi==2025.10.5
     # via requests
 cfgv==3.4.0
     # via pre-commit
@@ -24,10 +24,9 @@ chardet==5.2.0
     # via frictionless
 charset-normalizer==3.4.3
     # via requests
-click==8.2.1
+click==8.3.0
     # via
     #   mkdocs
-    #   mkdocs-material
     #   typer
 colorama==0.4.6
     # via mkdocs-material
@@ -43,7 +42,7 @@ email-validator==2.3.0
     # via hdx-python-utilities (pyproject.toml)
 et-xmlfile==2.0.0
     # via openpyxl
-filelock==3.19.1
+filelock==3.20.0
     # via virtualenv
 frictionless==5.18.1
     # via hdx-python-utilities (pyproject.toml)
@@ -53,7 +52,7 @@ html5lib==1.1
     # via hdx-python-utilities (pyproject.toml)
 humanize==4.13.0
     # via frictionless
-identify==2.6.14
+identify==2.6.15
     # via pre-commit
 idna==3.10
     # via
@@ -92,7 +91,7 @@ markdown-it-py==4.0.0
     # via rich
 marko==2.2.0
     # via frictionless
-markupsafe==3.0.2
+markupsafe==3.0.3
     # via
     #   jinja2
     #   mkdocs
@@ -110,7 +109,7 @@ mkdocs==1.6.1
     #   mkdocs-material
 mkdocs-get-deps==0.2.0
     # via mkdocs
-mkdocs-material==9.6.20
+mkdocs-material==9.6.21
     # via mkapi
 mkdocs-material-extensions==1.3.1
     # via mkdocs-material
@@ -128,7 +127,7 @@ pathspec==0.12.1
     # via mkdocs
 petl==1.7.17
     # via frictionless
-platformdirs==4.4.0
+platformdirs==4.5.0
     # via
     #   mkdocs-get-deps
     #   virtualenv
@@ -138,9 +137,9 @@ pluggy==1.6.0
     #   pytest-cov
 pre-commit==4.3.0
     # via hdx-python-utilities (pyproject.toml)
-pydantic==2.11.9
+pydantic==2.12.0
     # via frictionless
-pydantic-core==2.33.2
+pydantic-core==2.41.1
     # via pydantic
 pygments==2.19.2
     # via
@@ -166,7 +165,7 @@ python-dateutil==2.9.0.post0
     #   ghp-import
 python-slugify==8.0.4
     # via frictionless
-pyyaml==6.0.2
+pyyaml==6.0.3
     # via
     #   frictionless
     #   mkdocs
@@ -218,7 +217,7 @@ tabulate==0.9.0
     # via frictionless
 text-unidecode==1.3
     # via python-slugify
-typer==0.19.1
+typer==0.19.2
     # via frictionless
 typing-extensions==4.15.0
     # via
@@ -228,7 +227,7 @@ typing-extensions==4.15.0
     #   pydantic-core
     #   typer
     #   typing-inspection
-typing-inspection==0.4.1
+typing-inspection==0.4.2
     # via pydantic
 unidecode==1.4.0
     # via pyphonetics

{hdx_python_utilities-3.9.3 → hdx_python_utilities-3.9.4}/src/hdx/utilities/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.9.3'
-__version_tuple__ = version_tuple = (3, 9, 3)
+__version__ = version = '3.9.4'
+__version_tuple__ = version_tuple = (3, 9, 4)
 
 __commit_id__ = commit_id = None

{hdx_python_utilities-3.9.3 → hdx_python_utilities-3.9.4}/src/hdx/utilities/saver.py

@@ -4,7 +4,7 @@ import csv
 import json
 from collections import OrderedDict
 from os.path import join
-from typing import Any, Dict
+from typing import Any, Dict, Iterable, Optional, Union
 
 from ruamel.yaml import (
     YAML,
@@ -13,6 +13,7 @@ from ruamel.yaml import (
     add_representer,
 )
 
+from .frictionless_wrapper import get_frictionless_tableresource
 from .matching import match_template_variables
 from .typehint import ListTuple, ListTupleDict
 
@@ -268,3 +269,91 @@ def save_hxlated_output(
         output_json.close()
     if csv_file:
         csv_file.close()
+
+
+def save_iterable(
+    filepath: str,
+    rows: Iterable[ListTupleDict],
+    headers: Union[int, ListTuple[str], None] = None,
+    columns: Union[ListTuple[int], ListTuple[str], None] = None,
+    format: str = "csv",
+    encoding: Optional[str] = None,
+) -> bool:
+    """Save an iterable of rows in dict or list form to a csv. (The headers
+    argument is either a row number (rows start counting at 1), or the actual
+    headers defined as a list of strings. If not set, all rows will be treated
+    as containing values.)
+
+    Args:
+        filepath (str): Path to write to
+        rows (Iterable[ListTupleDict]): List of rows in dict or list form
+        headers (Union[int, ListTuple[str], None]): Headers to write. Defaults to None.
+        columns (Union[ListTuple[int], ListTuple[str], None]): Columns to write. Defaults to all.
+        format (str): Format to write. Defaults to csv.
+        encoding (Optional[str]): Encoding to use. Defaults to None (infer encoding).
+
+    Returns:
+        bool: Whether iterable was saved
+    """
+    rows = iter(rows)
+    try:
+        row = next(rows)
+    except StopIteration:
+        return False
+    newrows = []
+    if isinstance(row, dict):
+        has_header = True
+        if columns:
+            newrow = {}
+            for column in columns:
+                if column in row:
+                    newrow[column] = row[column]
+            newrows.append(newrow)
+            for row in rows:
+                newrow = {}
+                for column in columns:
+                    if column in row:
+                        newrow[column] = row[column]
+                newrows.append(newrow)
+            if headers is None:
+                headers = columns
+        else:
+            newrows.append(row)
+            for row in rows:
+                newrows.append(row)
+    else:
+        if headers is None:
+            headers = 1
+        if isinstance(headers, int):
+            headers_rowno = headers
+            headers = row
+            for i in range(headers_rowno - 1):
+                headers = next(rows)
+        else:
+            headers_rowno = None
+        has_header = False
+        if columns:
+            if headers_rowno is None:
+                newrow = []
+                for column in columns:
+                    newrow.append(row[column - 1])
+                newrows.append(newrow)
+            for row in rows:
+                newrow = []
+                for column in columns:
+                    newrow.append(row[column - 1])
+                newrows.append(newrow)
+        else:
+            if headers_rowno is None:
+                newrows.append(row)
+            for row in rows:
+                newrows.append(row)
+    resource = get_frictionless_tableresource(
+        data=newrows,
+        has_header=has_header,
+        headers=headers,
+        encoding=encoding,
+    )
+    resource.write(filepath, format=format, encoding=encoding)
+    resource.close()
+    return True

{hdx_python_utilities-3.9.3 → hdx_python_utilities-3.9.4}/tests/hdx/utilities/test_saver.py

@@ -2,13 +2,16 @@
 
 import json
 from collections import OrderedDict
+from os import remove
 from os.path import exists, join
 
 import pytest
 
 from hdx.utilities.compare import assert_files_same
+from hdx.utilities.dictandlist import read_list_from_csv
 from hdx.utilities.loader import load_yaml
-from hdx.utilities.saver import save_hxlated_output, save_json, save_yaml
+from hdx.utilities.path import temp_dir
+from hdx.utilities.saver import save_hxlated_output, save_iterable, save_json, save_yaml
 
 
 class TestLoader:
@@ -290,3 +293,153 @@ class TestLoader:
         assert_files_same(join(saverfolder, filename), join(output_dir, filename))
         filename = "out8.json"
         assert_files_same(join(saverfolder, filename), join(output_dir, filename))
+
+    def test_save_iterable(self):
+        list_of_tuples = [(1, 2, 3, "a"), (4, 5, 6, "b"), (7, 8, 9, "c")]
+        list_of_lists = [[1, 2, 3, "a"], [4, 5, 6, "b"], [7, 8, 9, "c"]]
+        with temp_dir(
+            "TestSaveIterable",
+            delete_on_success=True,
+            delete_on_failure=False,
+        ) as tempdir:
+            filename = "test_save_iterable_to_csv.csv"
+            filepath = join(tempdir, filename)
+            res = save_iterable(
+                filepath, list_of_lists, headers=["h1", "h2", "h3", "h4"]
+            )
+            assert res is True
+            newll = read_list_from_csv(filepath)
+            newld = read_list_from_csv(filepath, headers=1, dict_form=True)
+            remove(filepath)
+            assert newll == [
+                ["h1", "h2", "h3", "h4"],
+                ["1", "2", "3", "a"],
+                ["4", "5", "6", "b"],
+                ["7", "8", "9", "c"],
+            ]
+            assert newld == [
+                {"h1": "1", "h2": "2", "h4": "a", "h3": "3"},
+                {"h1": "4", "h2": "5", "h4": "b", "h3": "6"},
+                {"h1": "7", "h2": "8", "h4": "c", "h3": "9"},
+            ]
+            xlfilepath = filepath.replace("csv", "xlsx")
+            res = save_iterable(
+                xlfilepath,
+                list_of_lists,
+                headers=["h1", "h2", "h3", "h4"],
+                format="xlsx",
+            )
+            assert res is True
+            assert exists(xlfilepath), "File should exist"
+
+            save_iterable(filepath, list_of_tuples, headers=("h1", "h2", "h3", "h4"))
+            newll = read_list_from_csv(filepath)
+            newld = read_list_from_csv(filepath, headers=1, dict_form=True)
+            remove(filepath)
+            assert newll == [
+                ["h1", "h2", "h3", "h4"],
+                ["1", "2", "3", "a"],
+                ["4", "5", "6", "b"],
+                ["7", "8", "9", "c"],
+            ]
+            assert newld == [
+                {"h1": "1", "h2": "2", "h4": "a", "h3": "3"},
+                {"h1": "4", "h2": "5", "h4": "b", "h3": "6"},
+                {"h1": "7", "h2": "8", "h4": "c", "h3": "9"},
+            ]
+            save_iterable(filepath, list_of_lists, headers=["h1", "h2", "h3"])
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [
+                ["h1", "h2", "h3"],
+                ["1", "2", "3"],
+                ["4", "5", "6"],
+                ["7", "8", "9"],
+            ]
+            save_iterable(
+                filepath,
+                list_of_lists,
+                headers=["h1", "h3", "h4"],
+                columns=[1, 3, 4],
+            )
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [
+                ["h1", "h3", "h4"],
+                ["1", "3", "a"],
+                ["4", "6", "b"],
+                ["7", "9", "c"],
+            ]
+
+            list_of_lists = [
+                ["1", "2", "3", "a"],
+                ["4", "5", "6", "b"],
+                [7, 8, 9, "c"],
+            ]
+            save_iterable(filepath, list_of_lists)
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [
+                ["1", "2", "3", "a"],
+                ["4", "5", "6", "b"],
+                ["7", "8", "9", "c"],
+            ]
+            save_iterable(filepath, list_of_lists, headers=2)
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [
+                ["4", "5", "6", "b"],
+                ["7", "8", "9", "c"],
+            ]
+
+            list_of_dicts = [
+                {"h1": 1, "h2": 2, "h3": 3, "h4": "a"},
+                {"h1": 4, "h2": 5, "h3": 6, "h4": "b"},
+                {"h1": 7, "h2": 8, "h3": 9, "h4": "c"},
+            ]
+            save_iterable(filepath, list_of_dicts, headers=["h1", "h2", "h3", "h4"])
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [
+                ["h1", "h2", "h3", "h4"],
+                ["1", "2", "3", "a"],
+                ["4", "5", "6", "b"],
+                ["7", "8", "9", "c"],
+            ]
+            save_iterable(filepath, list_of_dicts)
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [
+                ["h1", "h2", "h3", "h4"],
+                ["1", "2", "3", "a"],
+                ["4", "5", "6", "b"],
+                ["7", "8", "9", "c"],
+            ]
+            save_iterable(filepath, list_of_dicts, headers=2)
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [
+                ["1", "2", "3", "a"],
+                ["4", "5", "6", "b"],
+                ["7", "8", "9", "c"],
+            ]
+            save_iterable(
+                filepath,
+                list_of_dicts,
+                headers=["h1", "h3", "h4"],
+                columns=["h1", "h3", "h4"],
+            )
+            newll = read_list_from_csv(filepath)
+            remove(filepath)
+            assert newll == [
+                ["h1", "h3", "h4"],
+                ["1", "3", "a"],
+                ["4", "6", "b"],
+                ["7", "9", "c"],
+            ]
+
+            with pytest.raises(ValueError):
+                read_list_from_csv(filepath, dict_form=True)
+
+            res = save_iterable(filepath, [], headers=["h1", "h3", "h4"])
+            assert res is False