hdx-python-utilities 3.7.1__tar.gz → 3.7.3__tar.gz

{hdx_python_utilities-3.7.1 → hdx_python_utilities-3.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hdx-python-utilities
-Version: 3.7.1
+Version: 3.7.3
 Summary: HDX Python Utilities for streaming tabular data, date and time handling and other helpful functions
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-utilities
 Author-email: Michael Rans <rans@email.com>
@@ -36,6 +36,7 @@ Requires-Dist: requests-file
 Requires-Dist: ruamel-yaml
 Requires-Dist: tableschema-to-template>=0.0.13
 Requires-Dist: xlrd>=2.0.1
+Requires-Dist: xlsx2csv
 Requires-Dist: xlwt>=1.3.0
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'

{hdx_python_utilities-3.7.1 → hdx_python_utilities-3.7.3}/documentation/main.md

@@ -785,6 +785,9 @@ Examples:
 
     a = "The quick brown fox jumped over the lazy dog. It was so fast!"
 
+    # Normalise text eg. to support name matching
+    assert normalise("£^*& ()+-[]<>?|\ Al DhaleZ'eÉ / الضالع,,..1234''#~~### ") == "al dhalezee 1234"
+
     # Remove whitespace and punctuation from end of string
     assert remove_end_characters('lalala,.,"') == "lalala"
     assert remove_end_characters('lalala, .\t/,"', f"{punctuation}{whitespace}" == "lalala"

{hdx_python_utilities-3.7.1 → hdx_python_utilities-3.7.3}/pyproject.toml

@@ -51,6 +51,7 @@ dependencies = [
     "ratelimit",
     "requests-file",
     "ruamel.yaml",
+    "xlsx2csv",
 ]
 dynamic = ["version"]
 

{hdx_python_utilities-3.7.1 → hdx_python_utilities-3.7.3}/requirements.txt

@@ -10,7 +10,7 @@ attrs==23.2.0
     #   referencing
 beautifulsoup4==4.12.3
     # via hdx-python-utilities (pyproject.toml)
-certifi==2024.6.2
+certifi==2024.7.4
     # via requests
 cfgv==3.4.0
     # via pre-commit
@@ -20,7 +20,7 @@ charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via typer
-coverage==7.5.4
+coverage==7.6.0
     # via pytest-cov
 distlib==0.3.8
     # via virtualenv
@@ -36,9 +36,9 @@ frictionless==5.17.0
     # via hdx-python-utilities (pyproject.toml)
 html5lib==1.1
     # via hdx-python-utilities (pyproject.toml)
-humanize==4.9.0
+humanize==4.10.0
     # via frictionless
-identify==2.5.36
+identify==2.6.0
     # via pre-commit
 idna==3.7
     # via
@@ -54,7 +54,7 @@ jinja2==3.1.4
     # via frictionless
 jsonlines==4.0.0
     # via hdx-python-utilities (pyproject.toml)
-jsonschema==4.22.0
+jsonschema==4.23.0
     # via
     #   frictionless
     #   tableschema-to-template
@@ -84,15 +84,15 @@ platformdirs==4.2.2
     # via virtualenv
 pluggy==1.5.0
     # via pytest
-pre-commit==3.7.1
+pre-commit==3.8.0
     # via hdx-python-utilities (pyproject.toml)
-pydantic==2.8.0
+pydantic==2.8.2
     # via frictionless
-pydantic-core==2.20.0
+pydantic-core==2.20.1
     # via pydantic
 pygments==2.18.0
     # via rich
-pytest==8.2.2
+pytest==8.3.2
     # via
     #   hdx-python-utilities (pyproject.toml)
     #   pytest-cov
@@ -127,7 +127,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==13.7.1
     # via typer
-rpds-py==0.18.1
+rpds-py==0.19.1
     # via
     #   jsonschema
     #   referencing
@@ -164,7 +164,7 @@ typing-extensions==4.12.2
     #   typer
 urllib3==2.2.2
     # via requests
-validators==0.29.0
+validators==0.33.0
     # via frictionless
 virtualenv==20.26.3
     # via pre-commit
@@ -172,6 +172,8 @@ webencodings==0.5.1
     # via html5lib
 xlrd==2.0.1
     # via hdx-python-utilities (pyproject.toml)
+xlsx2csv==0.8.3
+    # via hdx-python-utilities (pyproject.toml)
 xlsxwriter==3.2.0
     # via tableschema-to-template
 xlwt==1.3.0

{hdx_python_utilities-3.7.1 → hdx_python_utilities-3.7.3}/src/hdx/utilities/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.7.1'
-__version_tuple__ = version_tuple = (3, 7, 1)
+__version__ = version = '3.7.3'
+__version_tuple__ = version_tuple = (3, 7, 3)

{hdx_python_utilities-3.7.1 → hdx_python_utilities-3.7.3}/src/hdx/utilities/downloader.py

@@ -15,6 +15,7 @@ from frictionless.resources import TableResource
 from ratelimit import RateLimitDecorator, sleep_and_retry
 from requests import Request
 from ruamel.yaml import YAML
+from xlsx2csv import Xlsx2csv
 
 from .base_downloader import BaseDownload, DownloadError
 from .frictionless_wrapper import get_frictionless_tableresource
@@ -669,6 +670,7 @@ class Download(BaseDownload):
             **kwargs:
             format (Optional[str]): Type of file. Defaults to inferring.
             file_type (Optional[str]): Type of file. Defaults to inferring.
+            xlsx2csv (bool): Whether to convert xlsx files. Defaults to False.
             encoding (Optional[str]): Type of encoding. Defaults to inferring.
             compression (Optional[str]): Type of compression. Defaults to inferring.
             delimiter (Optional[str]): Delimiter for values in csv rows. Defaults to inferring.
@@ -690,6 +692,20 @@ class Download(BaseDownload):
         """
         if headers is None:
             raise DownloadError("Argument headers cannot be None!")
+        xlsx2csv = kwargs.pop("xlsx2csv", False)
+        if xlsx2csv:
+            path = self.download_file(url)
+            outpath = path.replace(".xlsx", ".csv")
+            sheet = kwargs.pop("sheet", 1)
+            if isinstance(sheet, int):
+                sheet_args = {"sheetid": sheet}
+            else:
+                sheet_args = {"sheetname": sheet}
+            Xlsx2csv(path).convert(outpath, **sheet_args)
+            url = outpath
+            kwargs["format"] = "csv"  # format takes precedence over file_type
+            kwargs.pop("fill_merged_cells", None)
+
         resource = self.get_frictionless_tableresource(
             url,
             ignore_blank_rows=ignore_blank_rows,
@@ -771,6 +787,7 @@ class Download(BaseDownload):
             **kwargs:
             format (Optional[str]): Type of file. Defaults to inferring.
             file_type (Optional[str]): Type of file. Defaults to inferring.
+            xlsx2csv (bool): Whether to convert xlsx files. Defaults to False.
             encoding (Optional[str]): Type of encoding. Defaults to inferring.
             compression (Optional[str]): Type of compression. Defaults to inferring.
             delimiter (Optional[str]): Delimiter for values in csv rows. Defaults to inferring.
@@ -875,6 +892,7 @@ class Download(BaseDownload):
             **kwargs:
             format (Optional[str]): Type of file. Defaults to inferring.
             file_type (Optional[str]): Type of file. Defaults to inferring.
+            xlsx2csv (bool): Whether to convert xlsx files. Defaults to False.
             encoding (Optional[str]): Type of encoding. Defaults to inferring.
             compression (Optional[str]): Type of compression. Defaults to inferring.
             delimiter (Optional[str]): Delimiter for values in csv rows. Defaults to inferring.
@@ -950,6 +968,7 @@ class Download(BaseDownload):
             **kwargs:
             format (Optional[str]): Type of file. Defaults to inferring.
             file_type (Optional[str]): Type of file. Defaults to inferring.
+            xlsx2csv (bool): Whether to convert xlsx files. Defaults to False.
             encoding (Optional[str]): Type of encoding. Defaults to inferring.
             compression (Optional[str]): Type of compression. Defaults to inferring.
             delimiter (Optional[str]): Delimiter for values in csv rows. Defaults to inferring.

{hdx_python_utilities-3.7.1 → hdx_python_utilities-3.7.3}/src/hdx/utilities/retriever.py

@@ -418,7 +418,7 @@ class Retrieve(BaseDownload):
             filename (Optional[str]): Filename of saved file. Defaults to getting from url.
             logstr (Optional[str]): Text to use in log string to describe download. Defaults to filename.
             fallback (bool): Whether to use static fallback if download fails. Defaults to False.
-            **kwargs: Parameters to pass to download_file call
+            **kwargs: Parameters to pass to download_file and get_tabular_rows calls
 
         Returns:
             Tuple[List[str],Iterator[ListDict]]: Tuple (headers, iterator where each row is a list or dictionary)

{hdx_python_utilities-3.7.1 → hdx_python_utilities-3.7.3}/src/hdx/utilities/text.py

@@ -4,6 +4,7 @@ import difflib
 import logging
 import re
 import string
+import unicodedata
 from string import punctuation
 from typing import Any, Dict, List, Optional, Set, Tuple
 
@@ -16,6 +17,44 @@ PUNCTUATION_MINUS_BRACKETS = r"""!"#$%&'*+,-./:;<=>?@\^_`|~"""
 TEMPLATE_VARIABLES = re.compile("{{.*?}}")
 
 
+KEEP_CHARS_SAME = set(string.ascii_lowercase).union(set(string.digits))
+CHANGE_TO_LOWERCASE = set(string.ascii_uppercase)
+MAP_TO_SPACE = set(string.punctuation).union(set(string.whitespace))
+MAP_TO_SPACE.remove("'")
+
+
+def normalise(text: str) -> str:
+    """
+    Mormalise text for example to support name matching. Accented characters
+    are replaced with non-accented if possible. Any punctuation and whitespace
+    is replaced with a space except for ' which is replaced with blank.
+    Multiple spaces are replaced with a single space. Uppercase is replaced
+    with lowercase. Spaces at start and end are removed. All non-ASCII
+    characters are removed.
+
+    Args:
+        text (str): Text to normalise
+
+    Returns:
+        str: Normalised text
+    """
+    chars = []
+    space = False
+    for chr in unicodedata.normalize("NFD", text):
+        if chr in KEEP_CHARS_SAME:
+            chars.append(chr)
+            space = False
+        elif chr in CHANGE_TO_LOWERCASE:
+            chars.append(chr.lower())
+            space = False
+        elif chr in MAP_TO_SPACE:
+            if space:
+                continue
+            chars.append(" ")
+            space = True
+    return "".join(chars).strip()
+
+
 def remove_end_characters(
     string: str, characters_to_remove: str = punctuation
 ) -> str:

{hdx_python_utilities-3.7.1 → hdx_python_utilities-3.7.3}/tests/hdx/utilities/test_downloader.py

@@ -1220,6 +1220,82 @@ class TestDownloader:
                 "Tulkarm",
             ]
 
+    def test_get_tabular_rows_xlsx2csv(self, fixtureurlexcel):
+        expected_headers = [
+            "GWNO",
+            "EVENT_ID_CNTY",
+            "EVENT_ID_NO_CNTY",
+            "EVENT_DATE",
+            "YEAR",
+            "TIME_PRECISION",
+            "EVENT_TYPE",
+            "ACTOR1",
+            "ALLY_ACTOR_1",
+            "INTER1",
+            "ACTOR2",
+            "ALLY_ACTOR_2",
+            "INTER2",
+            "INTERACTION",
+            "COUNTRY",
+            "ADMIN1",
+            "ADMIN2",
+            "ADMIN3",
+            "LOCATION",
+            "LATITUDE",
+            "LONGITUDE",
+            "GEO_PRECISION",
+            "SOURCE",
+            "NOTES",
+            "FATALITIES",
+        ]
+
+        expected_row = [
+            "615",
+            "1416RTA",
+            None,
+            "18/04/2001",
+            "2001",
+            "1",
+            "Violence against civilians",
+            "Police Forces of Algeria (1999-)",
+            None,
+            "1",
+            "Civilians (Algeria)",
+            "Berber Ethnic Group (Algeria)",
+            "7",
+            "17",
+            "Algeria",
+            "Tizi Ouzou",
+            "Beni-Douala",
+            None,
+            "Beni Douala",
+            "36.61954",
+            "4.08282",
+            "1",
+            "Associated Press Online",
+            "A Berber student was shot while in police custody at a police station in "
+            "Beni Douala. He later died on Apr.21.",
+            "1",
+        ]
+
+        with Download() as downloader:
+            headers, iterator = downloader.get_tabular_rows(
+                fixtureurlexcel,
+                format="xlsx",
+                xlsx2csv=True,
+            )
+            assert headers == expected_headers
+            assert list(iterator)[0] == expected_row
+
+            headers, iterator = downloader.get_tabular_rows(
+                fixtureurlexcel,
+                format="xlsx",
+                xlsx2csv=True,
+                sheet="test_data",
+            )
+            assert headers == expected_headers
+            assert list(iterator)[0] == expected_row
+
     def test_get_tabular_rows_json(self, fixturejsonurl):
         with Download() as downloader:
             headers, iterator = downloader.get_tabular_rows(

{hdx_python_utilities-3.7.1 → hdx_python_utilities-3.7.3}/tests/hdx/utilities/test_text.py

@@ -15,6 +15,7 @@ from hdx.utilities.text import (
     get_words_in_sentence,
     match_template_variables,
     multiple_replace,
+    normalise,
     number_format,
     only_allowed_in_str,
     remove_end_characters,
@@ -28,6 +29,14 @@ class TestText:
     b = "The quicker brown fox leapt over the slower fox. It was so fast!"
     c = "The quick brown fox climbed over the lazy dog. It was so fast!"
 
+    def test_normalise(self):
+        assert (
+            normalise(
+                "£^*& ()+-[]<>?|\ Al DhaleZ'eÉ / الضالع,,..1234''#~~### "
+            )
+            == "al dhalezee 1234"
+        )
+
     def test_remove_end_characters(self):
         assert remove_end_characters('lalala,.,"') == "lalala"
         assert (