hdx-python-utilities 3.7.4__tar.gz → 3.8.0__tar.gz

{hdx_python_utilities-3.7.4 → hdx_python_utilities-3.8.0}/.config/pre-commit-config.yaml

@@ -2,7 +2,7 @@ default_language_version:
     python: python3.12
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
         exclude: test_loader.py
@@ -10,7 +10,7 @@ repos:
         exclude: (test_csv_processing_blanks.csv|test.txt)
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.4
+    rev: v0.8.6
     hooks:
       # Run the linter.
       - id: ruff
@@ -19,7 +19,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.4.5
+    rev: 0.5.15
     hooks:
       # Run the pip compile
       - id: pip-compile

{hdx_python_utilities-3.7.4 → hdx_python_utilities-3.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: hdx-python-utilities
-Version: 3.7.4
+Version: 3.8.0
 Summary: HDX Python Utilities for streaming tabular data, date and time handling and other helpful functions
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-utilities
 Author-email: Michael Rans <rans@email.com>
@@ -30,6 +30,7 @@ Requires-Dist: ijson>=3.2.3
 Requires-Dist: jsonlines>=4.0.0
 Requires-Dist: loguru
 Requires-Dist: openpyxl>=3.1.2
+Requires-Dist: pyphonetics
 Requires-Dist: python-dateutil<2.9.1,>=2.9.0
 Requires-Dist: ratelimit
 Requires-Dist: requests-file

{hdx_python_utilities-3.7.4 → hdx_python_utilities-3.8.0}/documentation/main.md

@@ -18,6 +18,7 @@ Python developers. Note that these are not specific to HDX.
 1. [State utility](#state-utility)
 1. [Path utilities](#path-utilities)
 1. [Text processing](#text-processing)
+1. [Matching utilities](#matching-utilities)
 1. [Encoding utilities](#encoding-utilities)
 1. [Check valid UUID](#valid-uuid)
 1. [Easy building and packaging](#easy-building-and-packaging)
@@ -31,6 +32,12 @@ The code for the library is [here](https://github.com/OCHA-DAP/hdx-python-utilit
 The library has detailed API documentation which can be found in the menu at the top.
 
 ## Breaking Changes
+From 3.8.0, multiple_replace, match_template_variables, earliest_index,
+get_matching_text_in_strs, get_matching_text,
+get_matching_then_nonmatching_text moved from hdx.utilities.text to
+hdx.utilities.matching. ErrorOnExit renamed ErrorHandler with changed
+functionality.
+
 From 3.5.5, Python 3.7 no longer supported
 
 From 3.3.7, improved parse_date and parse_date_range by default will attempt to parse
@@ -648,17 +655,23 @@ Then use the logger like this:
 There is a class that allows collecting of errors to be logged later, typically on exit.
 It is called ErrorsOnExit and can be used as follows:
 
-    with ErrorsOnExit() as errors:
+    with ErrorsHandler() as errors:
         ...
-        errors.add("MY ERROR MESSAGE")
+        errors.add_message("MY ERROR MESSAGE")
+        errors.add_message("MY WARNING MESSAGE", "category 1", "warning")
         ...
-        errors.add("ANOTHER ERROR MESSAGE")
-
-The above code will collect the errors, in this case "MY ERROR MESSAGE" and "ANOTHER
-ERROR MESSAGE". On leaving the `with` block, the errors will be logged and the code will
-exit with the error code 1 (ie. `sys.exit(1)` will be called). If there are no errors,
-the code will not exit and execution will continue after the `with` block (ie.
-`sys.exit(1)` will not be called).
+        errors.add("ERROR MESSAGE", "category 1", "warning")
+        errors.add("ANOTHER ERROR MESSAGE", "category 1", "warning")
+
+The above code will collect the errors and warnings, in this case
+"MY ERROR MESSAGE", "category 1 - ERROR MESSAGE",
+"category 1 - ANOTHER ERROR MESSAGE" and "category 1 - MY WARNING MESSAGE".
+On leaving the `with` block, the errors and warnings will be logged by category
+and sorted. The code will exit with the error code 1 (ie.`sys.exit(1)` will be
+called) if there are errors and `should_exit_on_error` is True (the default
+for this parameter in the constructor). If there are no errors, the code will
+not exit and execution will continue after the `with` block (ie.`sys.exit(1)`
+will not be called).
 
 ## State utility
 
@@ -803,13 +816,35 @@ Examples:
     assert remove_string("lala, 01/02/2020 ", "01/02/2020", PUNCTUATION_MINUS_BRACKETS) == "lala "
     assert remove_string("lala,(01/02/2020) ", "01/02/2020", PUNCTUATION_MINUS_BRACKETS) == "lala,() "
 
+    # Extract words from a string sentence into a list
+    result = get_words_in_sentence("Korea (Democratic People's Republic of)")
+    assert result == ["Korea", "Democratic", "People's", "Republic", "of"]
+
+## Matching utilities
+
+Examples:
+
+    possible_names = ["Al Maharah", "Ad Dali", "Dhamar"]
+    phonetics = Phonetics()
+    assert phonetics.match(possible_names, "al dali") == 1
+
+    org_type_lookup = {"Donor": "433", "National NGO": "441", "Other": "443"}
+    lookup = {
+        normalise(k): v for k, v in org_type_lookup.items()
+    }
+    assert get_code_from_name("NATIONAL_NGO", lookup, [], fuzzy_match=False) == "441"
+
+    a = "The quick brown fox jumped over the lazy dog. It was so fast!"
+
     # Replace multiple strings in a string simultaneously
     result = multiple_replace(a, {"quick": "slow", "fast": "slow", "lazy": "busy"})
     assert result == "The slow brown fox jumped over the busy dog. It was so slow!"
 
-    # Extract words from a string sentence into a list
-    result = get_words_in_sentence("Korea (Democratic People's Republic of)")
-    assert result == ["Korea", "Democratic", "People's", "Republic", "of"]
+    # Look for template variables in a string (ie. {{XXX}})
+    assert match_template_variables("dasdda{{abc}}gff") == ("{{abc}}", "abc")
+
+    # Search a string for each of a list of strings and return the earliest index
+    assert earliest_index(a, ["dog", "lala", "fox", "haha", "quick"]) == 4
 
     # Find matching text in strings
     a = "The quick brown fox jumped over the lazy dog. It was so fast!"
@@ -818,11 +853,6 @@ Examples:
     result = get_matching_text([a, b, c], match_min_size=10)
     assert result == " brown fox  over the  It was so fast!"
 
-    # Search a string for each of a list of strings and return the earliest index
-    assert earliest_index(a, ["dog", "lala", "fox", "haha", "quick"]) == 4
-
-    # Look for template variables in a string (ie. {{XXX}})
-    assert match_template_variables("dasdda{{abc}}gff") == ("{{abc}}", "abc")
 
 ## Encoding utilities
 

{hdx_python_utilities-3.7.4 → hdx_python_utilities-3.8.0}/pyproject.toml

@@ -47,6 +47,7 @@ dependencies = [
     "jsonlines>=4.0.0",
     # /end frictionless extras
     "loguru",
+    "pyphonetics",
     "python-dateutil>=2.9.0, <2.9.1",
     "ratelimit",
     "requests-file",
@@ -105,6 +106,9 @@ run = """
        --cov-report=lcov --cov-report=term-missing
        """
 
+[tool.hatch.envs.hatch-static-analysis]
+dependencies = ["ruff==0.8.6"]
+
 [tool.hatch.envs.hatch-static-analysis.scripts]
 format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]
 format-fix = ["ruff format --config .config/ruff.toml {args:.}",]

{hdx_python_utilities-3.7.4 → hdx_python_utilities-3.8.0}/requirements.txt

@@ -2,7 +2,7 @@
 #    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
 annotated-types==0.7.0
     # via pydantic
-attrs==24.2.0
+attrs==24.3.0
     # via
     #   frictionless
     #   jsonlines
@@ -10,25 +10,25 @@ attrs==24.2.0
     #   referencing
 beautifulsoup4==4.12.3
     # via hdx-python-utilities (pyproject.toml)
-certifi==2024.8.30
+certifi==2024.12.14
     # via requests
 cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.3.2
+charset-normalizer==3.4.1
     # via requests
-click==8.1.7
+click==8.1.8
     # via typer
-coverage==7.6.1
+coverage==7.6.10
     # via pytest-cov
-distlib==0.3.8
+distlib==0.3.9
     # via virtualenv
-dnspython==2.6.1
+dnspython==2.7.0
     # via email-validator
 email-validator==2.2.0
     # via hdx-python-utilities (pyproject.toml)
-et-xmlfile==1.1.0
+et-xmlfile==2.0.0
     # via openpyxl
 filelock==3.16.1
     # via virtualenv
@@ -36,9 +36,9 @@ frictionless==5.18.0
     # via hdx-python-utilities (pyproject.toml)
 html5lib==1.1
     # via hdx-python-utilities (pyproject.toml)
-humanize==4.10.0
+humanize==4.11.0
     # via frictionless
-identify==2.6.1
+identify==2.6.5
     # via pre-commit
 idna==3.10
     # via
@@ -48,9 +48,9 @@ ijson==3.3.0
     # via hdx-python-utilities (pyproject.toml)
 iniconfig==2.0.0
     # via pytest
-isodate==0.6.1
+isodate==0.7.2
     # via frictionless
-jinja2==3.1.4
+jinja2==3.1.5
     # via frictionless
 jsonlines==4.0.0
     # via hdx-python-utilities (pyproject.toml)
@@ -58,9 +58,9 @@ jsonschema==4.23.0
     # via
     #   frictionless
     #   tableschema-to-template
-jsonschema-specifications==2023.12.1
+jsonschema-specifications==2024.10.1
     # via jsonschema
-loguru==0.7.2
+loguru==0.7.3
     # via
     #   hdx-python-utilities (pyproject.toml)
     #   pytest-loguru
@@ -68,7 +68,7 @@ markdown-it-py==3.0.0
     # via rich
 marko==2.1.2
     # via frictionless
-markupsafe==2.1.5
+markupsafe==3.0.2
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
@@ -76,7 +76,7 @@ nodeenv==1.9.1
     # via pre-commit
 openpyxl==3.1.5
     # via hdx-python-utilities (pyproject.toml)
-packaging==24.1
+packaging==24.2
     # via pytest
 petl==1.7.15
     # via frictionless
@@ -84,19 +84,21 @@ platformdirs==4.3.6
     # via virtualenv
 pluggy==1.5.0
     # via pytest
-pre-commit==3.8.0
+pre-commit==4.0.1
     # via hdx-python-utilities (pyproject.toml)
-pydantic==2.9.2
+pydantic==2.10.5
     # via frictionless
-pydantic-core==2.23.4
+pydantic-core==2.27.2
     # via pydantic
-pygments==2.18.0
+pygments==2.19.1
     # via rich
-pytest==8.3.3
+pyphonetics==0.5.3
+    # via hdx-python-utilities (pyproject.toml)
+pytest==8.3.4
     # via
     #   hdx-python-utilities (pyproject.toml)
     #   pytest-cov
-pytest-cov==5.0.0
+pytest-cov==6.0.0
     # via hdx-python-utilities (pyproject.toml)
 pytest-loguru==0.4.0
     # via hdx-python-utilities (pyproject.toml)
@@ -125,24 +127,23 @@ requests-file==2.1.0
     # via hdx-python-utilities (pyproject.toml)
 rfc3986==2.0.0
     # via frictionless
-rich==13.8.1
+rich==13.9.4
     # via typer
-rpds-py==0.20.0
+rpds-py==0.22.3
     # via
     #   jsonschema
     #   referencing
-ruamel-yaml==0.18.6
+ruamel-yaml==0.18.10
     # via hdx-python-utilities (pyproject.toml)
-ruamel-yaml-clib==0.2.8
+ruamel-yaml-clib==0.2.12
     # via ruamel-yaml
 shellingham==1.5.4
     # via typer
-simpleeval==0.9.13
+simpleeval==1.0.3
     # via frictionless
-six==1.16.0
+six==1.17.0
     # via
     #   html5lib
-    #   isodate
     #   python-dateutil
 soupsieve==2.6
     # via beautifulsoup4
@@ -154,7 +155,7 @@ tabulate==0.9.0
     # via frictionless
 text-unidecode==1.3
     # via python-slugify
-typer==0.12.5
+typer==0.15.1
     # via frictionless
 typing-extensions==4.12.2
     # via
@@ -162,17 +163,19 @@ typing-extensions==4.12.2
     #   pydantic
     #   pydantic-core
     #   typer
-urllib3==2.2.3
+unidecode==1.3.8
+    # via pyphonetics
+urllib3==2.3.0
     # via requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.26.6
+virtualenv==20.28.1
     # via pre-commit
 webencodings==0.5.1
     # via html5lib
 xlrd==2.0.1
     # via hdx-python-utilities (pyproject.toml)
-xlsx2csv==0.8.3
+xlsx2csv==0.8.4
     # via hdx-python-utilities (pyproject.toml)
 xlsxwriter==3.2.0
     # via tableschema-to-template

{hdx_python_utilities-3.7.4 → hdx_python_utilities-3.8.0}/src/hdx/utilities/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.7.4'
-__version_tuple__ = version_tuple = (3, 7, 4)
+__version__ = version = '3.8.0'
+__version_tuple__ = version_tuple = (3, 8, 0)

hdx_python_utilities-3.8.0/src/hdx/utilities/error_handler.py

@@ -0,0 +1,185 @@
+"""Collect errors and warnings by category and log them."""
+
+import logging
+import sys
+from typing import Any, Optional
+
+from hdx.utilities.dictandlist import dict_of_sets_add
+from hdx.utilities.typehint import ListTuple
+
+logger = logging.getLogger(__name__)
+
+
+class ErrorHandler:
+    """Class that enables recording of errors and warnings.
+
+    Errors and warnings can be logged by calling the `output` method or
+    automatically logged on exit. Messages are output grouped by category and
+    sorted.
+
+    Args:
+        should_exit_on_error (bool): Whether to exit with a 1 code if there are errors. Default is True.
+
+    """
+
+    def __init__(
+        self,
+        should_exit_on_error: bool = True,
+    ):
+        self.should_exit_on_error = should_exit_on_error
+        self.shared_errors = {
+            "error": {},
+            "warning": {},
+        }
+
+    def add(
+        self, message: str, category: str = "", message_type: str = "error"
+    ) -> None:
+        """Add error to be logged. Prepend category if supplied. Output format:
+        error category - {text}
+
+        Args:
+            message (str): Error message
+            category (str): Error category. Defaults to "".
+            message_type (str): The type of message (error or warning). Default is "error"
+
+        Returns:
+            None
+        """
+        message = message.strip()
+        if category:
+            output = f"{category} - {message}"
+        else:
+            output = message
+        dict_of_sets_add(self.shared_errors[message_type], category, output)
+
+    @staticmethod
+    def missing_value_message(value_type: str, value: Any) -> str:
+        """
+        Generate a formatted message for a missing value of a specific type in
+        a fixed format:
+            error category - type n not found
+
+        Args:
+            value_type (str): The type of value that is missing
+            value (Any): The specific missing value
+
+        Returns:
+            str: A formatted message stating the missing value and its type
+        """
+        return f"{value_type} {str(value)} not found"
+
+    def add_missing_value(
+        self,
+        value_type: str,
+        value: Any,
+        category: str = "",
+        message_type: str = "error",
+    ) -> None:
+        """
+        Add a new message (typically a warning or error) concerning a missing value
+        to a dictionary of messages in a fixed format:
+            error category - type n not found
+        identifier is usually a dataset name.
+        Args:
+            value_type (str): Type of value e.g. "sector"
+            value (Any): Missing value
+            category (str): Error category. Defaults to "".
+            message_type (str): The type of message (error or warning). Default is "error"
+        Returns:
+            None
+        """
+        self.add(
+            self.missing_value_message(value_type, value),
+            category,
+            message_type,
+        )
+
+    def multi_valued_message(
+        self, text: str, values: ListTuple
+    ) -> Optional[str]:
+        """
+        Generate a formatted message for a list of values in a fixed format:
+            error category - n {text}. First 10 values: n1,n2,n3...
+        If less than 10 values, ". First 10 values" is omitted. identifier is usually
+        a dataset name. Values are cast to string.
+
+        Args:
+            text (str): Descriptive text for the issue (e.g., "invalid values")
+            values (ListTuple): The list of related values of concern
+
+        Returns:
+            Optional[str]: A formatted string in the format defined above
+        """
+        if not values:
+            return None
+        no_values = len(values)
+        if no_values > 10:
+            values = values[:10]
+            message_suffix = ". First 10 values"
+        else:
+            message_suffix = ""
+        return f"{no_values} {text}{message_suffix}: {', '.join(map(str, values))}"
+
+    def add_multi_valued(
+        self,
+        text: str,
+        values: ListTuple,
+        category: str = "",
+        message_type: str = "error",
+    ) -> bool:
+        """
+        Add a new message (typically a warning or error) concerning a list of
+        values to a set of messages in a fixed format:
+            error category - n {text}. First 10 values: n1,n2,n3...
+        If less than 10 values, ". First 10 values" is omitted. identifier is usually
+        a dataset name. Values are cast to string.
+
+        Args:
+            text (str): Text to use e.g. "negative values removed"
+            values (ListTuple): List of values of concern
+            category (str): Error category. Defaults to "".
+            message_type (str): The type of message (error or warning). Default is "error"
+        Returns:
+            bool: True if a message was added, False if not
+        """
+        message = self.multi_valued_message(text, values)
+        if message is None:
+            return False
+        self.add(message, category, message_type)
+        return True
+
+    def log(self) -> None:
+        """
+        Log errors and warning by category and sorted
+
+        Returns:
+            None
+        """
+
+        for _, errors in self.shared_errors["error"].items():
+            errors = sorted(errors)
+            for error in errors:
+                logger.error(error)
+        for _, warnings in self.shared_errors["warning"].items():
+            warnings = sorted(warnings)
+            for warning in warnings:
+                logger.warning(warning)
+
+    def exit_on_error(self) -> None:
+        """Exit with a 1 code if there are errors and should_exit_on_error
+        is True
+
+        Returns:
+            None
+        """
+        if self.should_exit_on_error and self.shared_errors["error"]:
+            sys.exit(1)
+
+    def __enter__(self) -> "ErrorHandler":
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.log()
+        if exc_type is None:
+            self.exit_on_error()

hdx_python_utilities-3.8.0/src/hdx/utilities/errors_onexit.py

@@ -0,0 +1,14 @@
+"""Collect and log errors on exit."""
+
+import warnings
+
+from hdx.utilities.error_handler import ErrorHandler
+
+
+class ErrorsOnExit(ErrorHandler):  # pragma: no cover
+    def __init__(self) -> None:
+        warnings.warn(
+            "The ErrorsOnExit class was renamed ErrorHandler and will be removed in future!",
+            DeprecationWarning,
+        )
+        super().__init__()