hdx-python-utilities 3.6.7__tar.gz → 3.8.2__tar.gz

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/.config/pre-commit-config.yaml

@@ -1,8 +1,8 @@
 default_language_version:
-    python: python3.11
+    python: python3.12
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
         exclude: test_loader.py
@@ -10,7 +10,7 @@ repos:
         exclude: (test_csv_processing_blanks.csv|test.txt)
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.2
+    rev: v0.8.6
     hooks:
       # Run the linter.
       - id: ruff
@@ -18,10 +18,11 @@ repos:
       # Run the formatter.
       - id: ruff-format
         args: [--config, .config/ruff.toml]
-  - repo: https://github.com/jazzband/pip-tools
-    rev: 7.4.1
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.5.15
     hooks:
+      # Run the pip compile
       - id: pip-compile
         name: pip-compile requirements.txt
         files: pyproject.toml
-        args: [pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt]
+        args: [ pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt ]

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/.config/ruff.toml

@@ -12,6 +12,5 @@ ignore = [
     "E501" # Line too long
 ]
 
-
 [lint.isort]
 known-local-folder = ["hdx"]

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/.github/workflows/publish.yaml

@@ -21,7 +21,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install --upgrade hatch
+    - name: Install Hatch
+      uses: pypa/hatch@install
     - name: Build with hatch
       run: |
         hatch build

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/.github/workflows/run-python-tests.yaml

@@ -26,14 +26,15 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install --upgrade hatch
+    - name: Install Hatch
+      uses: pypa/hatch@install
     - name: Test with hatch/pytest
       run: |
-        hatch run test:test
+        hatch test
     - name: Check styling
       if: always()
       run: |
-        hatch run lint:style
+        hatch fmt --check
     - name: Publish Unit Test Results
       uses: EnricoMi/publish-unit-test-result-action@v2
       if: always()

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/.gitignore

@@ -41,7 +41,9 @@ htmlcov/
 .coverage.*
 .cache
 nosetests.xml
+coverage.lcov
 coverage.xml
+test-results.xml
 *,cover
 .hypothesis/
 tests/*.log

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: hdx-python-utilities
-Version: 3.6.7
+Version: 3.8.2
 Summary: HDX Python Utilities for streaming tabular data, date and time handling and other helpful functions
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-utilities
 Author-email: Michael Rans <rans@email.com>
@@ -25,17 +25,19 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
-Requires-Dist: frictionless>=5.16.1
+Requires-Dist: frictionless>=5.18.0
 Requires-Dist: ijson>=3.2.3
 Requires-Dist: jsonlines>=4.0.0
 Requires-Dist: loguru
 Requires-Dist: openpyxl>=3.1.2
-Requires-Dist: python-dateutil==2.8.2
+Requires-Dist: pyphonetics
+Requires-Dist: python-dateutil<2.9.1,>=2.9.0
 Requires-Dist: ratelimit
 Requires-Dist: requests-file
 Requires-Dist: ruamel-yaml
 Requires-Dist: tableschema-to-template>=0.0.13
 Requires-Dist: xlrd>=2.0.1
+Requires-Dist: xlsx2csv
 Requires-Dist: xlwt>=1.3.0
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/documentation/.readthedocs.yaml

@@ -11,5 +11,6 @@ build:
   jobs:
     pre_build:
       - pip install --upgrade mkdocs
+      - pip install mkdocs-material
       - pip install pydoc-markdown
       - pydoc-markdown documentation/pydoc-markdown.yaml

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/documentation/main.md

@@ -18,6 +18,7 @@ Python developers. Note that these are not specific to HDX.
 1. [State utility](#state-utility)
 1. [Path utilities](#path-utilities)
 1. [Text processing](#text-processing)
+1. [Matching utilities](#matching-utilities)
 1. [Encoding utilities](#encoding-utilities)
 1. [Check valid UUID](#valid-uuid)
 1. [Easy building and packaging](#easy-building-and-packaging)
@@ -31,6 +32,12 @@ The code for the library is [here](https://github.com/OCHA-DAP/hdx-python-utilit
 The library has detailed API documentation which can be found in the menu at the top.
 
 ## Breaking Changes
+From 3.8.0, multiple_replace, match_template_variables, earliest_index,
+get_matching_text_in_strs, get_matching_text,
+get_matching_then_nonmatching_text moved from hdx.utilities.text to
+hdx.utilities.matching. ErrorOnExit renamed ErrorHandler with changed
+functionality.
+
 From 3.5.5, Python 3.7 no longer supported
 
 From 3.3.7, improved parse_date and parse_date_range by default will attempt to parse
@@ -648,17 +655,23 @@ Then use the logger like this:
 There is a class that allows collecting of errors to be logged later, typically on exit.
 It is called ErrorsOnExit and can be used as follows:
 
-    with ErrorsOnExit() as errors:
+    with ErrorsHandler() as errors:
         ...
-        errors.add("MY ERROR MESSAGE")
+        errors.add_message("MY ERROR MESSAGE")
+        errors.add_message("MY WARNING MESSAGE", "category 1", "warning")
         ...
-        errors.add("ANOTHER ERROR MESSAGE")
-
-The above code will collect the errors, in this case "MY ERROR MESSAGE" and "ANOTHER
-ERROR MESSAGE". On leaving the `with` block, the errors will be logged and the code will
-exit with the error code 1 (ie. `sys.exit(1)` will be called). If there are no errors,
-the code will not exit and execution will continue after the `with` block (ie.
-`sys.exit(1)` will not be called).
+        errors.add("ERROR MESSAGE", "category 1", "warning")
+        errors.add("ANOTHER ERROR MESSAGE", "category 1", "warning")
+
+The above code will collect the errors and warnings, in this case
+"MY ERROR MESSAGE", "category 1 - ERROR MESSAGE",
+"category 1 - ANOTHER ERROR MESSAGE" and "category 1 - MY WARNING MESSAGE".
+On leaving the `with` block, the errors and warnings will be logged by category
+and sorted. The code will exit with the error code 1 (ie.`sys.exit(1)` will be
+called) if there are errors and `should_exit_on_error` is True (the default
+for this parameter in the constructor). If there are no errors, the code will
+not exit and execution will continue after the `with` block (ie.`sys.exit(1)`
+will not be called).
 
 ## State utility
 
@@ -785,6 +798,9 @@ Examples:
 
     a = "The quick brown fox jumped over the lazy dog. It was so fast!"
 
+    # Normalise text eg. to support name matching
+    assert normalise("£^*& ()+-[]<>?|\ Al DhaleZ'eÉ / الضالع,,..1234''#~~### ") == "al dhalezee 1234"
+
     # Remove whitespace and punctuation from end of string
     assert remove_end_characters('lalala,.,"') == "lalala"
     assert remove_end_characters('lalala, .\t/,"', f"{punctuation}{whitespace}" == "lalala"
@@ -800,13 +816,35 @@ Examples:
     assert remove_string("lala, 01/02/2020 ", "01/02/2020", PUNCTUATION_MINUS_BRACKETS) == "lala "
     assert remove_string("lala,(01/02/2020) ", "01/02/2020", PUNCTUATION_MINUS_BRACKETS) == "lala,() "
 
+    # Extract words from a string sentence into a list
+    result = get_words_in_sentence("Korea (Democratic People's Republic of)")
+    assert result == ["Korea", "Democratic", "People's", "Republic", "of"]
+
+## Matching utilities
+
+Examples:
+
+    possible_names = ["Al Maharah", "Ad Dali", "Dhamar"]
+    phonetics = Phonetics()
+    assert phonetics.match(possible_names, "al dali") == 1
+
+    org_type_lookup = {"Donor": "433", "National NGO": "441", "Other": "443"}
+    lookup = {
+        normalise(k): v for k, v in org_type_lookup.items()
+    }
+    assert get_code_from_name("NATIONAL_NGO", lookup, [], fuzzy_match=False) == "441"
+
+    a = "The quick brown fox jumped over the lazy dog. It was so fast!"
+
     # Replace multiple strings in a string simultaneously
     result = multiple_replace(a, {"quick": "slow", "fast": "slow", "lazy": "busy"})
     assert result == "The slow brown fox jumped over the busy dog. It was so slow!"
 
-    # Extract words from a string sentence into a list
-    result = get_words_in_sentence("Korea (Democratic People's Republic of)")
-    assert result == ["Korea", "Democratic", "People's", "Republic", "of"]
+    # Look for template variables in a string (ie. {{XXX}})
+    assert match_template_variables("dasdda{{abc}}gff") == ("{{abc}}", "abc")
+
+    # Search a string for each of a list of strings and return the earliest index
+    assert earliest_index(a, ["dog", "lala", "fox", "haha", "quick"]) == 4
 
     # Find matching text in strings
     a = "The quick brown fox jumped over the lazy dog. It was so fast!"
@@ -815,11 +853,6 @@ Examples:
     result = get_matching_text([a, b, c], match_min_size=10)
     assert result == " brown fox  over the  It was so fast!"
 
-    # Search a string for each of a list of strings and return the earliest index
-    assert earliest_index(a, ["dog", "lala", "fox", "haha", "quick"]) == 4
-
-    # Look for template variables in a string (ie. {{XXX}})
-    assert match_template_variables("dasdda{{abc}}gff") == ("{{abc}}", "abc")
 
 ## Encoding utilities
 

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/documentation/pydoc-markdown.yaml

@@ -9,7 +9,7 @@ renderer:
   output_directory: docs
   mkdocs_config:
     site_name: HDX Python Utilities
-    theme: mkdocs
+    theme: material
     repo_url: "https://github.com/OCHA-DAP/hdx-python-utilities"
   markdown:
     source_linker:

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/pyproject.toml

@@ -36,7 +36,7 @@ requires-python = ">=3.8"
 # Extras for frictionless[excel,json] added explicitly
 # for conda-forge compatibility
 dependencies = [
-    "frictionless>=5.16.1",
+    "frictionless>=5.18.0",
     # frictionless[excel]
     "openpyxl>=3.1.2",
     "tableschema-to-template>=0.0.13",
@@ -47,10 +47,12 @@ dependencies = [
     "jsonlines>=4.0.0",
     # /end frictionless extras
     "loguru",
-    "python-dateutil==2.8.2",
+    "pyphonetics",
+    "python-dateutil>=2.9.0, <2.9.1",
     "ratelimit",
     "requests-file",
     "ruamel.yaml",
+    "xlsx2csv",
 ]
 dynamic = ["version"]
 
@@ -91,25 +93,24 @@ version_scheme = "python-simplified-semver"
 
 # Tests
 
-[tool.hatch.envs.test]
+[tool.hatch.envs.hatch-test]
 features = ["html", "email", "test"]
 
-[tool.hatch.envs.test.scripts]
-test = """
+[[tool.hatch.envs.hatch-test.matrix]]
+python = ["3.12"]
+
+[tool.hatch.envs.hatch-test.scripts]
+run = """
        pytest -c .config/pytest.ini --rootdir=. --junitxml=test-results.xml \
        --cov --cov-config=.config/coveragerc --no-cov-on-fail \
        --cov-report=lcov --cov-report=term-missing
        """
 
-[[tool.hatch.envs.test.matrix]]
-python = ["3.11"]
-
-[tool.hatch.envs.lint]
-detached = true
-dependencies = ["ruff"]
+[tool.hatch.envs.hatch-static-analysis]
+dependencies = ["ruff==0.8.6"]
 
-[tool.hatch.envs.lint.scripts]
-style = [
-  "ruff check --config .config/ruff.toml --diff {args:.}",
-  "ruff format --config .config/ruff.toml --diff {args:.}",
-]
+[tool.hatch.envs.hatch-static-analysis.scripts]
+format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]
+format-fix = ["ruff format --config .config/ruff.toml {args:.}",]
+lint-check = ["ruff check --config .config/ruff.toml {args:.}",]
+lint-fix = ["ruff check --config .config/ruff.toml --fix {args:.}",]

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/requirements.txt

@@ -1,148 +1,151 @@
-#
-# This file is autogenerated by pip-compile with Python 3.11
-# by the following command:
-#
-#    pip-compile --all-extras --output-file=requirements.txt pyproject.toml
-#
-annotated-types==0.6.0
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
+annotated-types==0.7.0
     # via pydantic
-attrs==23.2.0
+attrs==24.3.0
     # via
     #   frictionless
     #   jsonlines
     #   jsonschema
+    #   referencing
 beautifulsoup4==4.12.3
     # via hdx-python-utilities (pyproject.toml)
-certifi==2024.2.2
+certifi==2024.12.14
     # via requests
 cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.3.2
+charset-normalizer==3.4.1
     # via requests
-click==8.1.7
+click==8.1.8
     # via typer
-colorama==0.4.6
-    # via typer
-coverage[toml]==7.4.4
+coverage==7.6.10
     # via pytest-cov
-distlib==0.3.8
+distlib==0.3.9
     # via virtualenv
-dnspython==2.6.1
+dnspython==2.7.0
     # via email-validator
-email-validator==2.1.1
+email-validator==2.2.0
     # via hdx-python-utilities (pyproject.toml)
-et-xmlfile==1.1.0
+et-xmlfile==2.0.0
     # via openpyxl
-filelock==3.13.1
+filelock==3.16.1
     # via virtualenv
-frictionless==5.16.1
+frictionless==5.18.0
     # via hdx-python-utilities (pyproject.toml)
 html5lib==1.1
     # via hdx-python-utilities (pyproject.toml)
-humanize==4.9.0
+humanize==4.11.0
     # via frictionless
-identify==2.5.35
+identify==2.6.5
     # via pre-commit
-idna==3.6
+idna==3.10
     # via
     #   email-validator
     #   requests
-ijson==3.2.3
+ijson==3.3.0
     # via hdx-python-utilities (pyproject.toml)
 iniconfig==2.0.0
     # via pytest
-isodate==0.6.1
+isodate==0.7.2
     # via frictionless
-jinja2==3.1.3
+jinja2==3.1.5
     # via frictionless
 jsonlines==4.0.0
     # via hdx-python-utilities (pyproject.toml)
-jsonschema==4.17.3
+jsonschema==4.23.0
     # via
     #   frictionless
     #   tableschema-to-template
-loguru==0.7.2
+jsonschema-specifications==2024.10.1
+    # via jsonschema
+loguru==0.7.3
     # via
     #   hdx-python-utilities (pyproject.toml)
     #   pytest-loguru
 markdown-it-py==3.0.0
     # via rich
-marko==2.0.3
+marko==2.1.2
     # via frictionless
-markupsafe==2.1.5
+markupsafe==3.0.2
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
-nodeenv==1.8.0
+nodeenv==1.9.1
     # via pre-commit
-openpyxl==3.1.2
+openpyxl==3.1.5
     # via hdx-python-utilities (pyproject.toml)
-packaging==24.0
+packaging==24.2
     # via pytest
 petl==1.7.15
     # via frictionless
-platformdirs==4.2.0
+platformdirs==4.3.6
     # via virtualenv
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
-pre-commit==3.6.2
+pre-commit==4.0.1
     # via hdx-python-utilities (pyproject.toml)
-pydantic==2.6.4
+pydantic==2.10.5
     # via frictionless
-pydantic-core==2.16.3
+pydantic-core==2.27.2
     # via pydantic
-pygments==2.17.2
+pygments==2.19.1
     # via rich
-pyrsistent==0.20.0
-    # via jsonschema
-pytest==8.1.1
+pyphonetics==0.5.3
+    # via hdx-python-utilities (pyproject.toml)
+pytest==8.3.4
     # via
     #   hdx-python-utilities (pyproject.toml)
     #   pytest-cov
-    #   pytest-loguru
-pytest-cov==4.1.0
+pytest-cov==6.0.0
     # via hdx-python-utilities (pyproject.toml)
-pytest-loguru==0.3.0
+pytest-loguru==0.4.0
     # via hdx-python-utilities (pyproject.toml)
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via
-    #   frictionless
     #   hdx-python-utilities (pyproject.toml)
+    #   frictionless
 python-slugify==8.0.4
     # via frictionless
-pyyaml==6.0.1
+pyyaml==6.0.2
     # via
     #   frictionless
     #   pre-commit
     #   tableschema-to-template
 ratelimit==2.2.1
     # via hdx-python-utilities (pyproject.toml)
-requests==2.31.0
+referencing==0.35.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+requests==2.32.3
     # via
     #   frictionless
     #   requests-file
-requests-file==2.0.0
+requests-file==2.1.0
     # via hdx-python-utilities (pyproject.toml)
 rfc3986==2.0.0
     # via frictionless
-rich==13.7.1
+rich==13.9.4
     # via typer
-ruamel-yaml==0.18.6
+rpds-py==0.22.3
+    # via
+    #   jsonschema
+    #   referencing
+ruamel-yaml==0.18.10
     # via hdx-python-utilities (pyproject.toml)
-ruamel-yaml-clib==0.2.8
+ruamel-yaml-clib==0.2.12
     # via ruamel-yaml
 shellingham==1.5.4
     # via typer
-simpleeval==0.9.13
+simpleeval==1.0.3
     # via frictionless
-six==1.16.0
+six==1.17.0
     # via
     #   html5lib
-    #   isodate
     #   python-dateutil
-soupsieve==2.5
+soupsieve==2.6
     # via beautifulsoup4
 stringcase==1.2.0
     # via frictionless
@@ -152,28 +155,29 @@ tabulate==0.9.0
     # via frictionless
 text-unidecode==1.3
     # via python-slugify
-typer[all]==0.9.0
+typer==0.15.1
     # via frictionless
-typing-extensions==4.10.0
+typing-extensions==4.12.2
     # via
     #   frictionless
     #   pydantic
     #   pydantic-core
     #   typer
-urllib3==2.2.1
+unidecode==1.3.8
+    # via pyphonetics
+urllib3==2.3.0
     # via requests
-validators==0.22.0
+validators==0.34.0
     # via frictionless
-virtualenv==20.25.1
+virtualenv==20.28.1
     # via pre-commit
 webencodings==0.5.1
     # via html5lib
 xlrd==2.0.1
     # via hdx-python-utilities (pyproject.toml)
+xlsx2csv==0.8.4
+    # via hdx-python-utilities (pyproject.toml)
 xlsxwriter==3.2.0
     # via tableschema-to-template
 xlwt==1.3.0
     # via hdx-python-utilities (pyproject.toml)
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/src/hdx/utilities/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.6.7'
-__version_tuple__ = version_tuple = (3, 6, 7)
+__version__ = version = '3.8.2'
+__version_tuple__ = version_tuple = (3, 8, 2)

{hdx_python_utilities-3.6.7 → hdx_python_utilities-3.8.2}/src/hdx/utilities/dateparse.py

@@ -177,8 +177,10 @@ class _ymd(list):  # pragma: no cover
             self.ystridx = len(self) - 1
 
     def _resolve_from_stridxs(self, strids):
-        """Try to resolve the identities of year/month/day elements using
-        ystridx, mstridx, and dstridx, if enough of these are specified."""
+        """
+        Try to resolve the identities of year/month/day elements using
+        ystridx, mstridx, and dstridx, if enough of these are specified.
+        """
         if len(self) == 3 and len(strids) == 2:
             # we can back out the remaining stridx value
             missing = [x for x in range(3) if x not in strids.values()]
@@ -301,8 +303,9 @@ class DateParser(dateutil.parser.parser):  # pragma: no cover
         fuzzy=False,
         fuzzy_with_tokens=False,
     ):
-        """Private method which performs the heavy lifting of parsing, called
-        from ``parse()``, which passes on its ``kwargs`` to this function.
+        """
+        Private method which performs the heavy lifting of parsing, called from
+        ``parse()``, which passes on its ``kwargs`` to this function.
 
         :param timestr:
             The string to parse.
@@ -553,7 +556,8 @@ DEFAULTPARSER = DateParser(parserinfo(dayfirst=True))
 def parse(
     timestr, default=None, ignoretz=False, tzinfos=None, **kwargs
 ):  # pragma: no cover
-    """Parse the date/time string into a :class:`datetime.datetime` object.
+    """
+    Parse the date/time string into a :class:`datetime.datetime` object.
 
     :param timestr:
         Any date/time string using the supported formats.
@@ -650,6 +654,15 @@ def now_utc() -> datetime:
     return datetime.now(timezone.utc)
 
 
+def now_utc_notz() -> datetime:
+    """Return now in UTC but with timezone removed.
+
+    Returns:
+        datetime: Now in UTC but with timezone removed
+    """
+    return datetime.now(timezone.utc).replace(tzinfo=None)
+
+
 def parse_date_range(
     string: str,
     date_format: Optional[str] = None,