hdx-python-utilities 3.6.4__tar.gz → 3.6.5__tar.gz

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/.config/pre-commit-config.yaml

@@ -2,24 +2,24 @@ default_language_version:
     python: python3.11
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
+        exclude: test_loader.py
       - id: end-of-file-fixer
-        exclude: test_csv_processing_blanks.csv
+        exclude: (test_csv_processing_blanks.csv|test.txt)
       - id: check-ast
-  - repo: https://github.com/psf/black
-    rev: 23.3.0
-    hooks:
-      - id: black
-        args: [--config, .config/black.toml]
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.267
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.14
     hooks:
+      # Run the linter.
       - id: ruff
-        args: [--config, .config/ruff.toml, --fix, --exit-non-zero-on-fix]
+        args: [--config, .config/ruff.toml, --fix]
+      # Run the formatter.
+      - id: ruff-format
+        args: [--config, .config/ruff.toml]
   - repo: https://github.com/jazzband/pip-tools
-    rev: 6.13.0
+    rev: 7.3.0
     hooks:
       - id: pip-compile
         name: pip-compile requirements.txt

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/.config/ruff.toml

@@ -1,14 +1,17 @@
 line-length = 79
 exclude = ["_version.py"]
-ignore = [
-    "E501" # Line too long
-]
-# List of rules: https://beta.ruff.rs/docs/rules/
+
+[lint]
+# List of rules: https://docs.astral.sh/ruff/rules/
 select = [
   "E",   # pycodestyle - default
   "F",   # pyflakes - default
   "I"    # isort
 ]
+ignore = [
+    "E501" # Line too long
+]
+
 
-[isort]
+[lint.isort]
 known-local-folder = ["hdx"]

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/.gitignore

@@ -96,3 +96,5 @@ ENV/
 
 # hatch-vcs
 _version.py
+
+.vscode/settings.json

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/CONTRIBUTING.md

@@ -21,14 +21,13 @@ you make a git commit:
     pre-commit install
 
 The configuration file for this project is in a
-non-start location. Thus, you will need to edit your
+non-standard location. Thus, you will need to edit your
 `.git/hooks/pre-commit` file to reflect this. Change
 the line that begins with `ARGS` to:
 
     ARGS=(hook-impl --config=.config/pre-commit-config.yaml --hook-type=pre-commit)
 
 With pre-commit, all code is formatted according to
-[black]("https://github.com/psf/black") and
 [ruff]("https://github.com/charliermarsh/ruff") guidelines.
 
 To check if your changes pass pre-commit without committing, run:
@@ -41,7 +40,7 @@ To run the tests and view coverage, execute:
 
     pytest -c .config/pytest.ini --cov hdx --cov-config .config/coveragerc
 
-Follow the example set out already in ``api.rst`` as you write the documentation.
+Follow the example set out already in ``documentation/main.md`` as you write the documentation.
 
 ## Packages
 

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hdx-python-utilities
-Version: 3.6.4
+Version: 3.6.5
 Summary: HDX Python Utilities for streaming tabular data, date and time handling and other helpful functions
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-utilities
 Author-email: Michael Rans <rans@email.com>
@@ -25,7 +25,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
-Requires-Dist: frictionless>=5.16.0
+Requires-Dist: frictionless>=5.16.1
 Requires-Dist: ijson>=3.2.3
 Requires-Dist: jsonlines>=4.0.0
 Requires-Dist: loguru

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/pyproject.toml

@@ -36,7 +36,7 @@ requires-python = ">=3.8"
 # Extras for frictionless[excel,json] added explicitly
 # for conda-forge compatibility
 dependencies = [
-    "frictionless>=5.16.0",
+    "frictionless>=5.16.1",
     # frictionless[excel]
     "openpyxl>=3.1.2",
     "tableschema-to-template>=0.0.13",
@@ -106,18 +106,10 @@ python = ["3.11"]
 
 [tool.hatch.envs.lint]
 detached = true
-dependencies = [
-  "black",
-  "ruff",
-]
+dependencies = ["ruff"]
 
 [tool.hatch.envs.lint.scripts]
 style = [
-  "ruff --config .config/ruff.toml {args:.}",
-  "black --config .config/black.toml --check --diff {args:.}",
-]
-# Not used for anything at the moment
-fmt = [
-  "black --config .config/black.toml {args:.}",
-  "ruff --config .config/ruff.toml --fix {args:.}",
+  "ruff check --config .config/ruff.toml --diff {args:.}",
+  "ruff format --config .config/ruff.toml --diff {args:.}",
 ]

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/requirements.txt

@@ -2,18 +2,18 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --all-extras --output-file=requirements.txt --resolver=backtracking pyproject.toml
+#    pip-compile --all-extras --output-file=requirements.txt pyproject.toml
 #
 annotated-types==0.6.0
     # via pydantic
-attrs==23.1.0
+attrs==23.2.0
     # via
     #   frictionless
     #   jsonlines
     #   jsonschema
-beautifulsoup4==4.12.2
+beautifulsoup4==4.12.3
     # via hdx-python-utilities (pyproject.toml)
-certifi==2023.11.17
+certifi==2024.2.2
     # via requests
 cfgv==3.4.0
     # via pre-commit
@@ -25,11 +25,13 @@ click==8.1.7
     # via typer
 colorama==0.4.6
     # via typer
-coverage[toml]==7.3.2
-    # via pytest-cov
+coverage[toml]==7.4.1
+    # via
+    #   coverage
+    #   pytest-cov
 distlib==0.3.8
     # via virtualenv
-dnspython==2.4.2
+dnspython==2.5.0
     # via email-validator
 email-validator==2.1.0.post1
     # via hdx-python-utilities (pyproject.toml)
@@ -37,13 +39,13 @@ et-xmlfile==1.1.0
     # via openpyxl
 filelock==3.13.1
     # via virtualenv
-frictionless==5.16.0
+frictionless==5.16.1
     # via hdx-python-utilities (pyproject.toml)
 html5lib==1.1
     # via hdx-python-utilities (pyproject.toml)
 humanize==4.9.0
     # via frictionless
-identify==2.5.33
+identify==2.5.34
     # via pre-commit
 idna==3.6
     # via
@@ -55,7 +57,7 @@ iniconfig==2.0.0
     # via pytest
 isodate==0.6.1
     # via frictionless
-jinja2==3.1.2
+jinja2==3.1.3
     # via frictionless
 jsonlines==4.0.0
     # via hdx-python-utilities (pyproject.toml)
@@ -71,7 +73,7 @@ markdown-it-py==3.0.0
     # via rich
 marko==2.0.2
     # via frictionless
-markupsafe==2.1.3
+markupsafe==2.1.5
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
@@ -83,21 +85,21 @@ packaging==23.2
     # via pytest
 petl==1.7.14
     # via frictionless
-platformdirs==4.1.0
+platformdirs==4.2.0
     # via virtualenv
-pluggy==1.3.0
+pluggy==1.4.0
     # via pytest
-pre-commit==3.6.0
+pre-commit==3.6.1
     # via hdx-python-utilities (pyproject.toml)
-pydantic==2.5.2
+pydantic==2.6.1
     # via frictionless
-pydantic-core==2.14.5
+pydantic-core==2.16.2
     # via pydantic
 pygments==2.17.2
     # via rich
 pyrsistent==0.20.0
     # via jsonschema
-pytest==7.4.3
+pytest==8.0.0
     # via
     #   hdx-python-utilities (pyproject.toml)
     #   pytest-cov
@@ -110,7 +112,7 @@ python-dateutil==2.8.2
     # via
     #   frictionless
     #   hdx-python-utilities (pyproject.toml)
-python-slugify==8.0.1
+python-slugify==8.0.4
     # via frictionless
 pyyaml==6.0.1
     # via
@@ -123,13 +125,13 @@ requests==2.31.0
     # via
     #   frictionless
     #   requests-file
-requests-file==1.5.1
+requests-file==2.0.0
     # via hdx-python-utilities (pyproject.toml)
 rfc3986==2.0.0
     # via frictionless
 rich==13.7.0
     # via typer
-ruamel-yaml==0.18.5
+ruamel-yaml==0.18.6
     # via hdx-python-utilities (pyproject.toml)
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
@@ -142,7 +144,6 @@ six==1.16.0
     #   html5lib
     #   isodate
     #   python-dateutil
-    #   requests-file
 soupsieve==2.5
     # via beautifulsoup4
 stringcase==1.2.0
@@ -154,14 +155,16 @@ tabulate==0.9.0
 text-unidecode==1.3
     # via python-slugify
 typer[all]==0.9.0
-    # via frictionless
+    # via
+    #   frictionless
+    #   typer
 typing-extensions==4.9.0
     # via
     #   frictionless
     #   pydantic
     #   pydantic-core
     #   typer
-urllib3==2.1.0
+urllib3==2.2.0
     # via requests
 validators==0.22.0
     # via frictionless

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/src/hdx/utilities/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.6.4'
-__version_tuple__ = version_tuple = (3, 6, 4)
+__version__ = version = '3.6.5'
+__version_tuple__ = version_tuple = (3, 6, 5)

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/src/hdx/utilities/compare.py

@@ -4,7 +4,9 @@ from os import linesep
 from typing import List
 
 
-def compare_files(path1: str, path2: str) -> List[str]:
+def compare_files(
+    path1: str, path2: str, encoding: str = "utf-8"
+) -> List[str]:
     """Returns the delta between two files using -, ?, + format excluding lines
     that are the same.
 
@@ -16,12 +18,13 @@ def compare_files(path1: str, path2: str) -> List[str]:
         List[str]: Delta between the two files
     """
     diff = difflib.ndiff(
-        open(path1).read().splitlines(), open(path2).read().splitlines()
+        open(path1, encoding=encoding).read().splitlines(),
+        open(path2, encoding=encoding).read().splitlines(),
     )
     return [x for x in diff if x[0] in ["-", "+", "?"]]
 
 
-def assert_files_same(path1: str, path2: str) -> None:
+def assert_files_same(path1: str, path2: str, encoding: str = "utf-8") -> None:
     """Asserts that two files are the same and returns delta using.
 
     -, ?, + format if not
@@ -33,5 +36,5 @@ def assert_files_same(path1: str, path2: str) -> None:
     Returns:
         None
     """
-    difflines = compare_files(path1, path2)
+    difflines = compare_files(path1, path2, encoding)
     assert len(difflines) == 0, linesep.join([linesep] + difflines)

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/src/hdx/utilities/email.py

@@ -151,7 +151,7 @@ class Email:
 
     @staticmethod
     def get_normalised_emails(
-        recipients: Union[str, ListTuple[str]]
+        recipients: Union[str, ListTuple[str]],
     ) -> List[str]:
         """Get list of normalised emails.
 

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/src/hdx/utilities/loader.py

@@ -1,8 +1,8 @@
 """Loading utilities for YAML, JSON etc."""
 
 import json
-from os import linesep
 from typing import Any, Dict, Optional
+from warnings import warn
 
 from ruamel.yaml import YAML
 
@@ -19,7 +19,9 @@ def load_text(
     encoding: str = "utf-8",
     strip: bool = False,
     replace_newlines: Optional[str] = None,
+    replace_line_separators: Optional[str] = None,
     loaderror_if_empty: bool = True,
+    default_line_separator: str = "\n",
 ) -> str:
     """Load file into a string removing newlines.
 
@@ -27,16 +29,28 @@ def load_text(
         path (str): Path to file
         encoding (str): Encoding of file. Defaults to utf-8.
         strip (bool): Whether to strip whitespace from start and end. Defaults to False.
-        replace_newlines (Optional[str]): String with which tp replace newlines. Defaults to None (don't replace).
+        replace_newlines (Optional[str]): String with which to replace newlines. Defaults to None (don't replace). (deprecated 2024-02-07)
+        replace_line_separators (Optional[str]): String with which to replace newlines. Defaults to None (don't replace).
         loaderror_if_empty (bool): Whether to raise LoadError if file is empty. Default to True.
+        default_line_separator (str): line separator to be replaced if replace_line_separators is not None
 
     Returns:
         str: String contents of file
     """
+    if replace_newlines is not None:
+        warn(
+            "Keyword argument 'replace_newlines' to 'load_text' is deprecated on 2024-02-08 "
+            "in favour of 'replace_line_separators'",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        replace_line_separators = replace_newlines
     with open(path, encoding=encoding) as f:
         string = f.read()
-        if replace_newlines is not None:
-            string = string.replace(linesep, replace_newlines)
+        if replace_line_separators is not None:
+            string = string.replace(
+                default_line_separator, replace_line_separators
+            )
         if strip:
             string = string.strip()
     if not string:

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/tests/fixtures/compare/test_csv_processing.csv

@@ -1,4 +1,4 @@
-la1    ,ha1    ,ba1    ,ma1
+la1    ,hä1    ,ba1    ,ma1
 header1,header2,header3,header4
 coal   ,3      ,7.4    ,'needed'
 gas    ,2      ,6.5    ,'n/a'

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/tests/fixtures/compare/test_csv_processing2.csv

@@ -1,4 +1,4 @@
-la1    ,ha1    ,ba1    ,ma1
+la1    ,hä1    ,ba1    ,ma1
 header1,header2,header3,header4
 coal   ,1      ,7.4    ,'notneeded'
 gas    ,2      ,6.5    ,'n/a'

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/tests/hdx/utilities/test_compare.py

@@ -16,7 +16,7 @@ class TestCompare:
         return join(fixturesfolder, "compare", "test_csv_processing2.csv")
 
     def test_compare_files(self, testfile1, testfile2):
-        result = compare_files(testfile1, testfile2)
+        result = compare_files(testfile1, testfile2, encoding="utf-8")
         assert result == [
             "- coal   ,3      ,7.4    ,'needed'",
             "?         ^\n",

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/tests/hdx/utilities/test_dateparse.py

@@ -23,28 +23,32 @@ class TestDateParse:
         )
 
     def test_parse_date_range(self):
-        result = datetime(2013, 2, 10, 0, 0, tzinfo=timezone.utc), datetime(
-            2013, 2, 10, 0, 0, tzinfo=timezone.utc
+        result = (
+            datetime(2013, 2, 10, 0, 0, tzinfo=timezone.utc),
+            datetime(2013, 2, 10, 0, 0, tzinfo=timezone.utc),
         )
         assert parse_date_range("10/02/2013") == result
         assert parse_date_range("2013/02/10") == result
-        result = datetime(2013, 2, 20, 10, 0, tzinfo=timezone.utc), datetime(
-            2013, 2, 20, 10, 0, tzinfo=timezone.utc
+        result = (
+            datetime(2013, 2, 20, 10, 0, tzinfo=timezone.utc),
+            datetime(2013, 2, 20, 10, 0, tzinfo=timezone.utc),
         )
         assert parse_date_range("20/02/2013 10:00:00") == result
         assert (
             parse_date_range("20/02/2013 10:00:00", "%d/%m/%Y %H:%M:%S")
             == result
         )
-        result = datetime(2013, 2, 20, 0, 0, tzinfo=timezone.utc), datetime(
-            2013, 2, 20, 0, 0, tzinfo=timezone.utc
+        result = (
+            datetime(2013, 2, 20, 0, 0, tzinfo=timezone.utc),
+            datetime(2013, 2, 20, 0, 0, tzinfo=timezone.utc),
         )
         assert parse_date_range("20/02/2013") == result
         assert (
             parse_date_range("20/02/2013 10:00:00", zero_time=True) == result
         )
-        result2 = datetime(2013, 2, 20, 0, 0, tzinfo=timezone.utc), datetime(
-            2013, 2, 20, 23, 59, 59, tzinfo=timezone.utc
+        result2 = (
+            datetime(2013, 2, 20, 0, 0, tzinfo=timezone.utc),
+            datetime(2013, 2, 20, 23, 59, 59, tzinfo=timezone.utc),
         )
         assert (
             parse_date_range(
@@ -103,8 +107,9 @@ class TestDateParse:
         )
         fuzzyresult["nondate"] = None
         assert fuzzy == fuzzyresult
-        result = datetime(2013, 2, 1, 0, 0, tzinfo=timezone.utc), datetime(
-            2013, 2, 28, 0, 0, tzinfo=timezone.utc
+        result = (
+            datetime(2013, 2, 1, 0, 0, tzinfo=timezone.utc),
+            datetime(2013, 2, 28, 0, 0, tzinfo=timezone.utc),
         )
         assert parse_date_range("02/2013") == result
         assert parse_date_range("02/2013", "%m/%Y") == result
@@ -119,8 +124,9 @@ class TestDateParse:
             "nondate": ("date is ", " for this test"),
             "date": ("02/2013",),
         }
-        result = datetime(2013, 1, 1, 0, 0, tzinfo=timezone.utc), datetime(
-            2013, 12, 31, 0, 0, tzinfo=timezone.utc
+        result = (
+            datetime(2013, 1, 1, 0, 0, tzinfo=timezone.utc),
+            datetime(2013, 12, 31, 0, 0, tzinfo=timezone.utc),
         )
         assert parse_date_range("2013") == result
         assert parse_date_range("2013", "%Y") == result

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/tests/hdx/utilities/test_downloader.py

@@ -403,7 +403,7 @@ class TestDownloader:
                 filename=filename,
             )
             fpath = abspath(f)
-            with open(fpath) as fi:
+            with open(fpath, encoding="utf-8") as fi:
                 text = fi.read()
                 assert '"id": "10"' in text
                 assert '"lala": "a"' in text
@@ -419,7 +419,7 @@ class TestDownloader:
                 filename=filename,
             )
             fpath = abspath(f)
-            with open(fpath) as fi:
+            with open(fpath, encoding="utf-8") as fi:
                 text = fi.read()
                 assert '"id": "3"' in text
                 assert '"lala": "b"' in text

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/tests/hdx/utilities/test_loader.py

@@ -172,7 +172,7 @@ test"""
             assert result == TestLoader.text
             result = load_text(text_file, strip=True)
             assert result == TestLoader.expected_text_strip
-            result = load_text(text_file, replace_newlines=" ")
+            result = load_text(text_file, replace_line_separators=" ")
             assert result == TestLoader.expected_text_newlines_to_spaces
             with pytest.raises(IOError):
                 load_text(join(tmpdir, "NOTEXIST.txt"))

{hdx_python_utilities-3.6.4 → hdx_python_utilities-3.6.5}/tests/hdx/utilities/test_path.py

@@ -272,7 +272,7 @@ class TestPath:
             (
                 0,
                 {
-                    "folder": "/tmp/gaga/0",
+                    "folder": join(expected_dir, "0"),
                     "batch": "1234",
                     "progress": "emergency_id=911",
                 },
@@ -281,7 +281,7 @@ class TestPath:
             (
                 1,
                 {
-                    "folder": "/tmp/gaga/1",
+                    "folder": join(expected_dir, "1"),
                     "batch": "1234",
                     "progress": "iso3=AFG",
                 },
@@ -290,7 +290,7 @@ class TestPath:
             (
                 1,
                 {
-                    "folder": "/tmp/gaga/1",
+                    "folder": join(expected_dir, "1"),
                     "batch": "1234",
                     "progress": "iso3=SDN",
                 },
@@ -299,7 +299,7 @@ class TestPath:
             (
                 1,
                 {
-                    "folder": "/tmp/gaga/1",
+                    "folder": join(expected_dir, "1"),
                     "batch": "1234",
                     "progress": "iso3=YEM",
                 },
@@ -308,7 +308,7 @@ class TestPath:
             (
                 1,
                 {
-                    "folder": "/tmp/gaga/1",
+                    "folder": join(expected_dir, "1"),
                     "batch": "1234",
                     "progress": "iso3=ZAM",
                 },

hdx_python_utilities-3.6.4/.config/black.toml

@@ -1,2 +0,0 @@
-[tool.black]
-line-length = 79