hdx-python-api 6.2.6__tar.gz → 6.2.8__tar.gz

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/.config/pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
         exclude: test_scraper_.*\.json
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.2
+    rev: v0.4.2
     hooks:
       # Run the linter.
       - id: ruff
@@ -17,10 +17,11 @@ repos:
       # Run the formatter.
       - id: ruff-format
         args: [--config, .config/ruff.toml]
-  - repo: https://github.com/jazzband/pip-tools
-    rev: 7.4.1
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.1.38
     hooks:
+      # Run the pip compile
       - id: pip-compile
         name: pip-compile requirements.txt
         files: pyproject.toml
-        args: [pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt]
+        args: [ pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt ]

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.3
 Name: hdx-python-api
-Version: 6.2.6
+Version: 6.2.8
 Summary: HDX Python API for interacting with the Humanitarian Data Exchange
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-api
-Author-email: Michael Rans <rans@email.com>
+Author: Michael Rans
 License: MIT
 License-File: LICENSE
 Keywords: API,CKAN,HDX,humanitarian data exchange
@@ -25,11 +25,11 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
-Requires-Dist: ckanapi>=4.7
+Requires-Dist: ckanapi>=4.8
 Requires-Dist: defopt>=6.4.0
 Requires-Dist: email-validator
-Requires-Dist: hdx-python-country>=3.6.9
-Requires-Dist: hdx-python-utilities>=3.6.7
+Requires-Dist: hdx-python-country>=3.7.0
+Requires-Dist: hdx-python-utilities>=3.6.8
 Requires-Dist: libhxl>=5.2.1
 Requires-Dist: makefun
 Requires-Dist: ndg-httpsclient

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/documentation/.readthedocs.yaml

@@ -11,5 +11,6 @@ build:
   jobs:
     pre_build:
       - pip install --upgrade mkdocs
+      - pip install mkdocs-material
       - pip install pydoc-markdown
       - pydoc-markdown documentation/pydoc-markdown.yaml

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/documentation/main.md

@@ -52,6 +52,13 @@ The library has detailed API documentation which can be found in the menu at the
 
 
 ## Breaking Changes
+From 6.2.8, fix mark_data_updated which was broken due to an error in 
+dataset_update_filestore_resource in which timezone information was 
+incorrectly added to the iso formatted string
+
+From 6.2.7, generate_resource_from_iterator renamed to
+generate_resource_from_iterable with requirement of iterable rather iterator
+
 From 6.2.6, kwargs take preference over environment variables which take
 preference over configuration files
 

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/documentation/pydoc-markdown.yaml

@@ -11,7 +11,7 @@ renderer:
   output_directory: docs
   mkdocs_config:
     site_name: HDX Python API
-    theme: mkdocs
+    theme: material
     repo_url: "https://github.com/OCHA-DAP/hdx-python-api"
   markdown:
     source_linker:

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/pyproject.toml

@@ -9,7 +9,7 @@ build-backend = "hatchling.build"
 [project]
 name = "hdx-python-api"
 description = "HDX Python API for interacting with the Humanitarian Data Exchange"
-authors = [{name = "Michael Rans", email = "rans@email.com"}]
+authors = [{name = "Michael Rans"}]
 license = {text = "MIT"}
 keywords = ["HDX", "API", "CKAN", "humanitarian data exchange"]
 classifiers = [
@@ -34,11 +34,11 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "ckanapi>=4.7",
+    "ckanapi>=4.8",
     "defopt>=6.4.0",
     "email_validator",
-    "hdx-python-country>=3.6.9",
-    "hdx-python-utilities>=3.6.7",
+    "hdx-python-country>=3.7.0",
+    "hdx-python-utilities>=3.6.8",
     "libhxl>=5.2.1",
     "makefun",
     "ndg-httpsclient",

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/requirements.txt

@@ -1,9 +1,5 @@
-#
-# This file is autogenerated by pip-compile with Python 3.11
-# by the following command:
-#
-#    pip-compile --all-extras --output-file=requirements.txt pyproject.toml
-#
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
 annotated-types==0.6.0
     # via pydantic
 attrs==23.2.0
@@ -23,18 +19,14 @@ chardet==5.2.0
     # via frictionless
 charset-normalizer==3.3.2
     # via requests
-ckanapi==4.7
-    # via hdx-python-api (pyproject.toml)
+ckanapi==4.8
 click==8.1.7
     # via typer
-colorama==0.4.6
-    # via typer
-coverage[toml]==7.4.4
+coverage==7.5.0
     # via pytest-cov
 cryptography==42.0.5
     # via pyopenssl
 defopt==6.4.0
-    # via hdx-python-api (pyproject.toml)
 distlib==0.3.8
     # via virtualenv
 dnspython==2.6.1
@@ -43,41 +35,36 @@ docopt==0.6.2
     # via
     #   ckanapi
     #   num2words
-docutils==0.20.1
+docutils==0.21.2
     # via defopt
 email-validator==2.1.1
-    # via hdx-python-api (pyproject.toml)
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.13.1
+filelock==3.13.4
     # via virtualenv
 frictionless==5.16.1
     # via hdx-python-utilities
-google-auth==2.28.2
+google-auth==2.29.0
     # via
     #   google-auth-oauthlib
     #   gspread
 google-auth-oauthlib==1.2.0
     # via gspread
-gspread==6.0.2
-    # via hdx-python-api (pyproject.toml)
-hdx-python-country==3.6.9
-    # via hdx-python-api (pyproject.toml)
-hdx-python-utilities==3.6.7
-    # via
-    #   hdx-python-api (pyproject.toml)
-    #   hdx-python-country
+gspread==6.1.0
+hdx-python-country==3.7.0
+hdx-python-utilities==3.6.8
+    # via hdx-python-country
 humanize==4.9.0
     # via frictionless
-identify==2.5.35
+identify==2.5.36
     # via pre-commit
-idna==3.6
+idna==3.7
     # via
     #   email-validator
     #   requests
 ijson==3.2.3
     # via hdx-python-utilities
-inflect==7.0.0
+inflect==7.2.1
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
@@ -94,13 +81,10 @@ jsonschema==4.17.3
     #   frictionless
     #   tableschema-to-template
 libhxl==5.2.1
-    # via
-    #   hdx-python-api (pyproject.toml)
-    #   hdx-python-country
+    # via hdx-python-country
 loguru==0.7.2
     # via hdx-python-utilities
 makefun==1.15.2
-    # via hdx-python-api (pyproject.toml)
 markdown-it-py==3.0.0
     # via rich
 marko==2.0.3
@@ -109,8 +93,9 @@ markupsafe==2.1.5
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
+more-itertools==10.2.0
+    # via inflect
 ndg-httpsclient==0.5.1
-    # via hdx-python-api (pyproject.toml)
 nodeenv==1.8.0
     # via pre-commit
 num2words==0.5.13
@@ -123,9 +108,9 @@ packaging==24.0
     # via pytest
 petl==1.7.15
     # via frictionless
-platformdirs==4.2.0
+platformdirs==4.2.1
     # via virtualenv
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
 ply==3.11
     # via
@@ -133,40 +118,31 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==3.6.2
-    # via hdx-python-api (pyproject.toml)
-pyasn1==0.5.1
+pre-commit==3.7.0
+pyasn1==0.6.0
     # via
-    #   hdx-python-api (pyproject.toml)
     #   ndg-httpsclient
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.3.0
+pyasn1-modules==0.4.0
     # via google-auth
-pycparser==2.21
+pycparser==2.22
     # via cffi
-pydantic==2.6.4
-    # via
-    #   frictionless
-    #   inflect
-pydantic-core==2.16.3
+pydantic==2.7.1
+    # via frictionless
+pydantic-core==2.18.2
     # via pydantic
 pygments==2.17.2
     # via rich
 pyopenssl==24.1.0
-    # via
-    #   hdx-python-api (pyproject.toml)
-    #   ndg-httpsclient
+    # via ndg-httpsclient
 pyphonetics==0.5.3
     # via hdx-python-country
 pyrsistent==0.20.0
     # via jsonschema
 pytest==8.1.1
-    # via
-    #   hdx-python-api (pyproject.toml)
-    #   pytest-cov
-pytest-cov==4.1.0
-    # via hdx-python-api (pyproject.toml)
+    # via pytest-cov
+pytest-cov==5.0.0
 python-dateutil==2.8.2
     # via
     #   frictionless
@@ -183,21 +159,19 @@ pyyaml==6.0.1
     #   frictionless
     #   pre-commit
     #   tableschema-to-template
-quantulum3==0.9.0
-    # via hdx-python-api (pyproject.toml)
+quantulum3==0.9.1
 ratelimit==2.2.1
     # via hdx-python-utilities
 requests==2.31.0
     # via
     #   ckanapi
     #   frictionless
-    #   hdx-python-api (pyproject.toml)
     #   libhxl
     #   requests-file
     #   requests-oauthlib
 requests-file==2.0.0
     # via hdx-python-utilities
-requests-oauthlib==1.4.0
+requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
@@ -209,10 +183,16 @@ ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
+setuptools==69.5.1
+    # via
+    #   ckanapi
+    #   nodeenv
 shellingham==1.5.4
     # via typer
 simpleeval==0.9.13
     # via frictionless
+simplejson==3.19.2
+    # via ckanapi
 six==1.16.0
     # via
     #   ckanapi
@@ -234,14 +214,17 @@ tabulate==0.9.0
     # via frictionless
 text-unidecode==1.3
     # via python-slugify
-typer[all]==0.9.0
+typeguard==4.2.1
+    # via inflect
+typer==0.12.3
     # via frictionless
-typing-extensions==4.10.0
+typing-extensions==4.11.0
     # via
     #   frictionless
     #   inflect
     #   pydantic
     #   pydantic-core
+    #   typeguard
     #   typer
 unidecode==1.3.8
     # via
@@ -251,9 +234,9 @@ urllib3==2.2.1
     # via
     #   libhxl
     #   requests
-validators==0.23.1
+validators==0.28.1
     # via frictionless
-virtualenv==20.25.1
+virtualenv==20.26.0
     # via pre-commit
 wheel==0.43.0
     # via libhxl
@@ -265,6 +248,3 @@ xlsxwriter==3.2.0
     # via tableschema-to-template
 xlwt==1.3.0
     # via hdx-python-utilities
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/src/hdx/api/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '6.2.6'
-__version_tuple__ = version_tuple = (6, 2, 6)
+__version__ = version = '6.2.8'
+__version_tuple__ = version_tuple = (6, 2, 8)

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/src/hdx/data/dataset.py

@@ -5,7 +5,7 @@ import logging
 import sys
 import warnings
 from copy import deepcopy
-from datetime import datetime, timezone
+from datetime import datetime
 from os.path import isfile, join
 from typing import (
     TYPE_CHECKING,
@@ -41,6 +41,7 @@ from hdx.utilities.dateparse import (
     default_date,
     default_enddate,
     now_utc,
+    now_utc_notz,
     parse_date,
     parse_date_range,
 )
@@ -598,9 +599,9 @@ class Dataset(HDXObject):
             del kwargs["updated_by_script"]
         else:
             scriptinfo = self.configuration.get_user_agent()
-        # No need to output timezone info here
+        # Should not output timezone info here
         data["updated_by_script"] = (
-            f"{scriptinfo} ({datetime.now(timezone.utc).replace(tzinfo=None).isoformat(timespec='microseconds')})"
+            f"{scriptinfo} ({now_utc_notz().isoformat(timespec='microseconds')})"
         )
         batch = kwargs.get("batch")
         if batch:
@@ -2601,10 +2602,10 @@ class Dataset(HDXObject):
             encoding=encoding,
         )
 
-    def generate_resource_from_iterator(
+    def generate_resource_from_iterable(
         self,
         headers: ListTuple[str],
-        iterator: Iterator[Union[ListTuple, Dict]],
+        iterable: Iterable[Union[ListTuple, Dict]],
         hxltags: Dict[str, str],
         folder: str,
         filename: str,
@@ -2615,7 +2616,7 @@ class Dataset(HDXObject):
         quickcharts: Optional[Dict] = None,
         encoding: Optional[str] = None,
     ) -> Tuple[bool, Dict]:
-        """Given headers and an iterator, write rows to csv and create
+        """Given headers and an iterable, write rows to csv and create
         resource, adding to it the dataset. The returned dictionary will
         contain the resource in the key resource, headers in the key headers
         and list of rows in the key rows.
@@ -2652,7 +2653,7 @@ class Dataset(HDXObject):
 
         Args:
             headers (ListTuple[str]): Headers
-            iterator (Iterator[Union[ListTuple,Dict]]): Iterator returning rows
+            iterable (Iterable[Union[ListTuple,Dict]]): Iterable returning rows
             hxltags (Dict[str,str]): Header to HXL hashtag mapping
             folder (str): Folder to which to write file containing rows
             filename (str): Filename of file to write rows
@@ -2738,7 +2739,7 @@ class Dataset(HDXObject):
 
             date_function = datecol_function
 
-        for row in iterator:
+        for row in iterable:
             if date_function is not None:
                 result = date_function(row)
                 if result is None:
@@ -2813,6 +2814,38 @@ class Dataset(HDXObject):
                     retdict["qc_resource"] = resource
         return True, retdict
 
+    def generate_resource_from_iterator(
+        self,
+        headers: ListTuple[str],
+        iterator: Iterator[Union[ListTuple, Dict]],
+        hxltags: Dict[str, str],
+        folder: str,
+        filename: str,
+        resourcedata: Dict,
+        datecol: Optional[Union[int, str]] = None,
+        yearcol: Optional[Union[int, str]] = None,
+        date_function: Optional[Callable[[Dict], Optional[Dict]]] = None,
+        quickcharts: Optional[Dict] = None,
+        encoding: Optional[str] = None,
+    ) -> Tuple[bool, Dict]:
+        warnings.warn(
+            "generate_resource_from_iterator() is deprecated, use generate_resource_from_iterable() instead",
+            DeprecationWarning,
+        )
+        return self.generate_resource_from_iterable(
+            headers,
+            iterator,
+            hxltags,
+            folder,
+            filename,
+            resourcedata,
+            datecol,
+            yearcol,
+            date_function,
+            quickcharts,
+            encoding,
+        )
+
     def download_and_generate_resource(
         self,
         downloader: BaseDownload,
@@ -2898,7 +2931,7 @@ class Dataset(HDXObject):
             format="csv",
             **kwargs,
         )
-        return self.generate_resource_from_iterator(
+        return self.generate_resource_from_iterable(
             headers,
             iterator,
             hxltags,

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/src/hdx/data/filestore_helper.py

@@ -1,8 +1,9 @@
 """Helper to the Dataset class for handling resources with filestores."""
 
-from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any, Dict
 
+from hdx.utilities.dateparse import now_utc_notz
+
 if TYPE_CHECKING:
     from hdx.data.resource import Resource
 
@@ -90,7 +91,8 @@ class FilestoreHelper:
 
         data_updated = resource_data_to_update.is_marked_data_updated()
         if data_updated:
-            resource_data_to_update["last_modified"] = datetime.now(
-                timezone.utc
-            ).isoformat(timespec="microseconds")
+            # Should not output timezone info here
+            resource_data_to_update["last_modified"] = (
+                now_utc_notz().isoformat(timespec="microseconds")
+            )
             resource_data_to_update.data_updated = False

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/src/hdx/data/resource.py

@@ -2,7 +2,7 @@
 
 import logging
 import warnings
-from datetime import datetime, timezone
+from datetime import datetime
 from os.path import join
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union
@@ -13,7 +13,7 @@ from hdx.api.configuration import Configuration
 from hdx.data.date_helper import DateHelper
 from hdx.data.hdxobject import HDXError, HDXObject
 from hdx.data.resource_view import ResourceView
-from hdx.utilities.dateparse import now_utc, parse_date
+from hdx.utilities.dateparse import now_utc, now_utc_notz, parse_date
 from hdx.utilities.downloader import Download
 from hdx.utilities.typehint import ListTuple
 from hdx.utilities.uuid import is_valid_uuid
@@ -393,9 +393,10 @@ class Resource(HDXObject):
         """
         data_updated = kwargs.pop("data_updated", self.data_updated)
         if data_updated and not self.file_to_upload:
-            self.old_data["last_modified"] = datetime.now(
-                timezone.utc
-            ).isoformat(timespec="microseconds")
+            # Should not output timezone info here
+            self.old_data["last_modified"] = now_utc_notz().isoformat(
+                timespec="microseconds"
+            )
             self.data_updated = False
             # old_data will be merged into data in the next step
         self._merge_hdx_update(

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/tests/hdx/api/test_ckan.py

@@ -176,7 +176,9 @@ class TestCKAN:
         assert dataset["title"] == title
         assert dataset.get_tags() == tags
         assert dataset.get_maintainer()["id"] == maintainer_id
-        assert dataset.get_organization()["display_name"] == "INNAGO"
+        assert (
+            dataset.get_organization()["display_name"] == "INNAGO (inactive)"
+        )
         resources = dataset.get_resources()
         for i, resource in enumerate(resources):
             assert resource["name"] == f"test_resource_{i}"
@@ -227,7 +229,9 @@ class TestCKAN:
         assert dataset["caveats"] == caveats
         assert dataset.get_tags() == tags
         assert dataset.get_maintainer()["id"] == maintainer_id
-        assert dataset.get_organization()["display_name"] == "INNAGO"
+        assert (
+            dataset.get_organization()["display_name"] == "INNAGO (inactive)"
+        )
         updated_resources = dataset.get_resources()
         for i, updated_resource in enumerate(updated_resources):
             resource = resources[i]
@@ -269,7 +273,9 @@ class TestCKAN:
         assert "caveats" not in dataset
         assert dataset.get_tags() == tags
         assert dataset.get_maintainer()["id"] == maintainer_id
-        assert dataset.get_organization()["display_name"] == "INNAGO"
+        assert (
+            dataset.get_organization()["display_name"] == "INNAGO (inactive)"
+        )
         updated_resources = dataset.get_resources()
         for i, updated_resource in enumerate(updated_resources):
             resource = resources[i]

{hdx_python_api-6.2.6 → hdx_python_api-6.2.8}/tests/hdx/data/__init__.py

@@ -96,6 +96,16 @@ resources_data = [
     },
 ]
 
+resource_data = {
+    "name": "MyResource1",
+    "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d",
+    "format": "xlsx",
+    "url": "http://test/spreadsheet.xlsx",
+    "description": "My Resource",
+    "api_type": "api",
+    "resource_type": "api",
+}
+
 organization_data = {
     "name": "MyOrganization1",
     "title": "Humanitarian Organization",

hdx_python_api-6.2.8/tests/hdx/data/test_filestore_helper.py

@@ -0,0 +1,49 @@
+import copy
+import re
+
+from . import resource_data
+from hdx.data.filestore_helper import FilestoreHelper
+from hdx.data.resource import Resource
+
+
+class TestFilestoreHelper:
+    def test_dataset_update_filestore_resource(self, configuration):
+        resource_data_copy = copy.deepcopy(resource_data)
+        resource = Resource(resource_data_copy)
+        filestore_resources = {}
+        FilestoreHelper.dataset_update_filestore_resource(
+            resource, filestore_resources, 0
+        )
+        assert resource == {
+            "api_type": "api",
+            "description": "My Resource",
+            "format": "xlsx",
+            "name": "MyResource1",
+            "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d",
+            "resource_type": "api",
+            "url": "http://test/spreadsheet.xlsx",
+        }
+        assert filestore_resources == {}
+
+        resource.set_file_to_upload("test")
+        FilestoreHelper.dataset_update_filestore_resource(
+            resource, filestore_resources, 0
+        )
+        assert resource == {
+            "api_type": "api",
+            "description": "My Resource",
+            "format": "xlsx",
+            "name": "MyResource1",
+            "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d",
+            "resource_type": "api",
+            "url": "updated_by_file_upload_step",
+        }
+        assert filestore_resources == {0: "test"}
+
+        resource.mark_data_updated()
+        FilestoreHelper.dataset_update_filestore_resource(
+            resource, filestore_resources, 0
+        )
+        regex = r"^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\d\d\d\d$"
+        assert re.match(regex, resource["last_modified"])
+        assert filestore_resources == {0: "test"}