hdx-python-api 6.2.7__tar.gz → 6.2.9__tar.gz

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/.config/pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
         exclude: test_scraper_.*\.json
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.2
+    rev: v0.4.2
     hooks:
       # Run the linter.
       - id: ruff
@@ -18,7 +18,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: v0.1.24
+    rev: 0.1.38
     hooks:
       # Run the pip compile
       - id: pip-compile

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hdx-python-api
-Version: 6.2.7
+Version: 6.2.9
 Summary: HDX Python API for interacting with the Humanitarian Data Exchange
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-api
 Author: Michael Rans
@@ -28,8 +28,8 @@ Requires-Python: >=3.8
 Requires-Dist: ckanapi>=4.8
 Requires-Dist: defopt>=6.4.0
 Requires-Dist: email-validator
-Requires-Dist: hdx-python-country>=3.7.0
-Requires-Dist: hdx-python-utilities>=3.6.7
+Requires-Dist: hdx-python-country>=3.7.2
+Requires-Dist: hdx-python-utilities>=3.6.9
 Requires-Dist: libhxl>=5.2.1
 Requires-Dist: makefun
 Requires-Dist: ndg-httpsclient

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/documentation/main.md

@@ -52,6 +52,10 @@ The library has detailed API documentation which can be found in the menu at the
 
 
 ## Breaking Changes
+From 6.2.8, fix mark_data_updated which was broken due to an error in
+dataset_update_filestore_resource in which timezone information was
+incorrectly added to the iso formatted string
+
 From 6.2.7, generate_resource_from_iterator renamed to
 generate_resource_from_iterable with requirement of iterable rather iterator
 

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/pyproject.toml

@@ -37,8 +37,8 @@ dependencies = [
     "ckanapi>=4.8",
     "defopt>=6.4.0",
     "email_validator",
-    "hdx-python-country>=3.7.0",
-    "hdx-python-utilities>=3.6.7",
+    "hdx-python-country>=3.7.2",
+    "hdx-python-utilities>=3.6.9",
     "libhxl>=5.2.1",
     "makefun",
     "ndg-httpsclient",

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/requirements.txt

@@ -1,15 +1,16 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
-annotated-types==0.6.0
+annotated-types==0.7.0
     # via pydantic
 attrs==23.2.0
     # via
     #   frictionless
     #   jsonlines
     #   jsonschema
+    #   referencing
 cachetools==5.3.3
     # via google-auth
-certifi==2024.2.2
+certifi==2024.6.2
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -22,9 +23,9 @@ charset-normalizer==3.3.2
 ckanapi==4.8
 click==8.1.7
     # via typer
-coverage==7.4.4
+coverage==7.5.3
     # via pytest-cov
-cryptography==42.0.5
+cryptography==42.0.8
     # via pyopenssl
 defopt==6.4.0
 distlib==0.3.8
@@ -35,14 +36,14 @@ docopt==0.6.2
     # via
     #   ckanapi
     #   num2words
-docutils==0.21.post1
+docutils==0.21.2
     # via defopt
 email-validator==2.1.1
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.13.4
+filelock==3.14.0
     # via virtualenv
-frictionless==5.16.1
+frictionless==5.17.0
     # via hdx-python-utilities
 google-auth==2.29.0
     # via
@@ -50,36 +51,38 @@ google-auth==2.29.0
     #   gspread
 google-auth-oauthlib==1.2.0
     # via gspread
-gspread==6.1.0
-hdx-python-country==3.7.0
-hdx-python-utilities==3.6.7
+gspread==6.1.2
+hdx-python-country==3.7.2
+hdx-python-utilities==3.6.9
     # via hdx-python-country
 humanize==4.9.0
     # via frictionless
-identify==2.5.35
+identify==2.5.36
     # via pre-commit
-idna==3.6
+idna==3.7
     # via
     #   email-validator
     #   requests
 ijson==3.2.3
     # via hdx-python-utilities
-inflect==7.2.0
+inflect==7.2.1
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
 isodate==0.6.1
     # via frictionless
-jinja2==3.1.3
+jinja2==3.1.4
     # via frictionless
 jsonlines==4.0.0
     # via hdx-python-utilities
 jsonpath-ng==1.6.1
     # via libhxl
-jsonschema==4.17.3
+jsonschema==4.22.0
     # via
     #   frictionless
     #   tableschema-to-template
+jsonschema-specifications==2023.12.1
+    # via jsonschema
 libhxl==5.2.1
     # via hdx-python-country
 loguru==0.7.2
@@ -96,21 +99,21 @@ mdurl==0.1.2
 more-itertools==10.2.0
     # via inflect
 ndg-httpsclient==0.5.1
-nodeenv==1.8.0
+nodeenv==1.9.1
     # via pre-commit
 num2words==0.5.13
     # via quantulum3
 oauthlib==3.2.2
     # via requests-oauthlib
-openpyxl==3.1.2
+openpyxl==3.1.3
     # via hdx-python-utilities
 packaging==24.0
     # via pytest
 petl==1.7.15
     # via frictionless
-platformdirs==4.2.0
+platformdirs==4.2.2
     # via virtualenv
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
 ply==3.11
     # via
@@ -118,7 +121,7 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==3.7.0
+pre-commit==3.7.1
 pyasn1==0.6.0
     # via
     #   ndg-httpsclient
@@ -128,19 +131,17 @@ pyasn1-modules==0.4.0
     # via google-auth
 pycparser==2.22
     # via cffi
-pydantic==2.6.4
+pydantic==2.7.3
     # via frictionless
-pydantic-core==2.16.3
+pydantic-core==2.18.4
     # via pydantic
-pygments==2.17.2
+pygments==2.18.0
     # via rich
 pyopenssl==24.1.0
     # via ndg-httpsclient
 pyphonetics==0.5.3
     # via hdx-python-country
-pyrsistent==0.20.0
-    # via jsonschema
-pytest==8.1.1
+pytest==8.2.2
     # via pytest-cov
 pytest-cov==5.0.0
 python-dateutil==2.8.2
@@ -159,17 +160,21 @@ pyyaml==6.0.1
     #   frictionless
     #   pre-commit
     #   tableschema-to-template
-quantulum3==0.9.0
+quantulum3==0.9.1
 ratelimit==2.2.1
     # via hdx-python-utilities
-requests==2.31.0
+referencing==0.35.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+requests==2.32.3
     # via
     #   ckanapi
     #   frictionless
     #   libhxl
     #   requests-file
     #   requests-oauthlib
-requests-file==2.0.0
+requests-file==2.1.0
     # via hdx-python-utilities
 requests-oauthlib==2.0.0
     # via google-auth-oauthlib
@@ -177,16 +182,18 @@ rfc3986==2.0.0
     # via frictionless
 rich==13.7.1
     # via typer
+rpds-py==0.18.1
+    # via
+    #   jsonschema
+    #   referencing
 rsa==4.9
     # via google-auth
 ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-setuptools==69.2.0
-    # via
-    #   ckanapi
-    #   nodeenv
+setuptools==70.0.0
+    # via ckanapi
 shellingham==1.5.4
     # via typer
 simpleeval==0.9.13
@@ -202,11 +209,9 @@ six==1.16.0
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
-strenum==0.4.15
-    # via gspread
 stringcase==1.2.0
     # via frictionless
-structlog==24.1.0
+structlog==24.2.0
     # via libhxl
 tableschema-to-template==0.0.13
     # via hdx-python-utilities
@@ -214,11 +219,11 @@ tabulate==0.9.0
     # via frictionless
 text-unidecode==1.3
     # via python-slugify
-typeguard==4.2.1
+typeguard==4.3.0
     # via inflect
 typer==0.12.3
     # via frictionless
-typing-extensions==4.11.0
+typing-extensions==4.12.1
     # via
     #   frictionless
     #   inflect
@@ -234,9 +239,9 @@ urllib3==2.2.1
     # via
     #   libhxl
     #   requests
-validators==0.28.0
+validators==0.28.3
     # via frictionless
-virtualenv==20.25.1
+virtualenv==20.26.2
     # via pre-commit
 wheel==0.43.0
     # via libhxl

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/src/hdx/api/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '6.2.7'
-__version_tuple__ = version_tuple = (6, 2, 7)
+__version__ = version = '6.2.9'
+__version_tuple__ = version_tuple = (6, 2, 9)

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/src/hdx/data/dataset.py

@@ -5,7 +5,7 @@ import logging
 import sys
 import warnings
 from copy import deepcopy
-from datetime import datetime, timezone
+from datetime import datetime
 from os.path import isfile, join
 from typing import (
     TYPE_CHECKING,
@@ -41,6 +41,7 @@ from hdx.utilities.dateparse import (
     default_date,
     default_enddate,
     now_utc,
+    now_utc_notz,
     parse_date,
     parse_date_range,
 )
@@ -598,9 +599,9 @@ class Dataset(HDXObject):
             del kwargs["updated_by_script"]
         else:
             scriptinfo = self.configuration.get_user_agent()
-        # No need to output timezone info here
+        # Should not output timezone info here
         data["updated_by_script"] = (
-            f"{scriptinfo} ({datetime.now(timezone.utc).replace(tzinfo=None).isoformat(timespec='microseconds')})"
+            f"{scriptinfo} ({now_utc_notz().isoformat(timespec='microseconds')})"
         )
         batch = kwargs.get("batch")
         if batch:

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/src/hdx/data/filestore_helper.py

@@ -1,8 +1,9 @@
 """Helper to the Dataset class for handling resources with filestores."""
 
-from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any, Dict
 
+from hdx.utilities.dateparse import now_utc_notz
+
 if TYPE_CHECKING:
     from hdx.data.resource import Resource
 
@@ -90,7 +91,8 @@ class FilestoreHelper:
 
         data_updated = resource_data_to_update.is_marked_data_updated()
         if data_updated:
-            resource_data_to_update["last_modified"] = datetime.now(
-                timezone.utc
-            ).isoformat(timespec="microseconds")
+            # Should not output timezone info here
+            resource_data_to_update["last_modified"] = (
+                now_utc_notz().isoformat(timespec="microseconds")
+            )
             resource_data_to_update.data_updated = False

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/src/hdx/data/resource.py

@@ -2,7 +2,7 @@
 
 import logging
 import warnings
-from datetime import datetime, timezone
+from datetime import datetime
 from os.path import join
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union
@@ -13,7 +13,7 @@ from hdx.api.configuration import Configuration
 from hdx.data.date_helper import DateHelper
 from hdx.data.hdxobject import HDXError, HDXObject
 from hdx.data.resource_view import ResourceView
-from hdx.utilities.dateparse import now_utc, parse_date
+from hdx.utilities.dateparse import now_utc, now_utc_notz, parse_date
 from hdx.utilities.downloader import Download
 from hdx.utilities.typehint import ListTuple
 from hdx.utilities.uuid import is_valid_uuid
@@ -172,7 +172,8 @@ class Resource(HDXObject):
             if configuration is None:
                 configuration = Configuration.read()
             with Download(
-                full_agent=configuration.get_user_agent()
+                full_agent=configuration.get_user_agent(),
+                use_env=False,
             ) as downloader:
                 if url is None:
                     url = configuration["formats_mapping_url"]
@@ -393,9 +394,10 @@ class Resource(HDXObject):
         """
         data_updated = kwargs.pop("data_updated", self.data_updated)
         if data_updated and not self.file_to_upload:
-            self.old_data["last_modified"] = datetime.now(
-                timezone.utc
-            ).isoformat(timespec="microseconds")
+            # Should not output timezone info here
+            self.old_data["last_modified"] = now_utc_notz().isoformat(
+                timespec="microseconds"
+            )
             self.data_updated = False
             # old_data will be merged into data in the next step
         self._merge_hdx_update(
@@ -533,7 +535,9 @@ class Resource(HDXObject):
         else:
             headers = None
         with Download(
-            full_agent=self.configuration.get_user_agent(), headers=headers
+            full_agent=self.configuration.get_user_agent(),
+            use_env=False,
+            headers=headers,
         ) as downloader:
             path = downloader.download_file(
                 url, folder=folder, filename=filename

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/src/hdx/data/vocabulary.py

@@ -258,7 +258,9 @@ class Vocabulary(HDXObject):
         Returns:
             List[str]: List of approved tags
         """
-        with Download(full_agent=configuration.get_user_agent()) as downloader:
+        with Download(
+            full_agent=configuration.get_user_agent(), use_env=False
+        ) as downloader:
             if url is None:
                 url = configuration["tags_list_url"]
             return list(
@@ -402,7 +404,8 @@ class Vocabulary(HDXObject):
             if configuration is None:
                 configuration = Configuration.read()
             with Download(
-                full_agent=configuration.get_user_agent()
+                full_agent=configuration.get_user_agent(),
+                use_env=False,
             ) as downloader:
                 if url is None:
                     url = configuration["tags_mapping_url"]

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/tests/hdx/data/__init__.py

@@ -96,6 +96,16 @@ resources_data = [
     },
 ]
 
+resource_data = {
+    "name": "MyResource1",
+    "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d",
+    "format": "xlsx",
+    "url": "http://test/spreadsheet.xlsx",
+    "description": "My Resource",
+    "api_type": "api",
+    "resource_type": "api",
+}
+
 organization_data = {
     "name": "MyOrganization1",
     "title": "Humanitarian Organization",

hdx_python_api-6.2.9/tests/hdx/data/test_filestore_helper.py

@@ -0,0 +1,49 @@
+import copy
+import re
+
+from . import resource_data
+from hdx.data.filestore_helper import FilestoreHelper
+from hdx.data.resource import Resource
+
+
+class TestFilestoreHelper:
+    def test_dataset_update_filestore_resource(self, configuration):
+        resource_data_copy = copy.deepcopy(resource_data)
+        resource = Resource(resource_data_copy)
+        filestore_resources = {}
+        FilestoreHelper.dataset_update_filestore_resource(
+            resource, filestore_resources, 0
+        )
+        assert resource == {
+            "api_type": "api",
+            "description": "My Resource",
+            "format": "xlsx",
+            "name": "MyResource1",
+            "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d",
+            "resource_type": "api",
+            "url": "http://test/spreadsheet.xlsx",
+        }
+        assert filestore_resources == {}
+
+        resource.set_file_to_upload("test")
+        FilestoreHelper.dataset_update_filestore_resource(
+            resource, filestore_resources, 0
+        )
+        assert resource == {
+            "api_type": "api",
+            "description": "My Resource",
+            "format": "xlsx",
+            "name": "MyResource1",
+            "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d",
+            "resource_type": "api",
+            "url": "updated_by_file_upload_step",
+        }
+        assert filestore_resources == {0: "test"}
+
+        resource.mark_data_updated()
+        FilestoreHelper.dataset_update_filestore_resource(
+            resource, filestore_resources, 0
+        )
+        regex = r"^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\d\d\d\d$"
+        assert re.match(regex, resource["last_modified"])
+        assert filestore_resources == {0: "test"}

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/tests/hdx/data/test_resource.py

@@ -9,7 +9,7 @@ from os.path import basename, join
 
 import pytest
 
-from . import MockResponse, dataset_resultdict
+from . import MockResponse, dataset_resultdict, resource_data
 from .test_resource_view import resource_view_list, resource_view_mocklist
 from hdx.api.configuration import Configuration
 from hdx.data.hdxobject import HDXError
@@ -318,16 +318,6 @@ def mockresourceview(url, decodedata):
 
 
 class TestResource:
-    resource_data = {
-        "name": "MyResource1",
-        "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d",
-        "format": "xlsx",
-        "url": "http://test/spreadsheet.xlsx",
-        "description": "My Resource",
-        "api_type": "api",
-        "resource_type": "api",
-    }
-
     datastore = None
 
     @pytest.fixture(scope="class")
@@ -685,15 +675,15 @@ class TestResource:
             Resource.read_from_hdx("ABC")
 
     def test_check_url_filetoupload(self, configuration):
-        resource_data = copy.deepcopy(TestResource.resource_data)
-        resource = Resource(resource_data)
+        resource_data_copy = copy.deepcopy(resource_data)
+        resource = Resource(resource_data_copy)
         resource.check_url_filetoupload()
         resource.set_file_to_upload("abc")
         resource.check_url_filetoupload()
         resource["url"] = "lala"
         with pytest.raises(HDXError):
             resource.check_url_filetoupload()
-        resource = Resource(resource_data)
+        resource = Resource(resource_data_copy)
         resource["format"] = "NOTEXIST"
         with pytest.raises(HDXError):
             resource.check_url_filetoupload()
@@ -726,8 +716,8 @@ class TestResource:
         }
 
     def test_check_required_fields(self, configuration):
-        resource_data = copy.deepcopy(TestResource.resource_data)
-        resource = Resource(resource_data)
+        resource_data_copy = copy.deepcopy(resource_data)
+        resource = Resource(resource_data_copy)
         resource.check_url_filetoupload()
         resource.check_required_fields()
 
@@ -740,8 +730,8 @@ class TestResource:
         with pytest.raises(HDXError):
             resource.create_in_hdx()
 
-        resource_data = copy.deepcopy(TestResource.resource_data)
-        resource = Resource(resource_data)
+        resource_data_copy = copy.deepcopy(resource_data)
+        resource = Resource(resource_data_copy)
         resource.create_in_hdx()
         assert resource["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5"
         assert resource["url_type"] == "api"
@@ -751,8 +741,8 @@ class TestResource:
             == "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/main/tests/fixtures/test_data.csv"
         )
 
-        resource_data = copy.deepcopy(TestResource.resource_data)
-        resource = Resource(resource_data)
+        resource_data_copy = copy.deepcopy(resource_data)
+        resource = Resource(resource_data_copy)
         filetoupload = join("tests", "fixtures", "test_data.csv")
         resource.set_file_to_upload(filetoupload)
         assert resource.get_file_to_upload() == filetoupload
@@ -770,13 +760,13 @@ class TestResource:
             resource["url"]
             == "http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/test_data.csv"
         )
-        resource_data["name"] = "MyResource2"
-        resource = Resource(resource_data)
+        resource_data_copy["name"] = "MyResource2"
+        resource = Resource(resource_data_copy)
         with pytest.raises(HDXError):
             resource.create_in_hdx()
 
-        resource_data["name"] = "MyResource3"
-        resource = Resource(resource_data)
+        resource_data_copy["name"] = "MyResource3"
+        resource = Resource(resource_data_copy)
         with pytest.raises(HDXError):
             resource.create_in_hdx()
 
@@ -841,10 +831,10 @@ class TestResource:
         with pytest.raises(HDXError):
             resource.update_in_hdx()
 
-        resource_data = copy.deepcopy(TestResource.resource_data)
-        resource_data["name"] = "MyResource1"
-        resource_data["id"] = "74b74ae1-df0c-4716-829f-4f939a046811"
-        resource = Resource(resource_data)
+        resource_data_copy = copy.deepcopy(resource_data)
+        resource_data_copy["name"] = "MyResource1"
+        resource_data_copy["id"] = "74b74ae1-df0c-4716-829f-4f939a046811"
+        resource = Resource(resource_data_copy)
         resource.mark_data_updated()
         assert resource.data_updated is True
         assert resource.is_marked_data_updated() is True
@@ -872,8 +862,8 @@ class TestResource:
             resource.delete_from_hdx()
 
     def test_update_yaml(self, configuration, static_yaml):
-        resource_data = copy.deepcopy(TestResource.resource_data)
-        resource = Resource(resource_data)
+        resource_data_copy = copy.deepcopy(resource_data)
+        resource = Resource(resource_data_copy)
         assert resource["name"] == "MyResource1"
         assert resource.get_format() == "xlsx"
         resource.update_from_yaml(static_yaml)
@@ -881,8 +871,8 @@ class TestResource:
         assert resource.get_format() == "csv"
 
     def test_update_json(self, configuration, static_json):
-        resource_data = copy.deepcopy(TestResource.resource_data)
-        resource = Resource(resource_data)
+        resource_data_copy = copy.deepcopy(resource_data)
+        resource = Resource(resource_data_copy)
         assert resource["name"] == "MyResource1"
         assert resource.get_format() == "xlsx"
         resource.update_from_json(static_json)
@@ -898,8 +888,8 @@ class TestResource:
         assert resource["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5"
 
     def test_get_dataset(self, configuration, post_dataset):
-        resource_data = copy.deepcopy(TestResource.resource_data)
-        resource = Resource(resource_data)
+        resource_data_copy = copy.deepcopy(resource_data)
+        resource = Resource(resource_data_copy)
         dataset = resource.get_dataset()
         assert dataset["id"] == "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d"
         del resource["package_id"]

{hdx_python_api-6.2.7 → hdx_python_api-6.2.9}/tests/hdx/data/test_update_dataset_resources.py

@@ -130,6 +130,10 @@ class TestUpdateDatasetResourcesLogic:
             ("QuickCharts-SDG 4 Global and Thematic data", "csv"),
         ]
         dataset._prepare_hdx_call(dataset.old_data, {})
+        assert (
+            dataset["updated_by_script"]
+            == "HDX Scraper: UNESCO (2022-12-19T12:51:30.579185)"
+        )
         results = dataset._revise_dataset(
             tuple(),
             resources_to_update,