hdx-python-api 6.6.9__py3-none-any.whl → 6.7.1__py3-none-any.whl

hdx/api/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '6.6.9'
-__version_tuple__ = version_tuple = (6, 6, 9)
+__version__ = version = '6.7.1'
+__version_tuple__ = version_tuple = (6, 7, 1)
 
 __commit_id__ = commit_id = None

hdx/api/hdx_base_configuration.yaml

@@ -72,6 +72,10 @@ organization:
     - name
     - title
     - description
+group:
+  required_fields:
+    - name
+    - title
 "resource view":
   required_fields:
     - resource_id

hdx/api/locations.py

@@ -3,6 +3,7 @@
 from collections.abc import Sequence
 
 from hdx.api.configuration import Configuration
+from hdx.data.location import Location
 
 
 class Locations:
@@ -22,10 +23,8 @@ class Locations:
             A list of valid locations
         """
         if cls._validlocations is None:
-            if configuration is None:
-                configuration = Configuration.read()
-            cls._validlocations = configuration.call_remoteckan(
-                "group_list", {"all_fields": True}
+            cls._validlocations = Location.get_all_location_names(
+                configuration=configuration, all_fields=True
             )
         return cls._validlocations
 

hdx/data/location.py

@@ -0,0 +1,237 @@
+"""Location class containing all logic for creating, checking, and updating locations."""
+
+import logging
+from collections.abc import Sequence
+from typing import Any, Optional
+
+from hdx.api.configuration import Configuration
+from hdx.data.hdxobject import HDXObject
+
+logger = logging.getLogger(__name__)
+
+
+class Location(HDXObject):
+    """Location class containing all logic for creating, checking, and updating locations.
+    A location is a CKAN group.
+
+    Args:
+        initial_data: Initial location metadata dictionary. Defaults to None.
+        configuration: HDX configuration. Defaults to global configuration.
+    """
+
+    def __init__(
+        self,
+        initial_data: dict | None = None,
+        configuration: Configuration | None = None,
+    ) -> None:
+        if not initial_data:
+            initial_data = {}
+        super().__init__(initial_data, configuration=configuration)
+
+    @staticmethod
+    def actions() -> dict[str, str]:
+        """Dictionary of actions that can be performed on object
+
+        Returns:
+            Dictionary of actions that can be performed on object
+        """
+        return {
+            "show": "group_show",
+            "update": "group_update",
+            "create": "group_create",
+            "delete": "group_delete",
+            "list": "group_list",
+            "autocomplete": "group_autocomplete",
+        }
+
+    @classmethod
+    def read_from_hdx(
+        cls, identifier: str, configuration: Configuration | None = None
+    ) -> Optional["Location"]:
+        """Reads the location given by identifier from HDX and returns Location object
+
+        Args:
+            identifier: Identifier of location
+            configuration: HDX configuration. Defaults to global configuration.
+
+        Returns:
+            Location object if successful read, None if not
+        """
+        return cls._read_from_hdx_class("group", identifier, configuration)
+
+    def check_required_fields(self, ignore_fields: Sequence[str] = ()) -> None:
+        """Check that metadata for location is complete. The parameter ignore_fields should
+        be set if required to any fields that should be ignored for the particular operation.
+
+        Args:
+            ignore_fields: Fields to ignore. Default is ().
+
+        Returns:
+            None
+        """
+        self._check_required_fields("group", ignore_fields)
+
+    def update_in_hdx(self, **kwargs: Any) -> None:
+        """Check if location exists in HDX and if so, update location
+
+        Returns:
+            None
+        """
+        self._update_in_hdx("group", "id", **kwargs)
+
+    def create_in_hdx(self, **kwargs: Any) -> None:
+        """Check if location exists in HDX and if so, update it, otherwise create location
+
+        Returns:
+            None
+        """
+        self._create_in_hdx("group", "id", "name", **kwargs)
+
+    def delete_from_hdx(self) -> None:
+        """Deletes a location from HDX.
+
+        Returns:
+            None
+        """
+        self._delete_from_hdx("group", "id")
+
+    def get_datasets(self, query: str = "*:*", **kwargs: Any) -> list["Dataset"]:  # noqa: F821
+        """Get list of datasets in location
+
+        Args:
+            query: Restrict datasets returned to this query (in Solr format). Defaults to '*:*'.
+            **kwargs: See below
+            sort (string): Sorting of the search results. Defaults to 'relevance asc, metadata_modified desc'.
+            rows (int): Number of matching rows to return. Defaults to all datasets (sys.maxsize).
+            start (int): Offset in the complete result for where the set of returned datasets should begin
+            facet (string): Whether to enable faceted results. Default to True.
+            facet.mincount (int): Minimum counts for facet fields should be included in the results
+            facet.limit (int): Maximum number of values the facet fields return (- = unlimited). Defaults to 50.
+            facet.field (list[str]): Fields to facet upon. Default is empty.
+            use_default_schema (bool): Use default package schema instead of custom schema. Defaults to False.
+
+        Returns:
+            List of datasets in location
+        """
+        import hdx.data.dataset  # avoid circular import
+
+        return hdx.data.dataset.Dataset.search_in_hdx(
+            query=query,
+            configuration=self.configuration,
+            fq=f"groups:{self.data['name']}",
+            **kwargs,
+        )
+
+    @staticmethod
+    def get_all_location_names(
+        configuration: Configuration | None = None, **kwargs: Any
+    ) -> list[str]:
+        """Get all location names in HDX
+
+        Args:
+            configuration: HDX configuration. Defaults to global configuration.
+            **kwargs: See below
+            sort (str): Sort the search results according to field name and sort-order. Allowed fields are ‘name’, ‘package_count’ and ‘title’. Defaults to 'name asc'.
+            groups (list[str]): List of names of the groups to return.
+            all_fields (bool): Return group dictionaries instead of just names. Only core fields are returned - get some more using the include_* options. Defaults to False.
+            include_extras (bool): If all_fields, include the group extra fields. Defaults to False.
+            include_tags (bool): If all_fields, include the group tags. Defaults to False.
+            include_groups (bool): If all_fields, include the groups the groups are in. Defaults to False.
+
+        Returns:
+            List of all location names in HDX
+        """
+        location = Location(configuration=configuration)
+
+        # Early return for the standard, non-paginated case
+        if not kwargs.get("all_fields"):
+            return location._write_to_hdx("list", kwargs)
+
+        compiled_locations = {}
+        # Extract limit and offset to dictate our paging sizes, defaulting to 400 and 0
+        page_size = kwargs.pop("limit", 400)
+        current_offset = kwargs.pop("offset", 0)
+
+        # 1. Fetch in pages using the provided/default page size
+        while True:
+            page_kwargs = kwargs.copy()
+            page_kwargs["limit"] = page_size
+            page_kwargs["offset"] = current_offset
+
+            page_data = location._write_to_hdx("list", page_kwargs)
+
+            if not page_data:
+                break
+
+            # Store by name to automatically deduplicate offset shifts
+            for loc in page_data:
+                compiled_locations[loc["name"]] = loc
+
+            if len(page_data) < page_size:
+                break
+
+            current_offset += page_size
+
+        # 2. Establish Ground Truth LAST (without limit/offset constraints)
+        truth_kwargs = kwargs.copy()
+        truth_kwargs["all_fields"] = False
+
+        ground_truth_names = location._write_to_hdx("list", truth_kwargs)
+
+        # 3 & 4. Check for missing IDs, fetch individually, and implicitly prune deletes
+        final_locations = []
+        for name in ground_truth_names:
+            if name in compiled_locations:
+                final_locations.append(compiled_locations[name])
+            else:
+                try:
+                    missing_location = location._write_to_hdx("show", {"id": name})
+                    if missing_location:
+                        final_locations.append(missing_location)
+                except Exception:
+                    pass
+
+        return final_locations
+
+    @staticmethod
+    def get_data_grid_countries(
+        configuration: Configuration | None = None,
+    ) -> list[str]:
+        """Fetch HDX's active Data Grid countries (3-letter group names).
+
+        Treated as a live/short-TTL lookup rather than a frozen static list, since Data Grid
+        membership changes over time.
+
+        Args:
+            configuration: HDX configuration. Defaults to global configuration.
+
+        Returns:
+            Sorted list of active Data Grid country (group) names
+        """
+        groups = Location.get_all_location_names(
+            configuration=configuration, all_fields=True, include_extras=True
+        )
+        return sorted(
+            group["name"]
+            for group in groups
+            if len(group["name"]) == 3 and group.get("data_completeness") == "active"
+        )
+
+    @classmethod
+    def autocomplete(
+        cls,
+        name: str,
+        limit: int = 20,
+        configuration: Configuration | None = None,
+    ) -> list:
+        """Autocomplete a location name and return matches
+
+        Args:
+            name: Name to autocomplete
+            limit: Maximum number of matches to return
+            configuration: HDX configuration. Defaults to global configuration.
+
+        Returns:
+            Autocomplete matches
+        """
+        return cls._autocomplete(name, limit, configuration)

hdx/data/resource.py

@@ -2,7 +2,7 @@
 
 import json
 import logging
-from collections.abc import Sequence
+from collections.abc import Iterator, Sequence
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Optional
@@ -34,6 +34,7 @@ class Resource(HDXObject):
     """
 
     _formats_dict = None
+    _valid_datastore_dump_formats = frozenset({"csv", "tsv", "json", "xml"})
     _valid_data_dictionary_types = frozenset(
         {"text", "numeric", "date", "timestamp without time zone"}
     )
@@ -70,6 +71,7 @@ class Resource(HDXObject):
             "datastore_insert": "datastore_insert",
             "datastore_upsert": "datastore_upsert",
             "datastore_search": "datastore_search",
+            "datastore_search_sql": "datastore_search_sql",
         }
 
     def update_from_yaml(
@@ -711,6 +713,62 @@ class Resource(HDXObject):
             path = downloader.download_file(url, folder=folder, filename=filename)
             return url, path
 
+    def get_datastore_dump_url(self, file_format: str = "csv") -> str:
+        """Get the URL from which the resource datastore can be downloaded in
+        full.
+
+        Args:
+            file_format: Dump format (csv, tsv, json or xml). Defaults to csv.
+
+        Returns:
+            Datastore dump URL
+        """
+        file_format = file_format.lower()
+        if file_format not in self._valid_datastore_dump_formats:
+            valid_formats = ", ".join(sorted(self._valid_datastore_dump_formats))
+            raise HDXError(
+                f"Invalid datastore dump format '{file_format}'! Must be one of: {valid_formats}"
+            )
+        url = (
+            f"{self.configuration.get_hdx_site_url()}/datastore/dump/{self.data['id']}"
+        )
+        if file_format != "csv":
+            url = f"{url}?format={file_format}"
+        return url
+
+    def download_datastore(
+        self,
+        folder: Path | str | None = None,
+        filename: str | None = None,
+        file_format: str = "csv",
+        retriever: Retrieve | None = None,
+    ) -> tuple[str, Path]:
+        """Download the resource datastore in full to provided folder or
+        temporary folder if no folder supplied
+
+        Args:
+            folder: Folder to download datastore to. Defaults to None.
+            filename: Filename to use. Defaults to None (resource name with file_format extension).
+            file_format: Dump format (csv, tsv, json or xml). Defaults to csv.
+            retriever: Retrieve object to use. Defaults to None.
+
+        Returns:
+            (URL downloaded, Path to downloaded file)
+        """
+        url = self.get_datastore_dump_url(file_format)
+        logger.debug(f"Downloading {url}")
+        if filename is None:
+            filename = self.data["name"]
+            extension = f".{file_format.lower()}"
+            if not filename.endswith(extension):
+                filename = f"{filename}{extension}"
+        session = hdx.api.utilities.url_utils.get_ckan_ready_session(self.configuration)
+        with Download(session=session) as downloader:
+            if retriever:
+                downloader = retriever.clone(downloader)
+            path = downloader.download_file(url, folder=folder, filename=filename)
+            return url, path
+
     @staticmethod
     def get_all_resource_ids_in_datastore(
         configuration: Configuration | None = None,
@@ -805,6 +863,127 @@ class Resource(HDXObject):
         }
         self._write_to_hdx("datastore_upsert", data, "resource_id")
 
+    def search_datastore(
+        self,
+        filters: dict | None = None,
+        q: str | dict | None = None,
+        fields: Sequence[str] | None = None,
+        sort: str | None = None,
+        distinct: bool = False,
+        limit: int = 100,
+        offset: int = 0,
+        include_total: bool = True,
+        **kwargs: Any,
+    ) -> dict:
+        """Search the resource datastore using CKAN datastore_search.
+
+        Args:
+            filters: Dictionary of field name to value(s) to match. Defaults to None.
+            q: Full text query string or dictionary of field name to query. Defaults to None.
+            fields: Fields to return. Defaults to None (all fields).
+            sort: Comma separated field names with optional asc/desc eg. "code desc". Defaults to None.
+            distinct: Whether to return only distinct rows. Defaults to False.
+            limit: Maximum number of rows to return. Defaults to 100.
+            offset: Number of rows to skip. Defaults to 0.
+            include_total: Whether to include total number of matching rows. Defaults to True.
+            **kwargs: Other parameters to pass to datastore_search eg. plain, language.
+
+        Returns:
+            datastore_search result dictionary with keys including records, fields and total
+        """
+        data: dict = {
+            "resource_id": self.data["id"],
+            "limit": limit,
+            "offset": offset,
+            "include_total": include_total,
+        }
+        if filters is not None:
+            data["filters"] = filters
+        if q is not None:
+            data["q"] = q
+        if fields is not None:
+            data["fields"] = ",".join(fields)
+        if sort is not None:
+            data["sort"] = sort
+        if distinct:
+            data["distinct"] = True
+        data.update(kwargs)
+        return self._write_to_hdx("datastore_search", data, "resource_id")
+
+    def get_datastore_records(
+        self,
+        filters: dict | None = None,
+        q: str | dict | None = None,
+        fields: Sequence[str] | None = None,
+        sort: str | None = None,
+        distinct: bool = False,
+        page_size: int = 10000,
+        **kwargs: Any,
+    ) -> Iterator[dict]:
+        """Iterate over all matching records in the resource datastore, paging
+        through datastore_search results.
+
+        Args:
+            filters: Dictionary of field name to value(s) to match. Defaults to None.
+            q: Full text query string or dictionary of field name to query. Defaults to None.
+            fields: Fields to return. Defaults to None (all fields).
+            sort: Comma separated field names with optional asc/desc eg. "code desc". Defaults to None.
+            distinct: Whether to return only distinct rows. Defaults to False.
+            page_size: Number of rows to request per call. Defaults to 10000.
+            **kwargs: Other parameters to pass to datastore_search eg. plain, language.
+
+        Returns:
+            Iterator of record dictionaries
+        """
+        offset = 0
+        while True:
+            result = self.search_datastore(
+                filters=filters,
+                q=q,
+                fields=fields,
+                sort=sort,
+                distinct=distinct,
+                limit=page_size,
+                offset=offset,
+                include_total=False,
+                **kwargs,
+            )
+            records = result["records"]
+            yield from records
+            if len(records) < page_size:
+                return
+            offset += page_size
+
+    def get_datastore_fields(self) -> list[dict]:
+        """Get the field definitions of the resource datastore excluding the
+        internal _id field.
+
+        Returns:
+            List of field dictionaries each with id and type keys
+        """
+        result = self.search_datastore(limit=0, include_total=False)
+        return [field for field in result["fields"] if field["id"] != "_id"]
+
+    @staticmethod
+    def search_datastore_sql(
+        sql: str,
+        configuration: Configuration | None = None,
+    ) -> list[dict]:
+        """Run a read-only SQL query against the HDX datastore using CKAN
+        datastore_search_sql. Tables are named by resource id, which must be
+        double quoted eg. SELECT * FROM "RESOURCE_ID".
+
+        Args:
+            sql: SQL SELECT statement
+            configuration: HDX configuration. Defaults to global configuration.
+
+        Returns:
+            List of record dictionaries
+        """
+        resource = Resource(configuration=configuration)
+        result = resource._write_to_hdx("datastore_search_sql", {"sql": sql})
+        return result["records"]
+
     def delete_datastore(self) -> None:
         """Delete a resource from the HDX datastore
 

{hdx_python_api-6.6.9.dist-info → hdx_python_api-6.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hdx-python-api
-Version: 6.6.9
+Version: 6.7.1
 Summary: HDX Python API for interacting with the Humanitarian Data Exchange
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-api
 Author: Michael Rans
@@ -28,8 +28,8 @@ Requires-Python: >=3.10
 Requires-Dist: ckanapi>=4.11
 Requires-Dist: defopt>=7.0.0
 Requires-Dist: email-validator
-Requires-Dist: hdx-python-country>=4.1.3
-Requires-Dist: hdx-python-utilities>=4.1.2
+Requires-Dist: hdx-python-country>=4.2.3
+Requires-Dist: hdx-python-utilities>=4.2.1
 Requires-Dist: makefun
 Requires-Dist: requests
 Provides-Extra: docs

{hdx_python_api-6.6.9.dist-info → hdx_python_api-6.7.1.dist-info}/RECORD

@@ -1,8 +1,8 @@
 hdx/api/__init__.py,sha256=11ozJKiUsqDCZ3_mcAHhGYUyGK_Unl54djVSBBExFB4,59
-hdx/api/_version.py,sha256=DZ0RWsb-nM4U2OMHNx7xq0IT_NuQKLA1g5wIx_XHwr4,520
+hdx/api/_version.py,sha256=7uXJAam46P0GVJ-CiBxd6tVnUUkt-epJdnvQ6m2fpb4,520
 hdx/api/configuration.py,sha256=9WvuVI-3th2Dv70_t_4VoyrlJwB_9qUSN5_zUKbp4sQ,23805
-hdx/api/hdx_base_configuration.yaml,sha256=9yJYroZwoyMQo6tcxGfWHWvYahnTma7Szx0J5Z378z4,2173
-hdx/api/locations.py,sha256=blnqOjEASvh0mWgmo2iEE7ES_xOuRwQo3fOMqTPV_P0,4139
+hdx/api/hdx_base_configuration.yaml,sha256=jj2t5b7NnI499rHVwOjj3Q5BAjE8i7NDGVKnWZ5P5-4,2222
+hdx/api/locations.py,sha256=bFl_cKz_AEQzkE_xWOkrRlUQ0GT2LMPSk-G_LsbV-Ns,4099
 hdx/api/remotehdx.py,sha256=IjKwP3jutlkrjvWCJPP7YM-xzg3pP_QVmiJfblllBY0,2079
 hdx/api/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdx/api/utilities/date_helper.py,sha256=sXxjRtXz-2rpBVZBbJAk80gfjpYKl_AtHplWr-rniIk,6396
@@ -13,8 +13,9 @@ hdx/api/utilities/url_utils.py,sha256=Pb4r-BAqIHYmeoB_i7QjutDp6nOAUh_OHvCtiNLda3
 hdx/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdx/data/dataset.py,sha256=0gS7tQSswTgG0g8YeXg0FL8pFVkocBQSRPFpQAurN00,99455
 hdx/data/hdxobject.py,sha256=_jlwYjb_CcjaXsTXObknMfq-OiNOcpJfLJ9b1vKV1Zo,27494
+hdx/data/location.py,sha256=WIQRdY1boy6xPweIWRczWNiJLFrp4w0DVUwppHeNccU,8765
 hdx/data/organization.py,sha256=HGopFsCYOWrR7sUtZzZSXq6wLFw4djus2gzg6yNXOqc,12296
-hdx/data/resource.py,sha256=SBAEou9Fj5sgpfoDmQ6j33W0srMMmhlsj00hiTuW86Y,38097
+hdx/data/resource.py,sha256=iVARfA3_8gGsGjXgNxomm7umAadvxzFsGGqxTpBB3WQ,45196
 hdx/data/resource_matcher.py,sha256=Yzk9pWa4zk1p09m6zcZMj0XZf5Y16Pa3KToOdUt7rP8,4247
 hdx/data/resource_view.py,sha256=GcYkh0iDAe6_m2ErY_A6i3Z7ZFX949eGt1TLnLZv_8s,7346
 hdx/data/showcase.py,sha256=vnRZ45xd9Ct65eFQkWZtMIJPFifZfKOrH4obO5D5Pvc,14062
@@ -24,7 +25,7 @@ hdx/facades/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdx/facades/infer_arguments.py,sha256=DvyDuozfOPmoaV7HjhQySojGIrMWW9nos31auQBVVl8,4032
 hdx/facades/keyword_arguments.py,sha256=aVapHCfqBO6H7xxBgTdK9ISdgrBFsNypPDWihfQ3edU,1125
 hdx/facades/simple.py,sha256=QWsBkaHG_UM6X3Zu2u3LWVOQzeR24q-EUClrfsL_mkM,1064
-hdx_python_api-6.6.9.dist-info/METADATA,sha256=H29ZAlNGFdS46-ucNToWqy68nIrHQkcjULochC1Ijao,4946
-hdx_python_api-6.6.9.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-hdx_python_api-6.6.9.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
-hdx_python_api-6.6.9.dist-info/RECORD,,
+hdx_python_api-6.7.1.dist-info/METADATA,sha256=7SaqX2zZKJ828e94hbPjvjJ_3q5obaL0NU9Ghut_86A,4946
+hdx_python_api-6.7.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+hdx_python_api-6.7.1.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
+hdx_python_api-6.7.1.dist-info/RECORD,,

{hdx_python_api-6.6.9.dist-info → hdx_python_api-6.7.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.32.0
+Generator: hatchling 1.32.4
 Root-Is-Purelib: true
 Tag: py3-none-any