hdx-python-api 6.7.0__tar.gz → 6.7.2__tar.gz

{hdx_python_api-6.7.0 → hdx_python_api-6.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hdx-python-api
-Version: 6.7.0
+Version: 6.7.2
 Summary: HDX Python API for interacting with the Humanitarian Data Exchange
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-api
 Author: Michael Rans
@@ -28,8 +28,8 @@ Requires-Python: >=3.10
 Requires-Dist: ckanapi>=4.11
 Requires-Dist: defopt>=7.0.0
 Requires-Dist: email-validator
-Requires-Dist: hdx-python-country>=4.1.3
-Requires-Dist: hdx-python-utilities>=4.1.2
+Requires-Dist: hdx-python-country>=4.2.3
+Requires-Dist: hdx-python-utilities>=4.2.1
 Requires-Dist: makefun
 Requires-Dist: requests
 Provides-Extra: docs

{hdx_python_api-6.7.0 → hdx_python_api-6.7.2}/documentation/index.md

@@ -35,6 +35,7 @@ upload your datasets to HDX.
     -   [Organization Management](#organization-management)
     -   [Location Management](#location-management)
     -   [Vocabulary Management](#vocabulary-management)
+    -   [Activity Streams](#activity-streams)
     -   [Pipeline State](#pipeline-state)
 -   [Working Examples](#working-examples)
 -   [Project Framework](#project-framework)
@@ -832,9 +833,13 @@ dictionary. HEADERS is either a row number (rows start counting at 1), or the
 actual headers defined as a list of strings. If not set, all rows will be
 treated as containing values:
 
-    dataset.generate_resource("FOLDER", "FILENAME", ROWS, RESOURCE DATA, HEADERS,
-                              COLUMNS, "FORMAT", "ENCODING", DATECOL or YEARCOL or
-                              DATE_FUNCTION)
+    dataset.generate_resource("FOLDER", "FILENAME", ROWS, RESOURCE_DATA,
+                              headers=HEADERS, columns=COLUMNS, format="FORMAT",
+                              encoding="ENCODING", datecol=DATECOL)
+
+To use years or a custom date function instead, replace `datecol=DATECOL` with
+`yearcol=YEARCOL` or `date_function=DATE_FUNCTION`. Supply only one of these
+three arguments, using its keyword.
 
 The first 4 parameters are mandatory, the rest are optional. A resource can be generated
 from a given list or tuple or other iterable. The method returns a tuple with a bool
@@ -868,8 +873,12 @@ and outputs a modified row.
 The rest of the arguments are the same as for `generate_resource`.
 
     dataset.download_generate_resource(DOWNLOADER, "URL", "FOLDER", "FILENAME",
-                                       RESOURCE_DATA, HEADER_INSERTIONS, ROW_FUNCTION,
-                                       DATECOL or YEARCOL or DATE_FUNCTION, **KWARGS)
+                                       RESOURCE_DATA, header_insertions=HEADER_INSERTIONS,
+                                       row_function=ROW_FUNCTION, datecol=DATECOL,
+                                       **KWARGS)
+
+As with `generate_resource`, use `yearcol=YEARCOL` or
+`date_function=DATE_FUNCTION` instead of `datecol=DATECOL` when appropriate.
 
 ## Resource Specific Operations
 
@@ -965,6 +974,58 @@ using the getter:
 
     data_dictionary = resource.get_hdx_data_dictionary()
 
+### Datastore
+
+A resource's data can be loaded into the HDX datastore and queried. HDX
+requires an API key for all datastore actions and dumps, including reads of
+public resources. To create, update or delete the datastore for a resource:
+
+    resource.create_datastore(
+        [{"id": "code", "type": "text"}, {"id": "value", "type": "numeric"}],
+        primary_key="code",
+    )
+    resource.update_datastore([{"code": "A", "value": 1.0}])  # upsert by default
+    has_datastore = resource.has_datastore()
+    resource.delete_datastore()
+
+To download the whole datastore (rather than the resource's file) use
+**download_datastore**, which returns the URL and path like **download**.
+`file_format` can be `csv` (default), `tsv`, `json` or `xml`, and `filename`
+defaults to the resource name with that extension. **get_datastore_dump_url**
+returns the URL without downloading:
+
+    url, path = resource.download_datastore("FOLDER_TO_DOWNLOAD_TO")
+    url = resource.get_datastore_dump_url(file_format="json")
+
+**search_datastore** wraps CKAN's `datastore_search`, returning the result
+dictionary (with keys including `records`, `fields` and `total`). It accepts
+`filters`, `q`, `fields`, `sort`, `distinct`, `limit` (default 100), `offset`
+and `include_total`, and passes any other keyword arguments to CKAN:
+
+    result = resource.search_datastore(
+        filters={"code": ["A", "B"]}, fields=["code", "value"], sort="value desc"
+    )
+    records = result["records"]
+
+**get_datastore_records** takes the same query parameters but pages through
+all matching rows, yielding each record (`page_size` defaults to 10000):
+
+    for record in resource.get_datastore_records(filters={"code": "A"}):
+        ...
+
+**get_datastore_fields** returns the datastore field definitions excluding the
+internal `_id` field:
+
+    fields = resource.get_datastore_fields()
+
+**search_datastore_sql** wraps CKAN's `datastore_search_sql`, running a
+SELECT statement where tables are named by double quoted resource id and
+returning the list of records:
+
+    records = Resource.search_datastore_sql(
+        'SELECT code, SUM(value) AS total FROM "RESOURCE_ID" GROUP BY code'
+    )
+
 ## Showcase Management
 
 The **Showcase** class enables you to manage showcases, creating, deleting and updating
@@ -1128,6 +1189,40 @@ If you want to add a list of tags, you do it as follows:
 
     vocabulary.add_tags(["TAG","TAG","TAG"...])
 
+## Activity Streams
+
+The **Activity** class wraps CKAN activities, which record who changed what and when.
+Activities are read only. You can get the activity stream of a dataset, organization,
+location or user:
+
+    activities = dataset.get_activities(**kwargs)
+    activities = organization.get_activities(**kwargs)
+    activities = location.get_activities(**kwargs)
+    activities = user.get_activities(**kwargs)
+
+These return a list of **Activity** objects, each with fields like `timestamp`,
+`user_id`, `object_id` and `activity_type` (eg. "changed package"). One page of results
+is returned. Its size is set by `limit` (default 31, HDX maximum 100) and where it starts
+by `offset`. Passing `all_pages=True` gets every page. Other arguments (`**kwargs`) like
+`after`, `before` and `activity_types` depend on the object type and are detailed in the
+API documentation. Getting a user's activity stream requires authorisation to view the
+user's profile.
+
+You can get the activity stream of all recently added or changed datasets:
+
+    activities = Activity.get_recently_changed_packages(limit=10)
+
+With `all_pages=True`, this stream covers the whole history of HDX so it is capped by
+`max_activities` (default 1000). Pass `max_activities=None` to remove the cap.
+
+You can read a single activity, get its data (including a snapshot of the object at the
+time of the activity) or get a diff against the object's previous version. HDX requires
+permission to edit the object the activity relates to for these:
+
+    activity = Activity.read_from_hdx("ACTIVITY_ID")
+    data = activity.get_data()
+    diff = activity.get_diff(diff_type="unified")  # or "context" or "html"
+
 ## Pipeline State
 
 The HDXState class allows the reading and writing of state to a given dataset. Input

{hdx_python_api-6.7.0 → hdx_python_api-6.7.2}/pyproject.toml

@@ -39,8 +39,8 @@ dependencies = [
     "ckanapi>=4.11",
     "defopt>=7.0.0",
     "email_validator",
-    "hdx-python-country>=4.1.3",
-    "hdx-python-utilities>=4.1.2",
+    "hdx-python-country>=4.2.3",
+    "hdx-python-utilities>=4.2.1",
     "makefun",
     "requests",
 ]

{hdx_python_api-6.7.0 → hdx_python_api-6.7.2}/src/hdx/api/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '6.7.0'
-__version_tuple__ = version_tuple = (6, 7, 0)
+__version__ = version = '6.7.2'
+__version_tuple__ = version_tuple = (6, 7, 2)
 
 __commit_id__ = commit_id = None

hdx_python_api-6.7.2/src/hdx/data/activity.py

@@ -0,0 +1,245 @@
+"""Activity class containing all logic for reading activity streams."""
+
+import logging
+from collections.abc import Sequence
+from typing import Any, Optional
+
+from hdx.api.configuration import Configuration
+from hdx.data.hdxobject import HDXError, HDXObject
+
+logger = logging.getLogger(__name__)
+
+
+class Activity(HDXObject):
+    """Activity class containing all logic for reading activity streams. Activities
+    are read only.
+
+    Args:
+        initial_data: Initial activity metadata dictionary. Defaults to None.
+        configuration: HDX configuration. Defaults to global configuration.
+    """
+
+    def __init__(
+        self,
+        initial_data: dict | None = None,
+        configuration: Configuration | None = None,
+    ) -> None:
+        if not initial_data:
+            initial_data = {}
+        super().__init__(initial_data, configuration=configuration)
+
+    @staticmethod
+    def actions() -> dict[str, str]:
+        """Dictionary of actions that can be performed on object
+
+        Returns:
+            Dictionary of actions that can be performed on object
+        """
+        return {
+            "show": "activity_show",
+            "data_show": "activity_data_show",
+            "diff": "activity_diff",
+            "package_list": "package_activity_list",
+            "organization_list": "organization_activity_list",
+            "group_list": "group_activity_list",
+            "user_list": "user_activity_list",
+            "recently_changed_packages": "recently_changed_packages_activity_list",
+        }
+
+    @classmethod
+    def read_from_hdx(
+        cls, identifier: str, configuration: Configuration | None = None
+    ) -> Optional["Activity"]:
+        """Reads the activity given by identifier from HDX and returns Activity object.
+        HDX requires permission to edit the object the activity relates to.
+
+        Args:
+            identifier: Identifier of activity
+            configuration: HDX configuration. Defaults to global configuration.
+
+        Returns:
+            Activity object if successful read, None if not
+        """
+        return cls._read_from_hdx_class("activity", identifier, configuration)
+
+    def get_object_type(self) -> str:
+        """Get the CKAN object type the activity relates to eg. package,
+        organization, group or user. This is the last word of the activity type
+        (eg. "changed package") with "dataset" (from "follow dataset") mapped to
+        "package".
+
+        Returns:
+            Object type
+        """
+        object_type = self.data["activity_type"].split(" ")[-1]
+        if object_type == "dataset":
+            return "package"
+        return object_type
+
+    def get_data(self) -> dict:
+        """Get the data of the activity which includes a snapshot of the object
+        at the time of the activity. HDX requires permission to edit the object
+        the activity relates to.
+
+        Returns:
+            Activity data
+        """
+        return self._write_to_hdx("data_show", {"id": self.data["id"]}, "id")
+
+    def get_diff(self, diff_type: str = "unified") -> dict:
+        """Get a diff of the object compared to its previous version. HDX requires
+        permission to edit the object the activity relates to.
+
+        Args:
+            diff_type: One of unified, context or html. Defaults to unified.
+
+        Returns:
+            Activity diff
+        """
+        data = {
+            "id": self.data["id"],
+            "object_type": self.get_object_type(),
+            "diff_type": diff_type,
+        }
+        return self._write_to_hdx("diff", data, "id")
+
+    @classmethod
+    def _get_list(
+        cls,
+        action: str,
+        data: dict,
+        configuration: Configuration | None = None,
+        all_pages: bool = False,
+        max_activities: int | None = None,
+    ) -> list["Activity"]:
+        """Helper method to get a list of activities from HDX
+
+        Args:
+            action: Action to perform
+            data: Data to pass to CKAN including any offset and limit
+            configuration: HDX configuration. Defaults to global configuration.
+            all_pages: Whether to get all pages of results. Defaults to False.
+            max_activities: Maximum number of activities if all_pages is True. Defaults to None (no maximum).
+
+        Returns:
+            List of Activity objects
+        """
+        activity = cls(configuration=configuration)
+        if not all_pages:
+            result = activity._write_to_hdx(action, data)
+            return [cls(x, configuration=configuration) for x in result]
+        data = dict(data)
+        limit = data.setdefault("limit", 100)
+        offset = data.get("offset", 0)
+        activities = []
+        while True:
+            data["offset"] = offset
+            result = activity._write_to_hdx(action, data)
+            activities.extend(cls(x, configuration=configuration) for x in result)
+            if max_activities is not None and len(activities) >= max_activities:
+                return activities[:max_activities]
+            if len(result) < limit:
+                return activities
+            offset += limit
+
+    @classmethod
+    def get_activities(
+        cls,
+        object_type: str,
+        identifier: str,
+        configuration: Configuration | None = None,
+        all_pages: bool = False,
+        **kwargs: Any,
+    ) -> list["Activity"]:
+        """Get the activity stream of a CKAN object. One page of results (whose
+        size is given by limit) is returned unless all_pages is True.
+
+        Args:
+            object_type: One of package, organization, group or user
+            identifier: Id or name of object
+            configuration: HDX configuration. Defaults to global configuration.
+            all_pages: Whether to get all pages of results. Defaults to False.
+            **kwargs: See below
+            offset (int): Where to start getting activity items from. Defaults to 0.
+            limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
+            after (int | str): Only activities after this timestamp (package and user only)
+            before (int | str): Only activities before this timestamp (package and user only)
+            include_hidden_activity (bool): Include hidden activity, sysadmins only (package, organization and group only)
+            activity_types (list): Activity types to include (package only)
+            exclude_activity_types (list): Activity types to exclude (package only)
+
+        Returns:
+            List of Activity objects
+        """
+        if not identifier:
+            raise HDXError(f"No id or name for {object_type} to get activities!")
+        data = {"id": identifier}
+        data.update(kwargs)
+        return cls._get_list(f"{object_type}_list", data, configuration, all_pages)
+
+    @classmethod
+    def get_recently_changed_packages(
+        cls,
+        configuration: Configuration | None = None,
+        all_pages: bool = False,
+        max_activities: int | None = 1000,
+        **kwargs: Any,
+    ) -> list["Activity"]:
+        """Get the activity stream of all recently added or changed datasets.
+        One page of results (whose size is given by limit) is returned unless
+        all_pages is True. all_pages is capped by max_activities because the
+        stream covers the whole history of HDX.
+
+        Args:
+            configuration: HDX configuration. Defaults to global configuration.
+            all_pages: Whether to get all pages of results. Defaults to False.
+            max_activities: Maximum number of activities if all_pages is True. Defaults to 1000. None means no maximum (very slow).
+            **kwargs: See below
+            offset (int): Where to start getting activity items from. Defaults to 0.
+            limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
+
+        Returns:
+            List of Activity objects
+        """
+        return cls._get_list(
+            "recently_changed_packages",
+            kwargs,
+            configuration,
+            all_pages,
+            max_activities,
+        )
+
+    def check_required_fields(self, ignore_fields: Sequence[str] = ()) -> None:
+        """Activities are read only so this raises an exception
+
+        Args:
+            ignore_fields: Fields to ignore. Default is ().
+
+        Returns:
+            None
+        """
+        raise HDXError("Activities are read only!")
+
+    def update_in_hdx(self, **kwargs: Any) -> None:
+        """Activities are read only so this raises an exception
+
+        Returns:
+            None
+        """
+        raise HDXError("Activities are read only!")
+
+    def create_in_hdx(self, **kwargs: Any) -> None:
+        """Activities are read only so this raises an exception
+
+        Returns:
+            None
+        """
+        raise HDXError("Activities are read only!")
+
+    def delete_from_hdx(self) -> None:
+        """Activities are read only so this raises an exception
+
+        Returns:
+            None
+        """
+        raise HDXError("Activities are read only!")

{hdx_python_api-6.7.0 → hdx_python_api-6.7.2}/src/hdx/data/dataset.py

@@ -40,6 +40,7 @@ from hdx.api.configuration import Configuration
 from hdx.api.locations import Locations
 from hdx.api.utilities.date_helper import DateHelper
 from hdx.api.utilities.filestore_helper import FilestoreHelper
+from hdx.data.activity import Activity
 from hdx.data.hdxobject import HDXError, HDXObject
 from hdx.data.resource_matcher import ResourceMatcher
 
@@ -1428,6 +1429,32 @@ class Dataset(HDXObject):
                 resources.append(resource)
         return resources
 
+    def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activity]:
+        """Get the activity stream of this dataset. One page of results (whose size
+        is given by limit) is returned unless all_pages is True.
+
+        Args:
+            all_pages: Whether to get all pages of results. Defaults to False.
+            **kwargs: See below
+            offset (int): Where to start getting activity items from. Defaults to 0.
+            limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
+            after (int | str): Only activities after this timestamp
+            before (int | str): Only activities before this timestamp
+            include_hidden_activity (bool): Include hidden activity (sysadmins only)
+            activity_types (list): Activity types to include
+            exclude_activity_types (list): Activity types to exclude
+
+        Returns:
+            List of Activity objects
+        """
+        return Activity.get_activities(
+            "package",
+            self.data.get("id") or self.data.get("name"),
+            self.configuration,
+            all_pages,
+            **kwargs,
+        )
+
     @classmethod
     def autocomplete(
         cls,

{hdx_python_api-6.7.0 → hdx_python_api-6.7.2}/src/hdx/data/location.py

@@ -5,6 +5,7 @@ from collections.abc import Sequence
 from typing import Any, Optional
 
 from hdx.api.configuration import Configuration
+from hdx.data.activity import Activity
 from hdx.data.hdxobject import HDXObject
 
 logger = logging.getLogger(__name__)
@@ -217,6 +218,28 @@ class Location(HDXObject):
             if len(group["name"]) == 3 and group.get("data_completeness") == "active"
         )
 
+    def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activity]:
+        """Get the activity stream of this location. One page of results (whose size
+        is given by limit) is returned unless all_pages is True.
+
+        Args:
+            all_pages: Whether to get all pages of results. Defaults to False.
+            **kwargs: See below
+            offset (int): Where to start getting activity items from. Defaults to 0.
+            limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
+            include_hidden_activity (bool): Include hidden activity (sysadmins only)
+
+        Returns:
+            List of Activity objects
+        """
+        return Activity.get_activities(
+            "group",
+            self.data.get("id") or self.data.get("name"),
+            self.configuration,
+            all_pages,
+            **kwargs,
+        )
+
     @classmethod
     def autocomplete(
         cls,

{hdx_python_api-6.7.0 → hdx_python_api-6.7.2}/src/hdx/data/organization.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 import hdx.data.dataset
 import hdx.data.user as user_module
 from hdx.api.configuration import Configuration
+from hdx.data.activity import Activity
 from hdx.data.hdxobject import HDXError, HDXObject
 
 if TYPE_CHECKING:
@@ -311,6 +312,28 @@ class Organization(HDXObject):
 
         return final_orgs
 
+    def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activity]:
+        """Get the activity stream of this organization. One page of results (whose size
+        is given by limit) is returned unless all_pages is True.
+
+        Args:
+            all_pages: Whether to get all pages of results. Defaults to False.
+            **kwargs: See below
+            offset (int): Where to start getting activity items from. Defaults to 0.
+            limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
+            include_hidden_activity (bool): Include hidden activity (sysadmins only)
+
+        Returns:
+            List of Activity objects
+        """
+        return Activity.get_activities(
+            "organization",
+            self.data.get("id") or self.data.get("name"),
+            self.configuration,
+            all_pages,
+            **kwargs,
+        )
+
     @classmethod
     def autocomplete(
         cls,