hdx-python-api 6.6.4__py3-none-any.whl → 6.6.6__py3-none-any.whl

hdx/api/_version.py

@@ -1,5 +1,6 @@
-# file generated by setuptools-scm
+# file generated by vcs-versioning
 # don't change, don't track in version control
+from __future__ import annotations
 
 __all__ = [
     "__version__",
@@ -10,25 +11,14 @@ __all__ = [
     "commit_id",
 ]
 
-TYPE_CHECKING = False
-if TYPE_CHECKING:
-    from typing import Tuple
-    from typing import Union
-
-    VERSION_TUPLE = Tuple[Union[int, str], ...]
-    COMMIT_ID = Union[str, None]
-else:
-    VERSION_TUPLE = object
-    COMMIT_ID = object
-
 version: str
 __version__: str
-__version_tuple__: VERSION_TUPLE
-version_tuple: VERSION_TUPLE
-commit_id: COMMIT_ID
-__commit_id__: COMMIT_ID
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
 
-__version__ = version = '6.6.4'
-__version_tuple__ = version_tuple = (6, 6, 4)
+__version__ = version = '6.6.6'
+__version_tuple__ = version_tuple = (6, 6, 6)
 
 __commit_id__ = commit_id = None

hdx/data/dataset.py

@@ -3,8 +3,7 @@
 import json
 import logging
 import sys
-import warnings
-from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Iterable, Mapping, Sequence
 from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
@@ -26,7 +25,6 @@ from hdx.utilities.dateparse import (
     parse_date_range,
 )
 from hdx.utilities.dictandlist import merge_two_dictionaries
-from hdx.utilities.downloader import Download
 from hdx.utilities.loader import load_json
 from hdx.utilities.saver import save_iterable, save_json
 from hdx.utilities.uuid import is_valid_uuid
@@ -40,7 +38,6 @@ import hdx.data.user as user
 import hdx.data.vocabulary as vocabulary
 from hdx.api.configuration import Configuration
 from hdx.api.locations import Locations
-from hdx.api.utilities.dataset_title_helper import DatasetTitleHelper
 from hdx.api.utilities.date_helper import DateHelper
 from hdx.api.utilities.filestore_helper import FilestoreHelper
 from hdx.data.hdxobject import HDXError, HDXObject
@@ -203,7 +200,7 @@ class Dataset(HDXObject):
         session: Session | None = None,
     ) -> None:
         """Save dataset to JSON. If follow_urls is True, resource urls that point to
-        datasets, HXL proxy urls etc. are followed to retrieve final urls.
+        datasets are followed to retrieve final urls.
 
         Args:
             path: Path to save dataset
@@ -397,35 +394,6 @@ class Dataset(HDXObject):
         """
         return len(self._resources)
 
-    def reorder_resources(
-        self, resource_ids: Sequence[str], hxl_update: bool = True
-    ) -> None:
-        """Reorder resources in dataset according to provided list. Resources are
-        updated in the dataset object to match new order. However, the dataset is not
-        refreshed by rereading from HDX. If only some resource ids are supplied then
-        these are assumed to be first and the other resources will stay in their
-        original order.
-
-        Args:
-            resource_ids: List of resource ids
-
-        Returns:
-            None
-        """
-        dataset_id = self.data.get("id")
-        if not dataset_id:
-            raise HDXError(
-                "Dataset has no id! It must be read, created or updated first."
-            )
-        data = {"id": dataset_id, "order": resource_ids}
-        results = self._write_to_hdx("reorder", data)
-        ordered_ids = results["order"]
-        reordered_resources = []
-        for resource_id in ordered_ids:
-            resource = next(x for x in self._resources if x["id"] == resource_id)
-            reordered_resources.append(resource)
-        self._resources = reordered_resources
-
     def move_resource(
         self,
         resource_name: str,
@@ -519,6 +487,33 @@ class Dataset(HDXObject):
             self.init_resources()
             self.separate_resources()
 
+    def reorder_resources(self, resource_ids: Sequence[str]) -> None:
+        """Reorder resources in dataset according to provided list. Resources are
+        updated in the dataset object to match new order. However, the dataset is not
+        refreshed by rereading from HDX. If only some resource ids are supplied then
+        these are assumed to be first and the other resources will stay in their
+        original order.
+
+        Args:
+            resource_ids: List of resource ids
+
+        Returns:
+            None
+        """
+        dataset_id = self.data.get("id")
+        if not dataset_id:
+            raise HDXError(
+                "Dataset has no id! It must be read, created or updated first."
+            )
+        data = {"id": dataset_id, "order": resource_ids}
+        results = self._write_to_hdx("reorder", data)
+        ordered_ids = results["order"]
+        reordered_resources = []
+        for resource_id in ordered_ids:
+            resource = next(x for x in self._resources if x["id"] == resource_id)
+            reordered_resources.append(resource)
+        self._resources = reordered_resources
+
     def _dataset_load_from_hdx(self, id_or_name: str) -> bool:
         """Loads the dataset given by either id or name from HDX
 
@@ -1080,7 +1075,6 @@ class Dataset(HDXObject):
         remove_additional_resources: bool = False,
         match_resource_order: bool = False,
         create_default_views: bool = True,
-        hxl_update: bool = True,
         **kwargs: Any,
     ) -> dict:
         """Check if dataset exists in HDX and if so, update it. match_resources_by_metadata uses ids if they are
@@ -1146,7 +1140,6 @@ class Dataset(HDXObject):
         remove_additional_resources: bool = False,
         match_resource_order: bool = False,
         create_default_views: bool = True,
-        hxl_update: bool = True,
         **kwargs: Any,
     ) -> dict:
         """Check if dataset exists in HDX and if so, update it, otherwise create it. match_resources_by_metadata uses
@@ -1475,17 +1468,6 @@ class Dataset(HDXObject):
             self.data.get("dataset_date"), date_format, today
         )
 
-    def get_reference_period(
-        self,
-        date_format: str | None = None,
-        today: datetime = now_utc(),
-    ) -> dict:
-        warnings.warn(
-            "get_reference_period() is deprecated, use get_time_period() instead",
-            DeprecationWarning,
-        )
-        return self.get_time_period(date_format, today)
-
     def set_time_period(
         self,
         startdate: datetime | str,
@@ -1512,19 +1494,6 @@ class Dataset(HDXObject):
             startdate, enddate, ongoing, ignore_timeinfo
         )
 
-    def set_reference_period(
-        self,
-        startdate: datetime | str,
-        enddate: datetime | str | None = None,
-        ongoing: bool = False,
-        ignore_timeinfo: bool = True,
-    ) -> None:
-        warnings.warn(
-            "set_reference_period() is deprecated, use set_time_period() instead",
-            DeprecationWarning,
-        )
-        self.set_time_period(startdate, enddate, ongoing, ignore_timeinfo)
-
     def set_time_period_year_range(
         self,
         dataset_year: str | int | Iterable,
@@ -1545,17 +1514,6 @@ class Dataset(HDXObject):
         ) = DateHelper.get_hdx_time_period_from_years(dataset_year, dataset_end_year)
         return retval
 
-    def set_reference_period_year_range(
-        self,
-        dataset_year: str | int | Iterable,
-        dataset_end_year: str | int | None = None,
-    ) -> list[int]:
-        warnings.warn(
-            "set_reference_period_year_range() is deprecated, use set_time_period_year_range() instead",
-            DeprecationWarning,
-        )
-        return self.set_time_period_year_range(dataset_year, dataset_end_year)
-
     @classmethod
     def list_valid_update_frequencies(cls) -> list[str]:
         """List of valid update frequency values
@@ -2266,15 +2224,6 @@ class Dataset(HDXObject):
                 dataset_resource.disable_dataset_preview()
         return preview_resource
 
-    def set_quickchart_resource(
-        self, resource: Union["Resource", dict, str, int]
-    ) -> "Resource":
-        warnings.warn(
-            "set_quickchart_resource() is deprecated, use set_preview_resource() instead",
-            DeprecationWarning,
-        )
-        return self.set_preview_resource(resource)
-
     def create_default_views(self, create_datastore_views: bool = False) -> None:
         """Create default resource views for all resources in dataset
 
@@ -2305,28 +2254,6 @@ class Dataset(HDXObject):
                     self._preview_resourceview = None
                     break
 
-    def generate_quickcharts(
-        self,
-        resource: Union["Resource", dict, str, int] = 0,
-        path: Path | str | None = None,
-        bites_disabled: Sequence[bool] | None = None,
-        indicators: Sequence[dict] | None = None,
-        findreplace: dict | None = None,
-    ) -> None:
-        """To be removed
-
-        Args:
-            resource: Either resource id or name, resource metadata from a Resource object or a dictionary or position. Defaults to 0.
-            path: Path to YAML resource view metadata. Defaults to None (config/hdx_resource_view_static.yaml or internal template).
-            bites_disabled: Which QC bites should be disabled. Defaults to None (all bites enabled).
-            indicators: Indicator codes, QC titles and units for resource view template. Defaults to None (don't use template).
-            findreplace: Replacements for anything else in resource view. Defaults to None.
-
-        Returns:
-            None
-        """
-        return None
-
     def get_name_or_id(self, prefer_name: bool = True) -> str | None:
         """Get dataset name or id eg. for use in urls. If prefer_name is True,
         name is preferred over id if available, otherwise id is preferred over
@@ -2379,33 +2306,6 @@ class Dataset(HDXObject):
             return None
         return f"{self.configuration.get_hdx_site_url()}/api/3/action/package_show?id={name_or_id}"
 
-    def remove_dates_from_title(
-        self, change_title: bool = True, set_time_period: bool = False
-    ) -> list[tuple[datetime, datetime]]:
-        """Remove dates from dataset title returning sorted the dates that were found in
-        title. The title in the dataset metadata will be changed by default. The
-        dataset's metadata field time period will not be changed by default, but if
-        set_time_period is True, then the range with the lowest start date will be used
-        to set the time period field.
-
-        Args:
-            change_title: Whether to change the dataset title. Defaults to True.
-            set_time_period: Whether to set time period from date or range in title. Defaults to False.
-
-        Returns:
-            Date ranges found in title
-        """
-        if "title" not in self.data:
-            raise HDXError("Dataset has no title!")
-        title = self.data["title"]
-        newtitle, ranges = DatasetTitleHelper.get_dates_from_title(title)
-        if change_title:
-            self.data["title"] = newtitle
-        if set_time_period and len(ranges) != 0:
-            startdate, enddate = ranges[0]
-            self.set_time_period(startdate, enddate)
-        return ranges
-
     def generate_resource(
         self,
         folder: Path | str,
@@ -2541,193 +2441,6 @@ class Dataset(HDXObject):
         retdict["rows"] = rows
         return True, retdict
 
-    def generate_resource_from_rows(
-        self,
-        folder: Path | str,
-        filename: str,
-        rows: Iterable[Sequence | Mapping],
-        resourcedata: dict,
-        headers: Sequence[str] | None = None,
-        encoding: str | None = None,
-    ) -> Optional["Resource"]:
-        """Write rows to csv and create resource, adding it to the dataset.
-        The headers argument is either a row number (rows start counting at
-        1), or the actual headers defined as a list of strings. If not set, all
-        rows will be treated as containing values.
-
-        Args:
-            folder: Folder to which to write file containing rows
-            filename: Filename of file to write rows
-            rows: List of rows in dict or list form
-            resourcedata: Resource data
-            headers: List of headers. Defaults to None.
-            encoding: Encoding to use. Defaults to None (infer encoding).
-
-        Returns:
-            The created resource or None if not created
-        """
-        warnings.warn(
-            "generate_resource_from_rows() is deprecated, use generate_resource() instead",
-            DeprecationWarning,
-        )
-        res, retdict = self.generate_resource(
-            folder, filename, rows, resourcedata, headers, headers, "csv", encoding
-        )
-        return retdict["resource"] if res else None
-
-    def generate_resource_from_iterable(
-        self,
-        headers: Sequence[str],
-        iterable: Iterable[Sequence | dict],
-        hxltags: dict[str, str],
-        folder: Path | str,
-        filename: str,
-        resourcedata: dict,
-        datecol: int | str | None = None,
-        yearcol: int | str | None = None,
-        date_function: Callable[[dict], dict | None] | None = None,
-        quickcharts: dict | None = None,
-        encoding: str | None = None,
-    ) -> tuple[bool, dict]:
-        """Given headers and an iterable, write rows to csv and create
-        resource, adding to it the dataset. The returned dictionary will
-        contain the resource in the key resource, headers in the key headers
-        and list of rows in the key rows.
-
-        The time period can optionally be set by supplying a column in
-        which the date or year is to be looked up. Note that any timezone
-        information is ignored and UTC assumed. Alternatively, a function can
-        be supplied to handle any dates in a row. It should accept a row and
-        should return None to ignore the row or a dictionary which can either
-        be empty if there are no dates in the row or can be populated with
-        keys startdate and/or enddate which are of type timezone-aware
-        datetime. The lowest start date and highest end date are used to set
-        the time period and are returned in the results dictionary in keys
-        startdate and enddate.
-
-        Args:
-            headers: Headers
-            iterable: Iterable returning rows
-            hxltags: Header to HXL hashtag mapping
-            folder: Folder to which to write file containing rows
-            filename: Filename of file to write rows
-            resourcedata: Resource data
-            datecol: Date column for setting time period. Defaults to None (don't set).
-            yearcol: Year column for setting dataset year range. Defaults to None (don't set).
-            date_function: Date function to call for each row. Defaults to None.
-            encoding: Encoding to use. Defaults to None (infer encoding).
-
-        Returns:
-            (True if resource added, dictionary of results)
-        """
-        warnings.warn(
-            "generate_resource_from_iterable() is deprecated, use generate_resource() instead",
-            DeprecationWarning,
-        )
-        if [datecol, yearcol, date_function].count(None) < 2:
-            raise HDXError("Supply one of datecol, yearcol or date_function!")
-        retdict = {}
-        if headers is None:
-            return False, retdict
-        rows = [Download.hxl_row(headers, hxltags, dict_form=True)]
-        dates = [default_enddate, default_date]
-        if yearcol is not None:
-
-            def yearcol_function(row):
-                result = {}
-                year = row[yearcol]
-                if year:
-                    result["startdate"], result["enddate"] = parse_date_range(
-                        year,
-                        zero_time=True,
-                        max_endtime=True,
-                    )
-                return result
-
-            date_function = yearcol_function
-        elif datecol is not None:
-
-            def datecol_function(row):
-                result = {}
-                date = row[datecol]
-                if date:
-                    date = parse_date(date)
-                    result["startdate"] = date
-                    result["enddate"] = date
-                return result
-
-            date_function = datecol_function
-
-        for row in iterable:
-            if date_function is not None:
-                result = date_function(row)
-                if result is None:
-                    continue
-                startdate = result.get("startdate")
-                if startdate is not None:
-                    if startdate < dates[0]:
-                        dates[0] = startdate
-                enddate = result.get("enddate")
-                if enddate is not None:
-                    if enddate > dates[1]:
-                        dates[1] = enddate
-            rows.append(row)
-        if len(rows) == 1:
-            logger.error(f"No data rows in {filename}!")
-            return False, retdict
-        if yearcol is not None or date_function is not None:
-            if dates[0] == default_enddate or dates[1] == default_date:
-                logger.error(f"No dates in {filename}!")
-                return False, retdict
-            else:
-                retdict["startdate"] = dates[0]
-                retdict["enddate"] = dates[1]
-                self.set_time_period(dates[0], dates[1])
-        resource = self.generate_resource_from_rows(
-            folder,
-            filename,
-            rows,
-            resourcedata,
-            headers=headers,
-            encoding=encoding,
-        )
-        retdict["resource"] = resource
-        retdict["headers"] = headers
-        retdict["rows"] = rows
-        return True, retdict
-
-    def generate_resource_from_iterator(
-        self,
-        headers: Sequence[str],
-        iterator: Iterator[Sequence | dict],
-        hxltags: dict[str, str],
-        folder: Path | str,
-        filename: str,
-        resourcedata: dict,
-        datecol: int | str | None = None,
-        yearcol: int | str | None = None,
-        date_function: Callable[[dict], dict | None] | None = None,
-        quickcharts: dict | None = None,
-        encoding: str | None = None,
-    ) -> tuple[bool, dict]:
-        warnings.warn(
-            "generate_resource_from_iterator() is deprecated, use generate_resource() instead",
-            DeprecationWarning,
-        )
-        return self.generate_resource_from_iterable(
-            headers,
-            iterator,
-            hxltags,
-            folder,
-            filename,
-            resourcedata,
-            datecol,
-            yearcol,
-            date_function,
-            quickcharts,
-            encoding,
-        )
-
     def download_generate_resource(
         self,
         downloader: BaseDownload,
@@ -2773,7 +2486,6 @@ class Dataset(HDXObject):
         Args:
             downloader: A Download or Retrieve object
             url: URL to download
-            hxltags: Header to HXL hashtag mapping
             folder: Folder to which to write file containing rows
             filename: Filename of file to write rows
             resourcedata: Resource data
@@ -2814,89 +2526,6 @@ class Dataset(HDXObject):
             no_empty=no_empty,
         )
 
-    def download_and_generate_resource(
-        self,
-        downloader: BaseDownload,
-        url: str,
-        hxltags: dict[str, str],
-        folder: Path | str,
-        filename: str,
-        resourcedata: dict,
-        header_insertions: Sequence[tuple[int, str]] | None = None,
-        row_function: Callable[[list[str], dict], dict] | None = None,
-        datecol: str | None = None,
-        yearcol: str | None = None,
-        date_function: Callable[[dict], dict | None] | None = None,
-        quickcharts: dict | None = None,
-        **kwargs: Any,
-    ) -> tuple[bool, dict]:
-        """Download url, write rows to csv and create resource, adding to it
-        the dataset. The returned dictionary will contain the resource in the
-        key resource, headers in the key headers and list of rows in the key
-        rows.
-
-        Optionally, headers can be inserted at specific positions. This is
-        achieved using the header_insertions argument. If supplied, it is a
-        list of tuples of the form (position, header) to be inserted. A
-        function is called for each row. If supplied, it takes as arguments:
-        headers (prior to any insertions) and row (which will be in dict or
-        list form depending upon the dict_rows argument) and outputs a modified
-        row.
-
-        The time period can optionally be set by supplying a column in
-        which the date or year is to be looked up. Note that any timezone
-        information is ignored and UTC assumed. Alternatively, a function can
-        be supplied to handle any dates in a row. It should accept a row and
-        should return None to ignore the row or a dictionary which can either
-        be empty if there are no dates in the row or can be populated with
-        keys startdate and/or enddate which are of type timezone-aware
-        datetime. The lowest start date and highest end date are used to set
-        the time period and are returned in the results dictionary in keys
-        startdate and enddate.
-
-        Args:
-            downloader: A Download or Retrieve object
-            url: URL to download
-            hxltags: Header to HXL hashtag mapping
-            folder: Folder to which to write file containing rows
-            filename: Filename of file to write rows
-            resourcedata: Resource data
-            header_insertions: List of (position, header) to insert. Defaults to None.
-            row_function: Function to call for each row. Defaults to None.
-            datecol: Date column for setting time period. Defaults to None (don't set).
-            yearcol: Year column for setting dataset year range. Defaults to None (don't set).
-            date_function: Date function to call for each row. Defaults to None.
-            **kwargs: Any additional args to pass to downloader.get_tabular_rows
-
-        Returns:
-            (True if resource added, dictionary of results)
-        """
-        warnings.warn(
-            "download_and_generate_resource() is deprecated, use download_generate_resource() instead",
-            DeprecationWarning,
-        )
-        headers, iterator = downloader.get_tabular_rows(
-            url,
-            dict_form=True,
-            header_insertions=header_insertions,
-            row_function=row_function,
-            format="csv",
-            **kwargs,
-        )
-        return self.generate_resource_from_iterable(
-            headers,
-            iterator,
-            hxltags,
-            folder,
-            filename,
-            resourcedata,
-            datecol=datecol,
-            yearcol=yearcol,
-            date_function=date_function,
-            quickcharts=quickcharts,
-            encoding=kwargs.get("encoding", None),
-        )
-
     def add_hapi_error(
         self,
         error_message: str,

hdx/data/resource.py

@@ -1,7 +1,6 @@
 """Resource class containing all logic for creating, checking, and updating resources."""
 
 import logging
-import warnings
 from collections.abc import Sequence
 from datetime import datetime
 from pathlib import Path
@@ -244,13 +243,6 @@ class Resource(HDXObject):
             file_format = file_format.lower()
         return file_format
 
-    def get_file_type(self) -> str | None:
-        warnings.warn(
-            "get_file_type() is deprecated, use get_format() instead",
-            DeprecationWarning,
-        )
-        return self.get_format()
-
     def set_format(self, format: str) -> str:
         """Set the resource's file type
 
@@ -268,13 +260,6 @@ class Resource(HDXObject):
         self.data["format"] = file_format
         return file_format
 
-    def set_file_type(self, file_type: str) -> str:
-        warnings.warn(
-            "set_file_type() is deprecated, use set_format() instead",
-            DeprecationWarning,
-        )
-        return self.set_format(file_type)
-
     def clean_format(self) -> str:
         """Clean the resource's format, setting it to None if it is invalid and
         cannot be mapped
@@ -284,13 +269,6 @@ class Resource(HDXObject):
         """
         return self.set_format(self.data.get("format"))
 
-    def clean_file_type(self) -> str:
-        warnings.warn(
-            "clean_file_type() is deprecated, use clean_format() instead",
-            DeprecationWarning,
-        )
-        return self.set_format(self.data.get("format"))
-
     def get_file_to_upload(self) -> str | None:
         """Get the file uploaded
 

{hdx_python_api-6.6.4.dist-info → hdx_python_api-6.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hdx-python-api
-Version: 6.6.4
+Version: 6.6.6
 Summary: HDX Python API for interacting with the Humanitarian Data Exchange
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-api
 Author: Michael Rans
@@ -28,10 +28,9 @@ Requires-Python: >=3.10
 Requires-Dist: ckanapi>=4.8
 Requires-Dist: defopt>=7.0.0
 Requires-Dist: email-validator
-Requires-Dist: hdx-python-country>=4.0.1
-Requires-Dist: hdx-python-utilities>=4.0.4
+Requires-Dist: hdx-python-country>=4.1.1
+Requires-Dist: hdx-python-utilities>=4.0.8
 Requires-Dist: makefun
-Requires-Dist: quantulum3
 Requires-Dist: requests
 Provides-Extra: docs
 Requires-Dist: mkapi; extra == 'docs'
@@ -53,3 +52,87 @@ For more information, please read the
 This library is part of the [Humanitarian Data Exchange](https://data.humdata.org/)
 (HDX) project. If you have humanitarian related data, please upload your datasets to
 HDX.
+
+# Development
+
+## Environment
+
+Development is currently done using Python 3.13. The environment can be created with:
+
+```shell
+    uv sync
+```
+
+This creates a .venv folder with the versions specified in the project's uv.lock file.
+
+### Pre-commit
+
+pre-commit will be installed when syncing uv. It is run every time you make a git
+commit if you call it like this:
+
+```shell
+    pre-commit install
+```
+
+With pre-commit, all code is formatted according to
+[ruff](https://docs.astral.sh/ruff/) guidelines.
+
+To check if your changes pass pre-commit without committing, run:
+
+```shell
+    pre-commit run --all-files
+```
+
+## Packages
+
+[uv](https://github.com/astral-sh/uv) is used for package management.  If
+you’ve introduced a new package to the source code (i.e. anywhere in `src/`),
+please add it to the `project.dependencies` section of `pyproject.toml` with
+any known version constraints.
+
+To add packages required only for testing, add them to the
+`[dependency-groups]`.
+
+Any changes to the dependencies will be automatically reflected in
+`uv.lock` with `pre-commit`, but you can re-generate the files without committing by
+executing:
+
+```shell
+    uv lock --upgrade
+```
+
+## Project
+
+[uv](https://github.com/astral-sh/uv) is used for project management. The project can be
+built using:
+
+```shell
+    uv build
+```
+
+Linting and syntax checking can be run with:
+
+```shell
+    uv run ruff check
+```
+
+To run the tests and view coverage, execute:
+
+```shell
+    uv run pytest
+```
+
+For the `test_ckan.py` test to run successfully, some configuration is required:
+
+1. The environment variable HDX_KEY_TEST needs to contain a valid key from the HDX demo
+server at https://demo.data-humdata-org.ahconu.org/
+1. Authentication details for Google Sheets need to be obtained and either saved in a
+file named `.gsheet_auth.json` in the home directory (~) or placed in an environment
+variable `GSHEET_AUTH`. The file is preferred for Windows systems since adding such a
+long text string to an environment variable in Windows is challenging.
+
+## Documentation
+
+The documentation, including API documentation, is generated using ReadtheDocs and
+MkDocs with Material. As you change the source code, remember to update the
+documentation at `documentation/index.md`.

{hdx_python_api-6.6.4.dist-info → hdx_python_api-6.6.6.dist-info}/RECORD

@@ -1,23 +1,20 @@
 hdx/api/__init__.py,sha256=11ozJKiUsqDCZ3_mcAHhGYUyGK_Unl54djVSBBExFB4,59
-hdx/api/_version.py,sha256=UEE9I6WlhPAe2Jt5wF9PG4WaLKPa_1VgRmxjpizBjCA,704
+hdx/api/_version.py,sha256=ExjJQQs0P3ByYbVeKqojfK1IwhKHopRm142FJqldcBU,520
 hdx/api/configuration.py,sha256=9WvuVI-3th2Dv70_t_4VoyrlJwB_9qUSN5_zUKbp4sQ,23805
 hdx/api/hdx_base_configuration.yaml,sha256=9yJYroZwoyMQo6tcxGfWHWvYahnTma7Szx0J5Z378z4,2173
 hdx/api/locations.py,sha256=blnqOjEASvh0mWgmo2iEE7ES_xOuRwQo3fOMqTPV_P0,4139
 hdx/api/remotehdx.py,sha256=IjKwP3jutlkrjvWCJPP7YM-xzg3pP_QVmiJfblllBY0,2079
 hdx/api/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hdx/api/utilities/dataset_title_helper.py,sha256=b3Q-ycVC6dtspgJDUqhY7GK7XpXPVeFtlzg_T5gz4Ts,9110
 hdx/api/utilities/date_helper.py,sha256=sXxjRtXz-2rpBVZBbJAk80gfjpYKl_AtHplWr-rniIk,6396
 hdx/api/utilities/filestore_helper.py,sha256=0FcUpGHpYlAzlNDpDxdXbSqV6VHFfGCLakCmK37hieU,6917
 hdx/api/utilities/hdx_error_handler.py,sha256=x1rb2qBO0C6CJjqToSFj_XB1F3pf3nHpA8otxXfTFzA,8711
 hdx/api/utilities/hdx_state.py,sha256=KzrK0A-H5R6cRPqwF4d23_S_zfIa1ow4K5u46FesGmY,2442
 hdx/api/utilities/url_utils.py,sha256=Pb4r-BAqIHYmeoB_i7QjutDp6nOAUh_OHvCtiNLda3Y,4534
 hdx/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hdx/data/dataset.py,sha256=vmqCUgpyx-Dm7oORZu5BBI4gZ5yNbtGBXgRnKOW6DMU,114531
-hdx/data/hdx_datasource_topline.yaml,sha256=kBsb7UPl9RGkPbXfJ-OBQBzIse57-o0T41sbOCd2KVg,366
+hdx/data/dataset.py,sha256=0gS7tQSswTgG0g8YeXg0FL8pFVkocBQSRPFpQAurN00,99455
 hdx/data/hdxobject.py,sha256=_jlwYjb_CcjaXsTXObknMfq-OiNOcpJfLJ9b1vKV1Zo,27494
-hdx/data/indicator_resource_view_template.yaml,sha256=iww2EjGSJdUpN3vhsUEva6ohrXjS-tjR56K5Aly0Xo4,2915
 hdx/data/organization.py,sha256=JHlCl50g1nh65DzN75OgMdvEcT_wfSIPLkhI5Ms-u8g,10588
-hdx/data/resource.py,sha256=URzTM0PJC1rNMNAVhYtom-0wo26twqRnPnYk0EbDjpo,34722
+hdx/data/resource.py,sha256=5j6QK5_J-UFKCT3jZVUZ66VgGsOcCssmGy4jpqdb1N4,34027
 hdx/data/resource_matcher.py,sha256=Yzk9pWa4zk1p09m6zcZMj0XZf5Y16Pa3KToOdUt7rP8,4247
 hdx/data/resource_view.py,sha256=GcYkh0iDAe6_m2ErY_A6i3Z7ZFX949eGt1TLnLZv_8s,7346
 hdx/data/showcase.py,sha256=vnRZ45xd9Ct65eFQkWZtMIJPFifZfKOrH4obO5D5Pvc,14062
@@ -27,7 +24,7 @@ hdx/facades/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdx/facades/infer_arguments.py,sha256=DvyDuozfOPmoaV7HjhQySojGIrMWW9nos31auQBVVl8,4032
 hdx/facades/keyword_arguments.py,sha256=aVapHCfqBO6H7xxBgTdK9ISdgrBFsNypPDWihfQ3edU,1125
 hdx/facades/simple.py,sha256=QWsBkaHG_UM6X3Zu2u3LWVOQzeR24q-EUClrfsL_mkM,1064
-hdx_python_api-6.6.4.dist-info/METADATA,sha256=7mTxQ0xIXxI-E3lFkZJ89oGrcte0I7pHTJLiNBnXK7o,2723
-hdx_python_api-6.6.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-hdx_python_api-6.6.4.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
-hdx_python_api-6.6.4.dist-info/RECORD,,
+hdx_python_api-6.6.6.dist-info/METADATA,sha256=o4V01ehlDyD2YzWP0EbQ_hRuMnOoyXuW_etzGWA-kTk,4945
+hdx_python_api-6.6.6.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+hdx_python_api-6.6.6.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
+hdx_python_api-6.6.6.dist-info/RECORD,,

{hdx_python_api-6.6.4.dist-info → hdx_python_api-6.6.6.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.28.0
+Generator: hatchling 1.29.0
 Root-Is-Purelib: true
 Tag: py3-none-any

hdx/api/utilities/dataset_title_helper.py

@@ -1,254 +0,0 @@
-"""Helper to the Dataset class for handling processing dataset titles."""
-
-import logging
-import re
-from datetime import datetime, timedelta
-from re import Match
-from string import punctuation, whitespace
-
-from dateutil.parser import ParserError
-from hdx.utilities.dateparse import parse_date, parse_date_range
-from hdx.utilities.text import (
-    PUNCTUATION_MINUS_BRACKETS,
-    remove_end_characters,
-    remove_from_end,
-    remove_string,
-)
-from quantulum3 import parser
-
-logger = logging.getLogger(__name__)
-
-
-class DatasetTitleHelper:
-    YEAR_PATTERN = re.compile(r"([12]\d\d\d)")
-    VERSION_PATTERN = re.compile(r"(\d)?(\d)?\d")
-    YEAR_RANGE_PATTERN = re.compile(
-        r"([12]\d\d\d)(/(\d{1,2}))?(-| & | and )([12]\d\d\d)"
-    )
-    YEAR_RANGE_PATTERN2 = re.compile(r"([12]\d\d\d)([/-])(\d{1,2})")
-    PUNCTUATION_PATTERN = re.compile(rf"[{punctuation}]")
-    EMPTY_BRACKET_PATTERN = re.compile(r"(\s?\(\s*\)\s?)")
-    WORD_RIGHT_BRACKET_PATTERN = re.compile(r"\b(\s*)(\w{2,})\b\)")
-    DATE_INTRO_WORDS = ["on", "at", "for", "of", "in"]
-
-    @classmethod
-    def fuzzy_match_dates_in_title(
-        cls,
-        title: str,
-        ranges: list[tuple[datetime, datetime]],
-        ignore_wrong_years: list[int],
-    ) -> str:
-        """
-        Fuzzy match dates in title appending to ranges
-
-        Args:
-            title: Title to parse
-            ranges: List of date ranges found so far
-            ignore_wrong_years: Numbers identified as years that probably are not years
-
-        Returns:
-            Title with dates removed
-
-        """
-        for quant in parser.parse(title):
-            if quant.unit.name == "dimensionless":
-                continue
-            ignore_wrong_years.append(int(quant.value))
-        for match in cls.YEAR_PATTERN.finditer(title):
-            year = match.group(0)
-            if int(year) in ignore_wrong_years:
-                continue
-            start = match.start()
-            end = match.end()
-            stringlr = title[max(start - 13, 0) : end]
-            fuzzylr = {}
-            startdatelr = None
-            enddatelr = None
-            deltalr = timedelta(days=1000)
-            try:
-                startdatelr, enddatelr = parse_date_range(
-                    stringlr,
-                    fuzzy=fuzzylr,
-                    zero_time=True,
-                    max_endtime=True,
-                )
-                if startdatelr and enddatelr:
-                    deltalr = enddatelr - startdatelr
-            except ParserError:
-                pass
-            fuzzyrl = {}
-            stringrl = title[start : min(end + 13, len(title))]
-            startdaterl = None
-            enddaterl = None
-            deltarl = timedelta(days=1000)
-            try:
-                startdaterl, enddaterl = parse_date_range(
-                    stringrl,
-                    fuzzy=fuzzyrl,
-                    zero_time=True,
-                    max_endtime=True,
-                )
-                if startdaterl and enddaterl:
-                    deltarl = enddaterl - startdaterl
-            except ParserError:
-                pass
-            if startdatelr and deltalr <= deltarl:
-                date_components = fuzzylr["date"]
-                ranges.append((startdatelr, enddatelr))
-            elif startdaterl:
-                date_components = fuzzyrl["date"]
-                ranges.append((startdaterl, enddaterl))
-            else:
-                date_components = year
-                ranges.append(parse_date_range(year, zero_time=True, max_endtime=True))
-            newtitle = title
-            for date_component in date_components:
-                newtitle = remove_string(
-                    newtitle, date_component, PUNCTUATION_MINUS_BRACKETS
-                )
-            logger.info(f"Removing date from title: {title} -> {newtitle}")
-            title = newtitle
-        try:
-            fuzzy = {}
-            startdate, enddate = parse_date_range(
-                title,
-                fuzzy=fuzzy,
-                zero_time=True,
-                max_endtime=True,
-            )
-            datestrs = fuzzy["date"]
-            if (
-                startdate.date() == enddate.date() and len(datestrs) == 1
-            ):  # only accept dates where day, month and year are
-                # all together not split throughout the string and where the date is a precise day not a range
-                date_component = datestrs[0]
-                index = title.find(date_component) - 1
-                if index >= 0 and title[index].lower() == "v":
-                    return title
-                ranges.append((startdate, enddate))
-                newtitle = remove_string(
-                    title, date_component, PUNCTUATION_MINUS_BRACKETS
-                )
-                logger.info(f"Removing date from title: {title} -> {newtitle}")
-                title = newtitle
-        except (ParserError, OverflowError):
-            pass
-
-        return title
-
-    @classmethod
-    def get_month_year_in_slash_range(
-        cls, match: Match, ignore_wrong_years: list[int]
-    ) -> tuple[int | None, int | None, int | None]:
-        """
-        Get year(s) and month from slash range eg. 2007/08. If second value is between 1 and 12, take it as a month.
-
-        Args:
-            match (Match): Match object
-            ignore_wrong_years: Numbers identified as years that probably are not years
-
-        Returns:
-            First year, first month, second year
-
-        """
-        first_year_str = match.group(1)
-        first_year = int(first_year_str)
-        if first_year in ignore_wrong_years:
-            return None, None, None
-        two_digits_str = match.group(3)
-        if two_digits_str:
-            two_digits = int(two_digits_str)
-            if 1 <= two_digits <= 12:
-                return first_year, two_digits, None
-            else:
-                second_year = int(f"{first_year_str[:2]}{two_digits_str}")
-                if second_year > first_year:
-                    return first_year, None, second_year
-                else:
-                    ignore_wrong_years.append(first_year)
-                    return None, None, None
-        else:
-            return first_year, None, None
-
-    @classmethod
-    def get_dates_from_title(
-        cls, title: str
-    ) -> tuple[str, list[tuple[datetime, datetime]]]:
-        """
-        Get dataset dates (start and end dates in a list) from title and clean title of dates
-
-        Args:
-            title: Title to get date from and clean
-
-        Returns:
-            Cleaned title, list of start and end dates
-
-        """
-        ranges = []
-        ignore_wrong_years = []
-        for match in cls.YEAR_RANGE_PATTERN.finditer(title):
-            (
-                first_year,
-                first_month,
-                second_year,
-            ) = cls.get_month_year_in_slash_range(match, ignore_wrong_years)
-            if first_year is None:
-                continue
-            if first_month is None:
-                first_month = 1
-            startdate = parse_date(
-                f"{first_year}-{first_month}-01",
-                "%Y-%m-%d",
-                zero_time=True,
-            )
-            enddate = parse_date(
-                f"{match.group(5)}-12-31",
-                "%Y-%m-%d",
-                max_time=True,
-            )
-            ranges.append((startdate, enddate))
-            newtitle = remove_string(title, match.group(0))
-            logger.info(f"Removing date range from title: {title} -> {newtitle}")
-            title = newtitle
-
-        for match in cls.YEAR_RANGE_PATTERN2.finditer(title):
-            (
-                first_year,
-                first_month,
-                second_year,
-            ) = cls.get_month_year_in_slash_range(match, ignore_wrong_years)
-            if first_year is None or second_year is None:
-                continue
-            startdate = parse_date(
-                f"{first_year}-01-01",
-                "%Y-%m-%d",
-                zero_time=True,
-            )
-            enddate = parse_date(
-                f"{second_year}-12-31",
-                "%Y-%m-%d",
-                max_time=True,
-            )
-            ranges.append((startdate, enddate))
-            newtitle = remove_string(title, match.group(0))
-            logger.info(f"Removing date range from title: {title} -> {newtitle}")
-            title = newtitle
-
-        title = cls.fuzzy_match_dates_in_title(title, ranges, ignore_wrong_years)
-
-        for match in cls.WORD_RIGHT_BRACKET_PATTERN.finditer(title):
-            word = match.group(2)
-            if word in cls.DATE_INTRO_WORDS:
-                title = title.replace(match.group(0), ")")
-
-        for match in cls.EMPTY_BRACKET_PATTERN.finditer(title):
-            title = title.replace(match.group(0), " ")
-        title = remove_end_characters(
-            title, f"{PUNCTUATION_MINUS_BRACKETS}{whitespace}"
-        )
-        title = remove_from_end(
-            title,
-            ["as of"] + cls.DATE_INTRO_WORDS,
-            "Removing - from title: %s -> %s",
-        )
-        return title, sorted(ranges)

hdx/data/hdx_datasource_topline.yaml

@@ -1,30 +0,0 @@
-#Topline configuration
-schema:
-  -
-    id: code
-    type: text
-  -
-    id: title
-    type: text
-  -
-    id: value
-    type: float
-  -
-    id: latest_date
-    type: timestamp
-  -
-    id: source
-    type: text
-  -
-    id: source_link
-    type: text
-  -
-    id: notes
-    type: text
-  -
-    id: explore
-    type: text
-  -
-    id: units
-    type: text
-primary_key: code

hdx/data/indicator_resource_view_template.yaml

@@ -1,4 +0,0 @@
-description: ""
-title: "Quick Charts"
-view_type: "hdx_hxl_preview"
-hxl_preview_config: '{"configVersion":5,"bites":[{"tempShowSaveCancelButtons":false,"ingredient":{"aggregateColumn":"AGGREGATE_COL_1","valueColumn":"VALUE_COL_1","aggregateFunction":"sum","dateColumn":"DATE_COL_1","comparisonValueColumn":null,"comparisonOperator":null,"filters":{"filterWith":[{"CODE_COL_1":"CODE_VALUE_1"}]},"description":"DESCRIPTION_VALUE_1"},"type":"timeseries","errorMsg":null,"computedProperties":{"explainedFiltersMap":{"remove empty valued rows":{"filterWith":[{"VALUE_COL_1":"is not empty"}],"filterWithout":[]}},"pieChart":false,"title":"Sum of Value by Year","dataTitle":"UNIT_VALUE_1"},"uiProperties":{"swapAxis":true,"showGrid":true,"color":"#0077ce","sortingByValue1":"DESC","sortingByCategory1":null,"showPoints":true,"internalColorPattern":["#1ebfb3","#0077ce","#f2645a","#9C27B0"],"title":"TITLE_VALUE_1","dateFormat":"DATE_FORMAT_1"},"dataProperties":{},"displayCategory":"Timeseries","hashCode":498306664},{"tempShowSaveCancelButtons":false,"ingredient":{"aggregateColumn":"AGGREGATE_COL_2","valueColumn":"VALUE_COL_2","aggregateFunction":"sum","dateColumn":"DATE_COL_2","comparisonValueColumn":null,"comparisonOperator":null,"filters":{"filterWith":[{"CODE_COL_2":"CODE_VALUE_2"}]},"description":"DESCRIPTION_VALUE_2"},"type":"timeseries","errorMsg":null,"computedProperties":{"explainedFiltersMap":{"remove empty valued rows":{"filterWith":[{"VALUE_COL_2":"is not empty"}],"filterWithout":[]}},"pieChart":false,"title":"Sum of Value by Year","dataTitle":"UNIT_VALUE_2"},"uiProperties":{"swapAxis":true,"showGrid":true,"color":"#0077ce","sortingByValue1":"DESC","sortingByCategory1":null,"showPoints":true,"internalColorPattern":["#1ebfb3","#0077ce","#f2645a","#9C27B0"],"title":"TITLE_VALUE_2","dateFormat":"DATE_FORMAT_2"},"dataProperties":{},"displayCategory":"Timeseries","hashCode":498306664},{"tempShowSaveCancelButtons":false,"ingredient":{"aggregateColumn":"AGGREGATE_COL_3","valueColumn":"VALUE_COL_3","aggregateFunction":"sum","dateColumn":"DATE_COL_3","comparisonValueColumn":null,"comparisonOperator":null,"filters":{"filterWith":[{"CODE_COL_3":"CODE_VALUE_3"}]},"description":"DESCRIPTION_VALUE_3"},"type":"timeseries","errorMsg":null,"computedProperties":{"explainedFiltersMap":{"remove empty valued rows":{"filterWith":[{"VALUE_COL_3":"is not empty"}],"filterWithout":[]}},"pieChart":false,"title":"Sum of Value by Year","dataTitle":"UNIT_VALUE_3"},"uiProperties":{"swapAxis":true,"showGrid":true,"color":"#0077ce","sortingByValue1":"DESC","sortingByCategory1":null,"showPoints":true,"internalColorPattern":["#1ebfb3","#0077ce","#f2645a","#9C27B0"],"title":"TITLE_VALUE_3","dateFormat":"DATE_FORMAT_3"},"dataProperties":{},"displayCategory":"Timeseries","hashCode":498306664}],"recipeUrl":"https://raw.githubusercontent.com/mcarans/hxl-recipes/dev/recipes/historic-values-filtered/recipe.json"}'