hdx-python-pipelineutils 0.0.1__py3-none-any.whl

hdx/python/pipelineutils/__init__.py

@@ -0,0 +1,55 @@
+import re
+from datetime import datetime
+
+from hdx.data.dataset import Dataset
+
+template = re.compile("{{.*?}}")
+
+
+def string_params_to_dict(string: str) -> dict[str, str]:
+    params = {}
+    if not string:
+        return params
+    for name_par in string.split(","):
+        name, par = name_par.strip().split(":")
+        params[name] = par.strip()
+    return params
+
+
+def match_template(input: str) -> tuple[str | None, str | None]:
+    """Try to match {{XXX}} in input string
+
+    Args:
+        input: String in which to look for template
+
+    Returns:
+        (Matched string with brackets, matched string without brackets)
+    """
+    match = template.search(input)
+    if match:
+        template_string = match.group()
+        return template_string, template_string[2:-2]
+    return None, None
+
+
+def get_startend_dates_from_time_period(
+    dataset: Dataset, today: datetime | None = None
+) -> dict | None:
+    """Return the time period in form required for source_date
+
+    Args:
+        dataset: Dataset object
+        today: Date to use for today. Default is None (datetime.utcnow)
+
+    Returns:
+        Time period in form required for source_date
+    """
+    if today is None:
+        date_info = dataset.get_time_period()
+    else:
+        date_info = dataset.get_time_period(today=today)
+    startdate = date_info.get("startdate")
+    enddate = date_info.get("enddate")
+    if enddate is None:
+        return None
+    return {"start": startdate, "end": enddate}

hdx/python/pipelineutils/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.0.1'
+__version_tuple__ = version_tuple = (0, 0, 1)
+
+__commit_id__ = commit_id = None

hdx/python/pipelineutils/hapi_admins.py

@@ -0,0 +1,117 @@
+from hdx.location.adminlevel import AdminLevel
+
+
+def complete_admins(
+    admins: list[AdminLevel],
+    countryiso3: str,
+    provider_adm_names: list,
+    adm_codes: list,
+    adm_names: list,
+    fuzzy_match: bool = True,
+) -> tuple[int, list[str]]:
+    """Use information from adm_codes to populate adm_names and from
+    provider_adm_names to populate adm_codes with outptu of the admin level
+    and arnings for unknown and mismatched p-codes. All provided lists
+    should be of the same length.
+
+    Args:
+        admins: List of AdminLevel objects
+        countryiso3: Country ISO3 code
+        provider_adm_names: List of provider adm names
+        adm_codes: List of adm codes
+        adm_names: List of adm names
+        fuzzy_match: Whether to use fuzzy matching. Default is True.
+
+    Returns:
+        Admin level and warnings
+    """
+
+    warnings = []
+    child = None
+    adm_level = len(provider_adm_names)
+
+    def check_unknown_pcode(adm_code: str, pcode: str) -> str:
+        if pcode:
+            warnings.append(f"PCode unknown {adm_code}->{pcode} ({warntxt})")
+            return pcode
+        else:
+            warnings.append(f"PCode unknown {adm_code}->''")
+            return ""
+
+    for i, provider_adm_name in reversed(list(enumerate(provider_adm_names))):
+        adm_code = adm_codes[i]
+        parent = admins[i].pcode_to_parent.get(adm_code)
+        if not parent and i > 0:
+            parent = adm_codes[i - 1]
+        if not provider_adm_name:
+            provider_adm_name = ""
+            provider_adm_names[i] = ""
+        if child:
+            pcode = admins[i + 1].pcode_to_parent.get(child)
+            warntxt = "parent"
+        elif provider_adm_name:
+            pcode, _ = admins[i].get_pcode(
+                countryiso3,
+                provider_adm_name,
+                parent=parent,
+                fuzzy_match=fuzzy_match,
+            )
+            warntxt = f"provider_adm{i + 1}_name"
+        else:
+            pcode = None
+        if adm_code:
+            if adm_code not in admins[i].pcodes:
+                if admins[i].looks_like_pcode(adm_code):
+                    adj_adm_code = admins[i].convert_admin_pcode_length(
+                        countryiso3, adm_code, parent=parent
+                    )
+                    if adj_adm_code:
+                        warnings.append(f"PCode length {adm_code}->{adj_adm_code}")
+                        adm_code = adj_adm_code
+                    else:
+                        adm_code = check_unknown_pcode(adm_code, pcode)
+                else:
+                    adm_code = check_unknown_pcode(adm_code, pcode)
+            elif pcode and adm_code != pcode:
+                if child:
+                    warnings.append(f"PCode mismatch {adm_code}->{pcode} ({warntxt})")
+                    adm_code = pcode
+                else:
+                    warnings.append(f"PCode mismatch {adm_code} != {provider_adm_name}")
+        elif pcode:
+            adm_code = pcode
+        else:
+            adm_code = ""
+        adm_codes[i] = adm_code
+        if adm_code:
+            adm_names[i] = admins[i].pcode_to_name.get(adm_code, "")
+            child = adm_code
+        else:
+            adm_names[i] = ""
+            if provider_adm_name == "":
+                adm_level -= 1
+    return adm_level, warnings
+
+
+def pad_admins(
+    provider_adm_names: list[str],
+    adm_codes: list[str],
+    adm_names: list[str],
+    adm_level: int = 2,
+) -> None:
+    """Pad lists to size given in adm_level adding as many "" as needed.
+
+    Args:
+        provider_adm_names: List of provider adm names
+        adm_codes: List of adm codes
+        adm_names: List of adm names
+        adm_level: Admin level to which to pad. Default is 2.
+
+    Returns:
+        Admin level and warnings
+    """
+
+    for i in range(len(provider_adm_names), adm_level):
+        provider_adm_names.append("")
+        adm_codes.append("")
+        adm_names.append("")

hdx/python/pipelineutils/lookup.py

@@ -0,0 +1,113 @@
+import logging
+from copy import copy
+from pathlib import Path
+
+from hdx.utilities.loader import load_yaml
+from hdx.utilities.matching import get_code_from_name
+from hdx.utilities.path import script_dir_plus_file
+from hdx.utilities.text import normalise
+
+from hdx.python.pipelineutils.reader import Read
+
+logger = logging.getLogger(__name__)
+
+
+class Lookup:
+    """Lookup class. YAML input is in this form:
+    https://github.com/OCHA-DAP/hdx-python-scraper/blob/main/src/hdx/scraper/framework/utilities/sector_configuration.yaml
+
+    Args:
+        yaml_config_path: YAML configuration file
+        classobject: Child class
+    """
+
+    def __init__(self, yaml_config_path: Path | str, classobject: type):
+        configuration = load_yaml(script_dir_plus_file(yaml_config_path, classobject))
+        self._configuration = configuration
+        initial_lookup = configuration.get("initial_lookup", {})
+        self._code_lookup = copy(initial_lookup)
+        self._code_to_name = {}
+        self._unmatched = []
+        self.setup()
+
+    def add_to_lookup(self, code: str, name: str) -> None:
+        """Add code and name to lookup
+
+        Args:
+            code: Code to add to lookup
+            name: Name to add to lookup
+
+        Returns:
+            None
+        """
+
+        self._code_lookup[name] = code
+        self._code_lookup[code] = code
+        self._code_lookup[normalise(name)] = code
+        self._code_lookup[normalise(code)] = code
+        self._code_to_name[code] = name
+
+    def setup(self) -> None:
+        """Setup lookup from YAML configuration
+
+        Returns:
+            None
+        """
+
+        log_message = self._configuration["log_message"]
+        logger.info(f"Populating {log_message}")
+
+        reader = Read.get_reader()
+        datasetinfo = self._configuration["datasetinfo"]
+        headers, iterator = reader.read(
+            datasetinfo, file_prefix=self._configuration["file_prefix"]
+        )
+        code_key = self._configuration["code_key"]
+        name_key = self._configuration["name_key"]
+        for row in iterator:
+            self.add_to_lookup(
+                code=row[code_key],
+                name=row[name_key],
+            )
+
+        extra_entries = self._configuration.get("extra_entries", {})
+        for code, name in extra_entries.items():
+            self.add_to_lookup(code=code, name=name)
+
+    def get_code(self, code: str) -> str | None:
+        """Get code from lookup using fuzzy matching if needed
+
+        Args:
+            code: Code to get from lookup
+
+        Returns:
+            Code obtained from lookup or None if no code is found
+        """
+
+        return get_code_from_name(
+            name=code,
+            code_lookup=self._code_lookup,
+            unmatched=self._unmatched,
+        )
+
+    def get_name(self, code: str, default: str | None = None) -> str | None:
+        """Get name from code
+
+        Args:
+            code: Code to lookup
+            default: Default name to return if code is not found
+
+        Returns:
+            Name obtained from lookup or default if no name is found
+        """
+
+        return self._code_to_name.get(code, default)
+
+    def get_code_to_name(self) -> dict[str, str]:
+        """Get the code to name lookup dictionary
+
+        Returns:
+            Code to name lookup dictionary
+        """
+
+        return self._code_to_name

hdx/python/pipelineutils/org_type.py

@@ -0,0 +1,8 @@
+from .lookup import Lookup
+
+
+class OrgType(Lookup):
+    """Populate the org type mapping."""
+
+    def __init__(self):
+        super().__init__("org_type_configuration.yaml", OrgType)

hdx/python/pipelineutils/org_type_configuration.yaml

@@ -0,0 +1,76 @@
+log_message: "Org type mapping"
+file_prefix: "org_type"
+code_key: "HR.info ID"
+name_key: "Preferred Term"
+
+datasetinfo:
+  dataset: "organization-types-beta"
+  resource: "Organization Types (Beta) - CSV no HXL"
+  format: "csv"
+  headers: 1
+
+extra_entries:
+  "501": "Civil Society"
+  "502": "Observer"
+  "503": "Development Programme"
+  "504": "Local NGO"
+
+initial_lookup:
+  academy: "431"
+  agence un: "447"
+  agence des nations unies: "447"
+  agence du systeme des nations unies: "447"
+  autre type: "443"
+  civile society: "501"
+  cooperation internationale: "501"
+  croix rouge: "445"
+  donantes: "433"
+  entite gouvernementale: "435"
+  gouv: "435"
+  gouvernement: "435"
+  gov: "435"
+  government agency: "435"
+  gobierno: "435"
+  govt: "435"
+  iglesia grupo religioso: "446"
+  ingo: "437"
+  institution etatique: "435"
+  institución publica: "435"
+  international government: "435"
+  international institution: "438"
+  international ngos: "437"
+  local ngo: "504"
+  mouv cr: "445"
+  mouvement croix rouge: "445"
+  mouvement de la croix rouge: "445"
+  movimiento de la cruz roja media luna roja: "445"
+  national local ngo: "441"
+  nations unies: "447"
+  naciones unidas: "447"
+  nngo: "441"
+  observer: "502"
+  ong int: "437"
+  ong internacional: "437"
+  ong internationale: "437"
+  ong local: "504"
+  ong nat: "441"
+  ong national: "441"
+  ong nacional: "441"
+  ong nationale: "441"
+  organisation des nations unies: "447"
+  organismo internacional: "438"
+  others: "443"
+  otro: "443"
+  programme de developpement: "503"
+  red cross: "445"
+  red cross movement: "445"
+  red cross mvt: "445"
+  red cross and red crescent movement: "445"
+  religious group: "446"
+  religious organization: "446"
+  service etatique: "435"
+  sociedad civil: "501"
+  societe civile: "501"
+  un: "447"
+  un agency: "447"
+  un agencies: "447"

hdx/python/pipelineutils/reader.py

@@ -0,0 +1,595 @@
+import glob
+import logging
+from collections.abc import Iterator, Sequence
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import parse_qsl
+
+from hdx.api.configuration import Configuration
+from hdx.api.utilities.url_utils import follow_url
+from hdx.data.dataset import Dataset
+from hdx.data.resource import Resource
+from hdx.utilities.dateparse import parse_date
+from hdx.utilities.downloader import Download
+from hdx.utilities.retriever import Retrieve
+from hdx.utilities.saver import save_json
+from slugify import slugify
+
+from . import get_startend_dates_from_time_period, match_template
+
+logger = logging.getLogger(__name__)
+
+
+class Read(Retrieve):
+    """Read data from tabular source eg. csv, xls, xlsx
+
+    Args:
+        downloader: Download object
+        fallback_dir: Directory containing static fallback data
+        saved_dir: Directory to save or load downloaded data
+        temp_dir: Temporary directory for when data is not needed after downloading
+        save: Whether to save downloaded data. Default is False.
+        use_saved: Whether to use saved data. Default is False.
+        prefix: Prefix to add to filenames. Default is "".
+        delete: Whether to delete saved_dir if save is True. Default is True.
+        today: Value to use for today. Default is None (datetime.utcnow).
+    """
+
+    def __init__(
+        self,
+        downloader: Download,
+        fallback_dir: Path | str,
+        saved_dir: Path | str,
+        temp_dir: Path | str,
+        save: bool = False,
+        use_saved: bool = False,
+        prefix: str = "",
+        delete: bool = True,
+        today: datetime | None = None,
+    ):
+        super().__init__(
+            downloader,
+            fallback_dir,
+            saved_dir,
+            temp_dir,
+            save,
+            use_saved,
+            prefix,
+            delete,
+        )
+        self.today: datetime | None = today
+
+    @classmethod
+    def create_readers(
+        cls,
+        fallback_dir: Path | str,
+        saved_dir: Path | str,
+        temp_dir: Path | str,
+        save: bool = False,
+        use_saved: bool = False,
+        ignore: Sequence[str] = tuple(),
+        rate_limit: dict | None = {"calls": 1, "period": 0.1},
+        today: datetime | None = None,
+        **kwargs: Any,
+    ):
+        """Generate a default reader and an HDX reader. Additional readers are generated
+        if any of header_auths, basic_auths or extra_params are populated. header_auths
+        and basic_auths are dictionaries of form {"scraper name": "auth", ...}.
+        extra_params is of form {"scraper name": {"key": "auth", ...}, ...}.
+
+        Args:
+            fallback_dir: Directory containing static fallback data
+            saved_dir: Directory to save or load downloaded data
+            temp_dir: Temporary directory for when data is not needed after downloading
+            save: Whether to save downloaded data. Default is False.
+            use_saved: Whether to use saved data. Default is False.
+            ignore: Don't generate retrievers for these downloaders
+            rate_limit: Rate limiting per host. Default is {"calls": 1, "period": 0.1}
+            today: Value to use for today. Default is None (datetime.utcnow).
+            **kwargs: See below and parameters of Download class in HDX Python Utilities
+            hdx_auth (str): HDX API key
+            header_auths (Mapping[str, str]): Header authorisations
+            basic_auths (Mapping[str, str]): Basic authorisations
+            param_auths (Mapping[str, str]): Extra parameter authorisations
+
+        Returns:
+            None
+        """
+        if rate_limit:
+            kwargs["rate_limit"] = rate_limit
+        custom_configs = {}
+        hdx_auth = kwargs.get("hdx_auth")
+        if hdx_auth:
+            custom_configs["hdx"] = {"headers": {"Authorization": hdx_auth}}
+            del kwargs["hdx_auth"]
+        header_auths = kwargs.get("header_auths")
+        if header_auths is not None:
+            for name in header_auths:
+                custom_configs[name] = {
+                    "headers": {"Authorization": header_auths[name]}
+                }
+            del kwargs["header_auths"]
+        basic_auths = kwargs.get("basic_auths")
+        if basic_auths is not None:
+            for name in basic_auths:
+                custom_configs[name] = {"basic_auth": basic_auths[name]}
+            del kwargs["basic_auths"]
+        bearer_tokens = kwargs.get("bearer_tokens")
+        if bearer_tokens is not None:
+            for name in bearer_tokens:
+                custom_configs[name] = {"bearer_token": bearer_tokens[name]}
+            del kwargs["bearer_tokens"]
+        param_auths = kwargs.get("param_auths")
+        if param_auths is not None:
+            for name in param_auths:
+                custom_configs[name] = {
+                    "extra_params_dict": dict(parse_qsl(param_auths[name]))
+                }
+            del kwargs["param_auths"]
+        Download.generate_downloaders(custom_configs, **kwargs)
+        cls.generate_retrievers(
+            fallback_dir,
+            saved_dir,
+            temp_dir,
+            save,
+            use_saved,
+            ignore,
+            today=today,
+        )
+
+    @classmethod
+    def get_reader(cls, name: str | None = None) -> "Read":
+        """Get a generated reader given a name. If name is not supplied, the default
+        one will be returned.
+
+        Args:
+            name: Name of reader. Default is None (get default).
+
+        Returns:
+            Reader object
+        """
+        return cls.get_retriever(name)
+
+    @staticmethod
+    def get_url(url: str, **kwargs: Any) -> str:
+        """Get url from a string replacing any template arguments
+
+        Args:
+            url: Url to read
+            **kwargs: Variables to use when evaluating template arguments
+
+        Returns:
+            Url with any template arguments replaced
+        """
+        for kwarg in kwargs:
+            globals()[kwarg] = kwargs[kwarg]
+        template_string, match_string = match_template(url)
+        if template_string:
+            replace_string = eval(match_string)
+            url = url.replace(template_string, replace_string)
+        return url
+
+    def clone(self, downloader: Download) -> "Read":
+        """Clone a given reader but use the given downloader
+
+        Args:
+            downloader: Downloader to use
+
+        Returns:
+            Cloned reader
+
+        """
+        return Read(
+            downloader,
+            fallback_dir=self.fallback_dir,
+            saved_dir=self.saved_dir,
+            temp_dir=self.temp_dir,
+            save=self.save,
+            use_saved=self.use_saved,
+            prefix=self.prefix,
+            delete=False,
+            today=self.today,
+        )
+
+    def setup_tabular(self, datasetinfo: dict, kwargs: dict) -> str | list:
+        """Setup kwargs for tabular source eg. csv, xls, xlsx from
+        datasetinfo and return url.
+
+        Args:
+            datasetinfo: Dictionary of information about dataset
+            kwargs: Parameters to pass to download_file call
+
+        Returns:
+            url or list of urls
+        """
+        sheet = datasetinfo.get("sheet")
+        headers = datasetinfo.get("headers")
+        if headers is None:
+            headers = 1
+            datasetinfo["headers"] = 1
+        format = datasetinfo["format"]
+        kwargs["format"] = format
+        if format in ("xls", "xlsx"):
+            if not sheet:
+                sheet = 1
+            if isinstance(headers, list):
+                kwargs["fill_merged_cells"] = True
+            elif "fill_merged_cells" not in kwargs:
+                kwargs["fill_merged_cells"] = False
+            kwargs["xlsx2csv"] = datasetinfo.get("xlsx2csv", False)
+        if sheet:
+            kwargs["sheet"] = sheet
+        kwargs["headers"] = headers
+        compression = datasetinfo.get("compression")
+        if compression:
+            kwargs["compression"] = compression
+        url = datasetinfo["url"]
+        if isinstance(url, list):
+            url = [self.get_url(x, **kwargs) for x in url]
+        filename = kwargs.get("filename")
+        if not filename:
+            filename = datasetinfo.get("filename")
+            if filename:
+                kwargs["filename"] = filename
+        if filename:
+            # remove file_prefix if filename provided
+            kwargs.pop("file_prefix", None)
+        elif "file_prefix" not in kwargs:
+            file_prefix = datasetinfo.get("file_prefix")
+            if file_prefix:
+                kwargs["file_prefix"] = file_prefix
+        return url
+
+    def read_tabular(
+        self, datasetinfo: dict, **kwargs: Any
+    ) -> tuple[list[str], Iterator[dict]]:
+        """Read data from tabular source eg. csv, xls, xlsx
+
+        Args:
+            datasetinfo: Dictionary of information about dataset
+            **kwargs: Parameters to pass to download_file call
+
+        Returns:
+            Tuple (headers, iterator where each row is a dictionary)
+        """
+        url = self.setup_tabular(datasetinfo, kwargs)
+        return self.get_tabular_rows(
+            url,
+            dict_form=True,
+            **kwargs,
+        )
+
+    def read_dataset(
+        self, dataset_name: str, configuration: Configuration | None = None
+    ) -> Dataset | None:
+        """Read HDX dataset
+
+        Args:
+            dataset_name: Dataset name
+            configuration: HDX configuration. Default is global configuration.
+
+        Returns:
+            The dataset that was read or None
+        """
+        saved_path = self.saved_dir / f"{dataset_name}.json"
+        if self.use_saved:
+            logger.info(f"Using saved dataset {dataset_name} in {saved_path}")
+            dataset = Dataset.load_from_json(saved_path)
+        else:
+            dataset = Dataset.read_from_hdx(dataset_name, configuration)
+            if self.save:
+                logger.info(f"Saving dataset {dataset_name} in {saved_path}")
+                if dataset is None:
+                    save_json(None, saved_path)
+                else:
+                    dataset.save_to_json(saved_path, follow_urls=True)
+        return dataset
+
+    def search_datasets(
+        self,
+        filename: str,
+        query: str | None = "*:*",
+        configuration: Configuration | None = None,
+        page_size: int = 1000,
+        **kwargs: Any,
+    ) -> list[Dataset]:
+        """Read HDX dataset
+
+        Args:
+            filename: Filename for saved files. Will be prefixed by underscore and a number.
+            query: Query (in Solr format). Default is '*:*'.
+            configuration: HDX configuration. Default is global configuration.
+            page_size: Size of page to return. Default is 1000.
+            **kwargs: See below
+            fq (string): Any filter queries to apply
+            rows (int): Number of matching rows to return. Default is all datasets (sys.maxsize).
+            start (int): Offset in the complete result for where the set of returned datasets should begin
+            sort (string): Sorting of results. Default is 'relevance asc, metadata_modified desc' if rows<=page_size or 'metadata_modified asc' if rows>page_size.
+            facet (string): Whether to enable faceted results. Default to True.
+            facet.mincount (int): Minimum counts for facet fields should be included in the results
+            facet.limit (int): Maximum number of values the facet fields return (- = unlimited). Default is 50.
+            facet.field (list[str]): Fields to facet upon. Default is empty.
+            use_default_schema (bool): Use default package schema instead of custom schema. Default is False.
+
+        Returns:
+            list of datasets resulting from query
+        """
+
+        saved_path = self.saved_dir / filename
+        if self.use_saved:
+            logger.info(
+                f"Using saved datasets in {filename}_n.json in {self.saved_dir}"
+            )
+            datasets = []
+            for file_path in sorted(glob.glob(f"{saved_path}_*.json")):
+                datasets.append(Dataset.load_from_json(file_path))
+        else:
+            datasets = Dataset.search_in_hdx(query, configuration, page_size, **kwargs)
+            if self.save:
+                for i, dataset in enumerate(datasets):
+                    file_path = f"{saved_path}_{i}.json"
+                    name = dataset["name"]
+                    logger.info(f"Saving dataset {name} in {file_path}")
+                    dataset.save_to_json(file_path, follow_urls=True)
+        return datasets
+
+    @staticmethod
+    def construct_filename(name: str, format: str):
+        """Construct filename from name and format. The filename of the file
+        comes from the name and format.
+
+        Args:
+            name: Name for the download
+            format: Format of download
+
+        Returns:
+            Filename of file
+        """
+        filename = name.lower()
+        file_type = f".{format}"
+        if filename.endswith(file_type):
+            filename = filename[: -len(file_type)]
+        return f"{slugify(filename, separator='_')}{file_type}"
+
+    def construct_filename_and_download(
+        self, name: str, format: str, url: str, **kwargs: Any
+    ) -> tuple[str, str]:
+        """Construct filename, download file and return the url downloaded and
+        the path of the file. The filename of the file comes from the name and
+        format.
+
+        Args:
+            name: Name for the download
+            format: Format of download
+            url: URL of download
+            **kwargs: Parameters to pass to download_file call
+
+        Returns:
+            (URL that was downloaded, path to downloaded file)
+        """
+        filename = kwargs.get("filename")
+        if not filename:
+            kwargs["filename"] = self.construct_filename(name, format)
+        url = follow_url(url)
+        path = self.download_file(url, **kwargs)
+        return url, path
+
+    def download_resource(self, resource: Resource, **kwargs: Any) -> tuple[str, str]:
+        """Download HDX resource os a file and return the url downloaded and
+        the path of the file. The filename of the file comes from the name and
+        format.
+
+        Args:
+            resource: HDX resource
+            **kwargs: Parameters to pass to download_file call
+
+        Returns:
+            (URL that was downloaded, path to downloaded file)
+        """
+        return self.construct_filename_and_download(
+            resource["name"],
+            resource.get_format(),
+            resource["url"],
+            **kwargs,
+        )
+
+    @staticmethod
+    def get_hapi_dataset_metadata(dataset: Dataset, datasetinfo: dict) -> dict:
+        """Get HAPI dataset metadata from HDX dataset
+
+        Args:
+            dataset: HDX dataset
+            datasetinfo: Dictionary of information about dataset
+
+        Returns:
+            HAPI dataset metadata
+        """
+        license_id = dataset["license_id"]
+        if license_id == "hdx-other":
+            license = dataset["license_other"]
+        else:
+            license_title = dataset["license_title"]
+            license_url = dataset.get("license_url")
+            if license_url:
+                license = f"[{license_title}]({license_url})"
+            else:
+                license = license_title
+        return {
+            "hdx_id": dataset["id"],
+            "hdx_stub": dataset["name"],
+            "title": dataset["title"],
+            "hdx_provider_stub": dataset["organization"]["name"],
+            "hdx_provider_name": dataset["organization"]["title"],
+            "license": license,
+            "time_period": datasetinfo["time_period"],
+        }
+
+    @staticmethod
+    def get_hapi_resource_metadata(resource: Resource) -> dict:
+        """Get HAPI resource metadata from HDX resource
+
+        Args:
+            resource: HDX dataset
+
+        Returns:
+            HAPI resource metadata
+        """
+        return {
+            "hdx_id": resource["id"],
+            "name": resource["name"],
+            "format": resource.get_format(),
+            "update_date": parse_date(resource["last_modified"]),
+            "download_url": resource["url"],
+        }
+
+    def read_hdx_metadata(
+        self,
+        datasetinfo: dict,
+        do_resource_check: bool = True,
+        configuration: Configuration | None = None,
+    ) -> Resource | None:
+        """Read metadata from HDX dataset and add to input dictionary. If url
+        is not supplied, will look through resources for one that matches
+        specified format and use its url unless do_resource_check is False.
+        The dataset key of the parameter datasetinfo will usually point to a
+        string (single dataset) but where sources vary across HXL tags can be
+        a dictionary that maps from HXL tags to datasets with the key
+        default_dataset setting a default for HXL tags. For a single dataset,
+        the keys hapi_dataset_metadata and hapi_resource_metadata will be
+        populated with more detailed dataset and resource information required
+        by HAPI.
+
+        Args:
+            datasetinfo: Dictionary of information about dataset
+            do_resource_check: Whether to check resources. Default is False.
+            configuration: HDX configuration. Default is global configuration.
+
+        Returns:
+            The resource if a url was not given
+        """
+        dataset_nameinfo = datasetinfo["dataset"]
+        dataset = self.read_dataset(dataset_nameinfo, configuration)
+        resource = None
+        url = datasetinfo.get("url")
+        resource_name = datasetinfo.get("resource")
+        # Only loop through resources if do_resource_check is True and
+        # either there is no url in the datasetinfo dictionary or a
+        # resource name has been specified in there
+        if do_resource_check and (not url or resource_name):
+            format = datasetinfo["format"].lower()
+            found = False
+            for resource in dataset.get_resources():
+                if resource["format"].lower() == format:
+                    if resource_name:  # if resource is specified,
+                        if resource["name"] == resource_name:  # match it
+                            found = True
+                            break
+                        continue
+                    else:  # if resource is not specified, use first one
+                        found = True
+                        break
+            if not found:
+                error = [f"Cannot find {format} resource"]
+                if resource_name:
+                    error.append(f"with name {resource_name}")
+                error.append(f"in {dataset_nameinfo}!")
+                raise ValueError(" ".join(error))
+            if url:  # if there is a url in the datasetinfo dictionary,
+                resource["url"] = url  # set the resource url to it
+            else:
+                url = resource["url"]  # otherwise set the url key in
+                # datasetinfo to the resource url (by setting url here)
+            datasetinfo["hapi_resource_metadata"] = self.get_hapi_resource_metadata(
+                resource
+            )
+            datasetinfo["url"] = url
+        if "source_date" not in datasetinfo:
+            datasetinfo["source_date"] = get_startend_dates_from_time_period(
+                dataset, today=self.today
+            )
+
+        def set_date(date, startend):
+            if isinstance(date, str):
+                date = parse_date(date)
+                if startend == "end":
+                    date = date.replace(
+                        hour=23,
+                        minute=59,
+                        second=59,
+                        microsecond=999999,
+                    )
+
+            datasetinfo["time_period"][startend] = date
+
+        start_date = datasetinfo["source_date"]["start"]
+        end_date = datasetinfo["source_date"]["end"]
+        datasetinfo["time_period"] = {}
+        set_date(start_date, "start")
+        set_date(end_date, "end")
+        if "source" not in datasetinfo:
+            datasetinfo["source"] = dataset["dataset_source"]
+        if "source_url" not in datasetinfo:
+            datasetinfo["source_url"] = dataset.get_hdx_url()
+        datasetinfo["hapi_dataset_metadata"] = self.get_hapi_dataset_metadata(
+            dataset, datasetinfo
+        )
+        return resource
+
+    def read_hdx(
+        self,
+        datasetinfo: dict,
+        configuration: Configuration | None = None,
+        **kwargs: Any,
+    ) -> tuple[list[str], Iterator[dict]]:
+        """Read data and metadata from HDX dataset
+
+        Args:
+            datasetinfo: Dictionary of information about dataset
+            configuration: HDX configuration. Default is global configuration.
+            **kwargs: Parameters to pass to download_file call
+
+        Returns:
+            Tuple (headers, iterator where each row is a dictionary)
+        """
+        resource = self.read_hdx_metadata(datasetinfo, configuration=configuration)
+        filename = kwargs.get("filename")
+        if filename:
+            del kwargs["filename"]
+            datasetinfo["filename"] = filename
+        filename = datasetinfo.get("filename")
+        if resource and not filename:
+            filename = self.construct_filename(resource["name"], resource.get_format())
+            file_prefix = kwargs.get("file_prefix")
+            if not file_prefix:
+                file_prefix = datasetinfo.get("file_prefix")
+            if file_prefix:
+                filename = f"{file_prefix}_{filename}"
+            datasetinfo["filename"] = filename
+        return self.read_tabular(datasetinfo, **kwargs)
+
+    def read(
+        self,
+        datasetinfo: dict,
+        configuration: Configuration | None = None,
+        **kwargs: Any,
+    ) -> tuple[list[str], Iterator[dict]]:
+        """Read data and metadata from HDX dataset
+
+        Args:
+            datasetinfo: Dictionary of information about dataset
+            configuration: HDX configuration. Default is global configuration.
+            **kwargs: Parameters to pass to download_file call
+
+        Returns:
+            Tuple (headers, iterator where each row is a dictionary)
+        """
+        format = datasetinfo["format"]
+        if format in ["json", "csv", "xls", "xlsx"]:
+            if "dataset" in datasetinfo:
+                headers, iterator = self.read_hdx(datasetinfo, configuration, **kwargs)
+            else:
+                headers, iterator = self.read_tabular(datasetinfo, **kwargs)
+        else:
+            raise ValueError(f"Invalid format {format} for {datasetinfo['name']}!")
+        return headers, iterator

hdx/python/pipelineutils/sector.py

@@ -0,0 +1,8 @@
+from .lookup import Lookup
+
+
+class Sector(Lookup):
+    """Populate the sector mapping."""
+
+    def __init__(self):
+        super().__init__("sector_configuration.yaml", Sector)

hdx/python/pipelineutils/sector_configuration.yaml

@@ -0,0 +1,179 @@
+log_message: "Sector mapping"
+file_prefix: "sector"
+code_key: "ACRONYM"
+name_key: "Preferred Term"
+
+datasetinfo:
+  dataset: "global-coordination-groups-beta"
+  resource: "Global Coordination Groups (Beta) CSV no HXL"
+  format: "csv"
+  headers: 1
+
+extra_entries:
+  "Cash": "Cash programming"
+  "Hum": "Humanitarian assistance (unspecified)"
+  "Multi": "Multi-sector (unspecified)"
+  "Intersectoral": "Intersectoral"
+
+
+initial_lookup:
+  abna: "SHL"
+  abri: "SHL"
+  abri bna: "SHL"
+  abris: "SHL"
+  abris ame: "SHL"
+  abris bna: "SHL"
+  abris bna cccm: "SHL"
+  abris durgence et nfi: "SHL"
+  abris nfi: "SHL"
+  action contre les mines: "PRO-MIN"
+  aee: "SHL"
+  agr: "FSC"
+  agriculture: "FSC"
+  agua saneamiento e higiene: "WSH"
+  all: "Intersectoral"
+  alojamiento de emergencia: "SHL"
+  alojamiento de emergencia shelter: "SHL"
+  alojamiento energía y enseres: "SHL"
+  alojamientos y asentamientos: "SHL"
+  ame: "SHL"
+  ash: "WSH"
+  assainissement: "WSH"
+  basic assistance: "Cash"
+  camp coordination and camp management: "CCM"
+  camp coordination camp management: "CCM"
+  cash: "Cash"
+  cccm: "CCM"
+  ccs: "CCM"
+  cluster coordination: "CCM"
+  coord services support: "CCM"
+  coordinacion informacion: "CCM"
+  coord support services: "CCM"
+  coordination: "CCM"
+  coordination and common services: "CCM"
+  coordination et gestion des camps: "CCM"
+  cp: "PRO-CPN"
+  css: "CCM"
+  eah: "WSH"
+  eau: "WSH"
+  eau assainissement et hygiene: "WSH"
+  eau hygiene: "WSH"
+  eau hygiene assainissement: "WSH"
+  eau hygiene et assainissement: "WSH"
+  educacion: "EDU"
+  educacion en emergencias: "EDU"
+  education: "EDU"
+  efectivo multiproposito: "Cash"
+  eha: "WSH"
+  emergency shelter and non food items: "SHL"
+  epah: "WSH"
+  erl: "ERY"
+  esnfi: "SHL"
+  ets: "TEL"
+  explosive hazards: "PRO-MIN"
+  fss: "FSC"
+  food: "FSC"
+  food safety: "FSC"
+  food security and agriculture: "FSC"
+  food security and livelihoods: "FSC"
+  food security and nutrition: "FSC"
+  food security livelihood: "FSC"
+  formation professionnelle: "EDU"
+  fsl: "FSC"
+  gestion des sites daccueil temporaires: "SHL"
+  gbv: "PRO-GBV"
+  general protection: "PRO"
+  global protection: "PRO"
+  hlp: "PRO-HLP"
+  housing land property: "PRO-HLP"
+  humanitaire: "Hum"
+  hygiene: "WSH"
+  hygiene assainissement: "WSH"
+  intercluster: "Multi"  # From Somalia 3W, hopefully not to be confused with intersectoral
+  logement terre et biens: "PRO-HLP"
+  logistica: "LOG"
+  logistique: "LOG"
+  lutte contre la traite: "PRO-GBV"
+  ma: "PRO-MIN"
+  manejo y gestion de campamentos: "CCM"
+  migrant protection: "PRO"
+  mpc: "Cash"
+  mpca: "Cash"
+  ms: "Multi"
+  multi purpose cash: "Cash"
+  multi secteur: "Multi"
+  multisectoriel: "Multi"
+  nutricion: "NUT"
+  nutrition: "NUT"
+  operatioanl presence water sanitation hygiene: "WSH"
+  operational presence education in emergencies: "EDU"
+  operational presence emergency shelter non food items: "SHL"
+  operational presence food security agriculture: "FSC"
+  operational presence health: "HEA"
+  operational presence nutrition: "NUT"
+  operational presence protection: "PRO"
+  pro cpm: "PRO-CPN"
+  pronna: "PRO-CPN"
+  propg: "PRO"
+  proteccion infantil: "PRO-CPN"
+  proteccion ninos ninas adolescentes: "PRO-CPN"
+  proteccion violencia de genero: "PRO-GBV"
+  protection: "PRO"
+  protection de lenfance: "PRO-CPN"
+  protection de lenfant: "PRO-CPN"
+  protection generale: "PRO"
+  protection logement terre et propriete: "PRO-HLP"
+  protection ltb: "PRO-HLP"
+  protection lutte anti mines: "PRO-MIN"
+  protection pe: "PRO-CPN"
+  protection protection de lenfant: "PRO-CPN"
+  protection violences basees sur le genre: "PRO-GBV"
+  protection vgb: "PRO-GBV"
+  proteccion: "PRO"
+  provbg: "PRO-GBV"
+  psea: "PRO-GBV"
+  rapid response mechanism: "ERY"
+  rcf: "CCM"
+  rcf education: "EDU"
+  rcf food security and livelihoods: "FSC"
+  rcf health and nutrtion: "HEA"
+  rcf protection: "PRO"
+  recuperacion temprana: "ERY"
+  relevement precoce: "ERY"
+  relevement rapide: "ERY"
+  refugees: "CCM"
+  refugee response: "CCM"
+  refugees migrants multi sector: "CCM"
+  reponse aux refugies: "CCM"
+  sa: "FSC"
+  sal: "HEA"
+  salud: "HEA"
+  same: "FSC"
+  samv: "FSC"
+  sante: "HEA"
+  securite alimentaire: "FSC"
+  securite alimentaire et moyen dexistence: "FSC"
+  seguridad alimentaria: "FSC"
+  seguridad alimentaria y nutricion: "FSC"
+  services humanitaires communs: "Hum"
+  sexual and reproductive health: "HEA"
+  shelter: "SHL"
+  shelter nfi: "SHL"
+  shelter nfis: "SHL"
+  shelter and nfi: "SHL"
+  shelter and nfis: "SHL"
+  shelter and non food items: "SHL"
+  shelter site coordination: "SHL"
+  site management: "CCM"
+  snfi: "SHL"
+  sspe: "PRO-CPN"
+  telecommunications: "TEL"
+  telecommunications durgence: "TEL"
+  telecomunicaciones de emergencia: "TEL"
+  transversal: "Multi"
+  vbg: "PRO-GBV"
+  violences basees sur le genre: "PRO-GBV"
+  violence basee sur le genre: "PRO-GBV"
+  violencia basada en genero: "PRO-GBV"
+  wash: "WSH"
+  water sanitation and hygiene: "WSH"

hdx_python_pipelineutils-0.0.1.dist-info/METADATA

@@ -0,0 +1,50 @@
+Metadata-Version: 2.4
+Name: hdx-python-pipelineutils
+Version: 0.0.1
+Summary: HDX Python Pipeline Utilities
+Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-pipelineutils
+Author-email: Michael Rans <rans@email.com>
+License: MIT
+License-File: LICENSE
+Keywords: HDX,pipelines,scrapers,utilities
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: Unix
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: gspread
+Requires-Dist: hdx-python-api>=6.6.7
+Requires-Dist: hdx-python-country>=4.1.1
+Requires-Dist: hdx-python-utilities>=4.0.8
+Requires-Dist: libhxl
+Requires-Dist: regex
+Provides-Extra: docs
+Requires-Dist: mkapi; extra == 'docs'
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.3.3; extra == 'pandas'
+Description-Content-Type: text/markdown
+
+[![Build Status](https://github.com/OCHA-DAP/hdx-python-pipelineutils/actions/workflows/run-python-tests.yaml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-pipelineutils/actions/workflows/run-python-tests.yaml)
+[![Coverage Status](https://coveralls.io/repos/github/OCHA-DAP/hdx-python-pipelineutils/badge.svg?branch=main&ts=1)](https://coveralls.io/github/OCHA-DAP/hdx-python-pipelineutils?branch=main)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Downloads](https://img.shields.io/pypi/dm/hdx-python-pipelineutils.svg)](https://pypistats.org/packages/hdx-python-pipelineutils)
+
+
+The HDX Python PipelineUtils Library contains various utilities for use by pipelines.
+
+This library is part of the
+[Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have
+humanitarian related data, please upload your datasets to HDX.

hdx_python_pipelineutils-0.0.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+hdx/python/pipelineutils/__init__.py,sha256=KYANd5ijl4h95gKIWtzk8Nw2q2-etHJmXZq-bp-UtZQ,1477
+hdx/python/pipelineutils/_version.py,sha256=8OsTLsIVB9D0HdPTmt5rVwyVUBe9xTVGkRslXicxzkM,520
+hdx/python/pipelineutils/hapi_admins.py,sha256=3dDK_MggacaWoH5ZNV7y5_q3el7E9dmax8s2A3kkS5k,3935
+hdx/python/pipelineutils/lookup.py,sha256=-6_ILD04gqcUIUIO7er8lx4w0qEDJmmhaciqDV4TcJI,3361
+hdx/python/pipelineutils/org_type.py,sha256=euQyRV01yA8kJ3nMFvZxnTRLnvCuxgV1ZZQx8gEOB8Y,183
+hdx/python/pipelineutils/org_type_configuration.yaml,sha256=IwDfnHmIDM8PzLtEotArzAR37fc4FSviiLi4NvP8iew,1825
+hdx/python/pipelineutils/reader.py,sha256=dE22z2g88IiquKrw4CbvSgXflr8A9OqODqzHLT5BHhw,22685
+hdx/python/pipelineutils/sector.py,sha256=XGysivvPhTqQfK6z1y96sDJATk3zx7sS_qGqCa4PbaI,177
+hdx/python/pipelineutils/sector_configuration.yaml,sha256=9Z3qEI-9FLKUV8xXIixpxaqGoCaTmzelEGUXMVIrjkY,4958
+hdx_python_pipelineutils-0.0.1.dist-info/METADATA,sha256=SN4YZl1IMjuHsDgYWhHMx_rdx5Xe8RF21UKUuOjW2rw,2487
+hdx_python_pipelineutils-0.0.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+hdx_python_pipelineutils-0.0.1.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
+hdx_python_pipelineutils-0.0.1.dist-info/RECORD,,

hdx_python_pipelineutils-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

hdx_python_pipelineutils-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Michael Rans
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.