oda-data 2.0.0__tar.gz → 2.0.2__tar.gz

{oda_data-2.0.0 → oda_data-2.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: oda_data
-Version: 2.0.0
+Version: 2.0.2
 Summary: A python package to work with Official Development Assistance data from the OECD DAC.
 Author: Jorge Rivera
 Author-email: jorge.rivera@one.org
@@ -49,10 +49,10 @@ the [issues page](https://github.com/ONEcampaign/oda_data_package/issues).
 
 ### Installation
 
-The beta version of the package can be installed using pip:
+The latest version of the package can be installed using pip:
 
 ```bash
-pip install --pre oda-data --upgrade
+pip install oda-data --upgrade
 ```
 
 The package is compatible with Python 3.10 and above.

{oda_data-2.0.0 → oda_data-2.0.2}/README.md

@@ -22,10 +22,10 @@ the [issues page](https://github.com/ONEcampaign/oda_data_package/issues).
 
 ### Installation
 
-The beta version of the package can be installed using pip:
+The latest version of the package can be installed using pip:
 
 ```bash
-pip install --pre oda-data --upgrade
+pip install oda-data --upgrade
 ```
 
 The package is compatible with Python 3.10 and above.

{oda_data-2.0.0 → oda_data-2.0.2}/oda_data/__init__.py

@@ -3,7 +3,7 @@ from importlib.metadata import version
 __version__ = version("oda_data")
 
 from oda_data import tools
-from oda_reader import set_cache_dir
+from oda_reader import set_cache_dir, disable_cache, enable_cache, clear_cache
 from oda_data.api.oecd import OECDClient
 from oda_data.api.sources import DAC1Data, DAC2AData, MultiSystemData, CRSData
 from oda_data.indicators.research.policy_markers import bilateral_policy_marker
@@ -20,7 +20,7 @@ def set_data_path(path):
     from pydeflate import set_pydeflate_path
 
     """Set the path to the data folder."""
-    global OdaPATHS
+    global ODAPaths
 
     ODAPaths.raw_data = Path(path).resolve()
     set_pydeflate_path(ODAPaths.raw_data)
@@ -43,4 +43,7 @@ __all__ = [
     "add_names_columns",
     "add_gni_share_column",
     "set_cache_dir",
+    "disable_cache",
+    "enable_cache",
+    "clear_cache",
 ]

{oda_data-2.0.0 → oda_data-2.0.2}/oda_data/api/oecd.py

@@ -13,7 +13,7 @@ from oda_data.api.constants import (
     PRICES,
     _EXCLUDE,
 )
-from oda_data.api.representations import _OdaDict, _OdaList
+from oda_data.api.representations import _ODADict, _ODAList
 from oda_data.api.sources import DAC1Data, DAC2AData, CRSData, MultiSystemData
 from oda_data.clean_data import common as clean
 from oda_data.clean_data.validation import validate_currency, validate_measure
@@ -228,18 +228,18 @@ class OECDClient:
     def available_providers(cls) -> dict:
         """Returns a dictionary of available donor codes and their names"""
         logger.info("Note that not all providers may be available for all indicators")
-        return _OdaDict(provider_groupings()["all_official"])
+        return _ODADict(provider_groupings()["all_official"])
 
     @classmethod
     def available_recipients(cls) -> dict:
         """Returns a dictionary of available recipient codes"""
         logger.info("Note that not all recipients may be available for all indicators")
-        return _OdaDict(recipient_groupings()["all_developing_countries_regions"])
+        return _ODADict(recipient_groupings()["all_developing_countries_regions"])
 
     @classmethod
     def available_currencies(cls) -> list:
         """Returns a dictionary of available currencies"""
-        return _OdaList(CURRENCIES)
+        return _ODAList(CURRENCIES)
 
     @classmethod
     def available_indicators(cls) -> dict:
@@ -250,7 +250,7 @@ class OECDClient:
                 n: v for n, v in i.items() if n in ["name", "description", "sources"]
             }
 
-        return _OdaDict(indicators)
+        return _ODADict(indicators)
 
     @classmethod
     def export_available_indicators(cls, export_folder: str | Path) -> None:

{oda_data-2.0.0 → oda_data-2.0.2}/oda_data/api/representations.py

@@ -1,10 +1,10 @@
-class _OdaList(list):
+class _ODAList(list):
     def __repr__(self):
         # Return the elements with a line break between them
         return "[\n" + ",\n".join(str(x) for x in self) + "\n]"
 
 
-class _OdaDict(dict):
+class _ODADict(dict):
     def __repr__(self):
         # Return the elements with a line break between them
         return "{\n" + ",\n".join(f"{k}: {v}" for k, v in self.items()) + "\n}"

{oda_data-2.0.0 → oda_data-2.0.2}/oda_data/api/sources.py

@@ -2,6 +2,7 @@ from abc import abstractmethod
 from typing import Optional
 
 import pandas as pd
+from cachetools import TTLCache
 from oda_reader import (
     download_crs,
     bulk_download_crs,
@@ -10,6 +11,7 @@ from oda_reader import (
     bulk_download_multisystem,
     download_multisystem,
 )
+from oda_reader._cache import memory
 
 from oda_data import config
 from oda_data.clean_data.common import (
@@ -23,6 +25,7 @@ from oda_data.clean_data.validation import (
     check_strings,
 )
 from oda_data.logger import logger
+from oda_data.tools.cache import OnDiskCache, generate_param_hash
 
 
 def _filters_to_query(filters: list[tuple[str, str, list]]) -> str:
@@ -45,13 +48,15 @@ class DACSource:
     of data in the form of a pandas DataFrame.
     """
 
+    memory_cache = TTLCache(maxsize=20, ttl=6000)
+    disk_cache = OnDiskCache(config.ODAPaths.raw_data, ttl_seconds=86400)
+
     def __init__(self):
         self.de_providers = None
         self.de_recipients = None
         self.de_indicators = None
         self.filters = None
         self.de_sectors = None
-        self.refresh_data = False
 
     def _init_filters(
         self,
@@ -168,25 +173,48 @@ class DACSource:
                     f"Missing values in {column}: {', '.join(map(str, missing_values))}"
                 )
 
-    def _check_file_is_available(
-        self,
-        dataset: str,
-        using_bulk_download: bool = False,
-        filters: list[tuple] = None,
-    ) -> None:
-        """Checks if a dataset file is available locally, triggering a download if not.
+    def _get_read_filters(
+        self, additional_filters: Optional[list[tuple]] = None
+    ) -> list[tuple[str, str, list]] | None:
+        filters = (
+            self.filters + additional_filters if additional_filters else self.filters
+        )
+        return filters or None
 
-        Args:
-            dataset (str): The dataset name.
-            using_bulk_download (bool, optional): Whether to check for a filtered version. Defaults to False.
-        """
+    def _get_cached_dataset(
+        self, param_hash: str, filters: list[tuple], columns: list[str]
+    ) -> pd.DataFrame | None:
 
-        prefix = "full" if using_bulk_download else "filtered"
+        cached_df = self.memory_cache.get(param_hash)
 
-        if not (config.ODAPaths.raw_data / f"{prefix}{dataset}.parquet").exists():
-            logger.info(f"{dataset} data not found. Downloading...")
-            df = self.download(bulk=using_bulk_download).pipe(clean_raw_df)
-            df.to_parquet(config.ODAPaths.raw_data / f"{prefix}{dataset}.parquet")
+        if cached_df is None:
+            cached_df = self.disk_cache.load(
+                self.__class__.__name__, param_hash, filters, columns
+            )
+
+        return cached_df
+
+    def _cache_dataset(self, param_hash: str, df: pd.DataFrame) -> None:
+        """Caches the dataset in memory and on disk."""
+        self.memory_cache[param_hash] = df.copy(deep=True)
+        self.disk_cache.save(self.__class__.__name__, param_hash, df)
+
+    def _apply_query_and_cache(
+        self, df: pd.DataFrame, query: str | None, param_hash: str, bulk: bool
+    ):
+        if bulk:
+            self._cache_dataset(param_hash=param_hash, df=df)
+
+        if query:
+            df = df.query(query)
+
+        if not bulk:
+            self._cache_dataset(param_hash=param_hash, df=df)
+
+        return df
+
+    def _bulk_hash(self) -> str:
+        return generate_param_hash(filters=[("bulk", "==", self.__class__.__name__)])
 
     def read(
         self,
@@ -205,21 +233,35 @@ class DACSource:
             pd.DataFrame: The loaded dataset.
         """
         # Create filters
-        filters = (
-            self.filters + additional_filters
-            if additional_filters is not None
-            else self.filters
+        filters = self._get_read_filters(additional_filters=additional_filters)
+
+        # Generate a query string from the filters
+        query = _filters_to_query(filters) if filters else None
+
+        # Generate a hash string with the filters and the type of file
+        param_hash = (
+            self._bulk_hash()
+            if using_bulk_download
+            else generate_param_hash(filters=filters)
         )
 
-        if len(filters) == 0:
-            filters = None
+        # If caching is not active, clear the cache
+        if not memory().store_backend:
+            self.disk_cache.cleanup(hash_str=param_hash, force=True)
+            self.memory_cache.clear()
 
-        query = _filters_to_query(filters) if filters else None
+        # Check if the data is already cached in memory or on disk
+        df = self._get_cached_dataset(
+            param_hash=param_hash, filters=filters, columns=columns
+        )
 
-        df = self.download(bulk=using_bulk_download).pipe(clean_raw_df)
+        # If not cached, download the data
+        if df is None:
+            df = self.download(bulk=using_bulk_download).pipe(clean_raw_df)
+            df = self._apply_query_and_cache(
+                df=df, query=query, param_hash=param_hash, bulk=using_bulk_download
+            )
 
-        if query:
-            df = df.query(query)
         if columns:
             df = df.filter(columns)
 

{oda_data-2.0.0 → oda_data-2.0.2}/oda_data/indicators/dac1/dac1_functions.py

@@ -23,6 +23,37 @@ def official_oda(data: pd.DataFrame) -> pd.DataFrame:
     return data.query(oda_query)
 
 
+def official_oda_gni(data: pd.DataFrame) -> pd.DataFrame:
+    """Filters the dataset to include only Official Development Assistance (ODA)
+    and computes the ratio of ODA to Gross National Income (GNI).
+
+    Args:
+        data: A pandas DataFrame containing aid data.
+
+    Returns:
+        A filtered pandas DataFrame with only official ODA rows.
+    """
+    oda = official_oda(data)
+
+    gni = (
+        data.query("aidtype_code == 1")
+        .rename(columns={ODASchema.VALUE: "gni_value"})
+        .filter([ODASchema.YEAR, ODASchema.PROVIDER_CODE, "gni_value"])
+    )
+
+    df = oda.merge(gni, on=[ODASchema.YEAR, ODASchema.PROVIDER_CODE], how="left")
+    df[ODASchema.VALUE] = round(df[ODASchema.VALUE] / df["gni_value"], 6)
+
+    df = df.drop(columns=["gni_value"])
+
+    df[ODASchema.AIDTYPE_CODE] = 1010110101
+    df[ODASchema.AIDTYPE_NAME] = (
+        "Official Development Assistance, as a share of GNI (official definition)"
+    )
+
+    return df
+
+
 def one_core_oda(data: pd.DataFrame) -> pd.DataFrame:
     """Computes ONE's Core ODA metric: ODA excluding in-donor spending
     but including administrative costs.

{oda_data-2.0.0 → oda_data-2.0.2}/oda_data/indicators/research/eu.py

@@ -115,28 +115,28 @@ def get_eui_oda_weights(
 
 
 def get_eui_plus_bilateral_providers_indicator(
-    indicators_obj: OECDClient, indicator: str | list[str]
+    client_obj: OECDClient, indicator: str | list[str]
 ) -> pd.DataFrame:
     """Fetches indicator values with adjusted EU institution contributions.
 
     Args:
-        indicators_obj: An `OECDClient` instance to fetch indicator data.
+        client_obj: An `OECDClient` instance to fetch indicator data.
         indicator: Indicator code or list of codes.
 
     Returns:
         A DataFrame containing indicator values with adjusted EU contributions.
     """
     eui_weights = get_eui_oda_weights(
-        years=indicators_obj.years,
-        providers=indicators_obj.providers,
-        measure=indicators_obj.measure[0],
-        use_bulk_download=indicators_obj.use_bulk_download,
+        years=client_obj.years,
+        providers=client_obj.providers,
+        measure=client_obj.measure[0],
+        use_bulk_download=client_obj.use_bulk_download,
     )
 
-    if 918 not in indicators_obj.providers:
-        indicators_obj.providers.append(918)
+    if 918 not in client_obj.providers:
+        client_obj.providers.append(918)
 
-    data = indicators_obj.get_indicators(indicator)
+    data = client_obj.get_indicators(indicator)
     eui_mask = data[ODASchema.PROVIDER_CODE] == 918
 
     data.loc[eui_mask, ODASchema.VALUE] = data.loc[

{oda_data-2.0.0 → oda_data-2.0.2}/oda_data/tools/cache.py

@@ -37,7 +37,7 @@ class OnDiskCache:
 
     def get_file_path(self, dataset_name: str, param_hash: str) -> Path:
         """Returns the on-disk path for the given dataset and parameter hash."""
-        return self.base_dir / f"filtered{dataset_name}-{param_hash}.parquet"
+        return self.base_dir / f"{dataset_name}-{param_hash}.parquet"
 
     def load(
         self,
@@ -65,9 +65,9 @@ class OnDiskCache:
 
     def cleanup(self, hash_str: Optional[str] = None, force: bool = False):
         """Removes expired cache files."""
-        pattern = f"filtered*{hash_str}*.parquet" if hash_str else "filtered*.parquet"
+        pattern = f"*{hash_str}*.parquet"
 
         for file in self.base_dir.glob(pattern):
             if self._is_expired(file) or force:
                 file.unlink(missing_ok=True)
-                logger.info(f"Deleted expired cache file: {file}")
+                logger.info(f"Deleted cache file: {file}")

{oda_data-2.0.0 → oda_data-2.0.2}/oda_data/tools/gni.py

@@ -36,22 +36,22 @@ def _get_eu27_gni_as_eu_institutions(gni_obj: OECDClient) -> pd.DataFrame:
     )
 
 
-def _get_gni_data(indicators_obj: OECDClient) -> pd.DataFrame:
+def _get_gni_data(client_obj: OECDClient) -> pd.DataFrame:
     """Fetch GNI data and optionally include EU Institutions' aggregate.
 
     Args:
-        indicators_obj: An OECDClient object for the main data context.
+        client_obj: An OECDClient object for the main data context.
 
     Returns:
         A DataFrame with GNI values per provider and year.
     """
     # Create a copy of the indicators object and set the measure to "net_disbursement"
-    gni_obj = copy(indicators_obj)
+    gni_obj = copy(client_obj)
     gni_obj.measure = ["net_disbursement"]
 
     # Fetch GNI data for the specified indicators
     gni_df = gni_obj.get_indicators("DAC1.40.1")
-    providers = indicators_obj.providers or []
+    providers = client_obj.providers or []
 
     # Check if EU Institutions (provider code 918) is in the list of providers
     # If it is, aggregate GNI for EU27 countries and assign it to provider 918

{oda_data-2.0.0 → oda_data-2.0.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "oda_data"
-version = "2.0.0"
+version = "2.0.2"
 description = "A python package to work with Official Development Assistance data from the OECD DAC."
 readme = "README.md"
 authors = [{ name = "Jorge Rivera", email = "jorge.rivera@one.org" }, { name = "Miguel Haro Ruiz", email = "miguel.haroruiz@one.org" }]