pydeflate 1.4.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

pydeflate/get_data/oecd_data.py

@@ -1,146 +0,0 @@
-import warnings
-from dataclasses import dataclass
-
-import pandas as pd
-from oda_reader import download_dac1
-
-from pydeflate.get_data.deflate_data import Data
-from pydeflate.get_data.exchange_data import ExchangeOECD
-from pydeflate.pydeflate_config import PYDEFLATE_PATHS, logger
-from pydeflate.tools.update_data import update_update_date
-from pydeflate.utils import oecd_codes
-
-warnings.simplefilter("ignore", Warning, lineno=1013)
-
-
-def _compute_deflators_and_exchange(data: pd.DataFrame) -> pd.DataFrame:
-    return data.assign(
-        exchange=lambda d: round(d.N / d.A, 5),
-        deflator=lambda d: round(100 * d.A / d.D, 6),  # implied deflator
-        iso_code=lambda d: d.donor_code.map(oecd_codes()),
-        year=lambda d: pd.to_datetime(d.year, format="%Y"),
-    ).assign(exchange=lambda d: d.exchange.fillna(1))
-
-
-def _clean_dac1(df: pd.DataFrame) -> pd.DataFrame:
-    """Clean DAC1 to keep only relevant information for deflators and exchange.
-
-    Args:
-        df: the dataframe to clean
-
-    Returns:
-        A cleaned dataframe
-    """
-
-    # Columns to keep and rename
-    cols = {"amounttype_code": "type", "aidtype_code": "aid", "flows_code": "flow"}
-
-    # Get only the official definition of the data
-    query = (
-        "(aid == 1010 & flow == 1140 & year <2018 ) | "
-        "(aid == 11010 & flow == 1160 & year >=2018)"
-    )
-
-    # Clean the data
-    data = (
-        df.rename(columns=cols)
-        .query(query)
-        .filter(["donor_code", "type", "year", "value"], axis=1)
-        .pivot(index=["donor_code", "year"], columns=["type"], values="value")
-        .reset_index()
-    )
-
-    data = (
-        data.pipe(_compute_deflators_and_exchange)
-        .dropna(subset=["iso_code"])
-        .filter(["iso_code", "year", "exchange", "deflator"], axis=1)
-        .reset_index(drop=True)
-    )
-
-    return data
-
-
-def update_dac1() -> None:
-    """Update dac1 data from OECD site and save as feather"""
-
-    # Use oda_reader to get the data
-    df = download_dac1(
-        filters={"measure": ["1010", "11010"], "flow_type": ["1140", "1160"]}
-    )
-
-    # Clean the data
-    df = df.pipe(_clean_dac1)
-
-    # Save the data
-    df.to_feather(PYDEFLATE_PATHS.data / "pydeflate_dac1.feather")
-
-    # Update the update date
-    update_update_date("OECD DAC")
-
-
-def _identify_base_year(df: pd.DataFrame) -> int:
-    return (
-        df.query("iso_code in ['FRA','GBR','USA','CAN','DEU','EUI']")
-        .groupby(["year"], as_index=False)
-        .value.mean(numeric_only=True)
-        .round(2)
-        .loc[lambda d: d.value == 100.00]
-        .year.dt.year.item()
-    )
-
-
-def _calculate_price_deflator(deflators_df: pd.DataFrame) -> pd.DataFrame:
-    return deflators_df.assign(
-        value=lambda d: round(d.value_dac * d.value_exchange / 100, 6)
-    ).filter(["iso_code", "year", "indicator", "value"], axis=1)
-
-
-@dataclass
-class OECD(Data):
-    """An object to download and return the latest OECD DAC deflators data."""
-
-    def __post_init__(self):
-        self._available_methods = {"dac_deflator": "oecd_dac"}
-
-    def update(self, **kwargs) -> None:
-        update_dac1()
-
-    def load_data(self, **kwargs) -> None:
-        """Load the OECD DAC price deflators data.
-
-        If the data is not found, it will be downloaded.
-        DAC deflators are transformed into price deflators by using the
-        implied exchange rate information from the OECD DAC data.
-
-        The deflators that are loaded is therefore *not* the DAC deflator,
-        but the price deflator used to produce the DAC deflators.
-
-        """
-        try:
-            d_ = pd.read_feather(PYDEFLATE_PATHS.data / "pydeflate_dac1.feather")
-        except FileNotFoundError:
-            logger.info("Data not found, downloading...")
-            self.update()
-            self.load_data()
-            return
-
-        d_ = d_.assign(indicator="oecd_dac").rename(columns={"deflator": "value"})
-
-        # Identify base year
-        base_year = _identify_base_year(d_)
-
-        # Load exchange deflators
-        exchange_deflator = ExchangeOECD().exchange_deflator(
-            source_iso="USA", target_iso="USA", base_year=base_year
-        )
-
-        # Merge deflators and exchange deflators
-        deflators_df = d_.merge(
-            exchange_deflator,
-            on=["iso_code", "year"],
-            how="left",
-            suffixes=("_dac", "_exchange"),
-        )
-
-        # Calculate the price deflator
-        self._data = _calculate_price_deflator(deflators_df=deflators_df)

pydeflate/get_data/wb_data.py

@@ -1,75 +0,0 @@
-from dataclasses import dataclass
-
-import pandas as pd
-from pydeflate.get_data.deflate_data import Data
-from pydeflate.pydeflate_config import PYDEFLATE_PATHS
-
-from bblocks import WorldBankData, set_bblocks_data_path
-
-from pydeflate.tools.update_data import update_update_date
-
-set_bblocks_data_path(PYDEFLATE_PATHS.data)
-
-_INDICATORS: dict = {
-    "gdp": "NY.GDP.DEFL.ZS",
-    "gdp_linked": "NY.GDP.DEFL.ZS.AD",
-    "cpi": "FP.CPI.TOTL",
-    "exchange": "PA.NUS.FCRF",
-    "effective_exchange": "PX.REX.REER",
-}
-
-START: int = 1950
-END: int = 2025
-
-
-def update_world_bank_data() -> None:
-    """Update World Bank data."""
-    wb = WorldBankData()
-    wb.load_data(indicator=list(_INDICATORS.values()), start_year=START, end_year=END)
-    wb.update_data()
-    update_update_date(source="World Bank")
-
-
-@dataclass
-class WorldBank(Data):
-    """An object to download and return the latest WorldBank exchange and price data"""
-
-    def __post_init__(self):
-        self._available_methods = {
-            "gdp": "GDP deflator",
-            "gdp_linked": "GDP deflator linked",
-            "cpi": "Consumer price index",
-        }
-
-    def update(self, **kwargs) -> None:
-        """Update data for all WorldBank indicators"""
-        update_world_bank_data()
-
-    def load_data(self) -> None:
-        """Load data for all WorldBank indicators"""
-
-        # get the paths to the data
-        paths = [
-            PYDEFLATE_PATHS.data / f"{_INDICATORS[i_]}_{START}-{END}_.csv"
-            for i_ in _INDICATORS
-        ]
-
-        # check if data exists, if not update
-        for path in paths:
-            if not path.exists():
-                update_world_bank_data()
-                break
-
-        # load the data
-        files = []
-        for path in paths:
-            try:
-                files.append(
-                    pd.read_csv(path, parse_dates=["date"]).rename(
-                        columns={"date": "year"}
-                    )
-                )
-            except FileNotFoundError:
-                files.append(pd.DataFrame())
-
-        self._data = pd.concat(files, ignore_index=True)

pydeflate/tools/__init__.py

@@ -1,2 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-

pydeflate/tools/exchange.py

@@ -1,171 +0,0 @@
-import pandas as pd
-
-from pydeflate.get_data.exchange_data import (
-    ExchangeIMF,
-    ExchangeOECD,
-    ExchangeWorldBank,
-)
-from pydeflate.utils import check_year_as_number, to_iso3, oecd_codes
-
-
-_exchange_source = {
-    "world_bank": ExchangeWorldBank,
-    "wb": ExchangeWorldBank,
-    "oecd_dac": ExchangeOECD,
-    "imf": ExchangeIMF,
-}
-
-
-def _check_key_errors(
-    rates_source: str,
-    columns: str | list | pd.Index,
-    value_column: str,
-    date_column: str,
-) -> None:
-    """Check whether provided parameters are valid"""
-
-    if rates_source not in _exchange_source.keys():
-        raise KeyError(
-            f"{rates_source=} is not a valid exchange rates source. "
-            f"Please choose from {_exchange_source.keys()}"
-        )
-
-    if value_column not in columns:
-        raise KeyError(
-            f"{value_column} is not a valid column in the provided DataFrame"
-        )
-
-    if date_column not in columns:
-        raise KeyError(f"{date_column} is not a valid column in the provided DataFrame")
-
-
-def exchange(
-    df: pd.DataFrame,
-    source_currency: str,
-    target_currency: str,
-    rates_source: str = "world_bank",
-    id_column: str = "iso_code",
-    id_type: str = "ISO3",
-    value_column: str = "value",
-    target_column: str = "value",
-    date_column: str = "date",
-) -> pd.DataFrame:
-    """
-
-
-    Parameters
-    ----------
-    df : pd.DataFrame
-        A Pandas DataFrame, in long format, containing at least a date column,
-        a column with iso-3 codes to identify the source currency, and a
-        value column where the values to be converted are stored.
-    source_currency : str
-        The ISO-3 code of the country which owns the currency in which the data
-        is expressed. "LCU" can be used to indicate that data is in Local
-        Currency Unit. "emu" can be used for the EURO.
-    target_currency : str
-        The ISO-3 code of the country which owns the currency to which the data
-        will be converted. "LCU" can be used to convert from a given currency
-        (like the USD), back to each country's Local Currency.
-    rates_source : str, optional
-        The source of the exchange rate data. Current options include "wb" for
-        the World Bank and "oecd_dac" for the exchange rates used for ODA
-        statistics. The default is "wb".
-    id_column : str, optional
-        The name of the column containing the codes or names used to identify countries.
-        The default is "iso_code".
-    id_type : str, optional
-        The types of codes used to identify countries. Should match options in
-        Country Converter or the DAC codes.The default is "ISO3".
-    value_column : str, optional
-        The name of the column containing the values to be converted.
-        The default is "value".
-    target_column : str, optional
-        The name of the column where the converted values will be stored.
-        The default is "value_xe".
-    date_column : str, optional
-        The name of the column where the date/year is stored.
-        The default is "date".
-
-    Returns
-    -------
-    df : pd.DataFrame
-        Returns a dataframe containing the converted data stored in the
-        target column.
-
-    """
-
-    # create a copy of the dataframe to avoid modifying the original
-    df = df.copy(deep=True)
-
-    # Check whether provided parameters are valid
-    _check_key_errors(rates_source, df.columns, value_column, date_column)
-
-    # If source currency matches target currency, do nothing
-    if source_currency == target_currency:
-        df[target_column] = df[value_column]
-        return df
-
-    # keep track of original columns. This is so that the same order and columns can be
-    # preserved.
-    if target_column not in df.columns:
-        cols = [*df.columns, target_column]
-    else:
-        cols = df.columns
-
-    # check whether date is provided as integer
-    df, year_as_number = check_year_as_number(df, date_column)
-
-    # check whether target currency is LCU
-    if target_currency == "LCU":
-        target_currency = source_currency
-        source_currency = "LCU"
-        target_changed = True
-    else:
-        target_changed = False
-
-    # get the selected rates function
-    exchange_rates = (
-        _exchange_source[rates_source]()
-        .exchange_rate(target_currency)
-        .rename(columns={"year": date_column, "value": value_column, "iso_code": "id_"})
-    )
-
-    # Create ID col.
-    if id_type == "DAC":
-        df["id_"] = df[id_column].map(oecd_codes()).fillna("DAC")
-    else:
-        df = df.pipe(
-            to_iso3, codes_col=id_column, target_col="id_", src_classification=id_type
-        )
-
-    # merge exchange rates with data
-    if source_currency == "LCU":
-        df = df.merge(
-            exchange_rates,
-            on=["id_", date_column],
-            suffixes=("", "_xe"),
-        )
-
-    else:
-        xe = exchange_rates.loc[exchange_rates.id_ == source_currency]
-        df = df.merge(
-            xe.drop("id_", axis=1),
-            on=[date_column],
-            suffixes=("", "_xe"),
-        )
-
-    # revert change to target_currency if target_changed
-    if target_changed:
-        target_currency = "LCU"
-
-    if target_currency == "LCU":
-        df[target_column] = df[value_column] * df[f"{value_column}_xe"]
-
-    else:
-        df[target_column] = df[value_column] / df[f"{value_column}_xe"]
-
-    if year_as_number:
-        df[date_column] = df[date_column].dt.year
-
-    return df.filter(cols, axis=1)

pydeflate/tools/update_data.py

@@ -1,69 +0,0 @@
-import datetime
-import json
-
-from pydeflate.pydeflate_config import PYDEFLATE_PATHS, logger
-
-
-def _diff_from_today(date: datetime.datetime):
-    """Compare to today"""
-
-    today = datetime.datetime.today()
-
-    return (today - date).days
-
-
-def warn_updates():
-    if not (PYDEFLATE_PATHS.data / "data_updates.json").exists():
-        return
-
-    with open(PYDEFLATE_PATHS.data / "data_updates.json") as file:
-        updates = json.load(file)
-
-    for source, date in updates.items():
-        d = datetime.datetime.strptime(date, "%Y-%m-%d")
-        if _diff_from_today(d) > 50:
-            message = (
-                f'\n\nThe underlying data for "{source}" has not been updated'
-                f" in over {_diff_from_today(d)} days. \nIn order to use"
-                " pydeflate with the most recent data, please run:\n"
-                "`pydeflate.update_all_data()`"
-            )
-            logger.warning(message)
-
-
-def update_update_date(source: str):
-    """Update the most recent update date for data to today"""
-
-    today = datetime.datetime.today().strftime("%Y-%m-%d")
-
-    # Check to see if specified path contains an update file. Create one if not
-    if not (PYDEFLATE_PATHS.data / "data_updates.json").exists():
-        updates = {}
-        with open(PYDEFLATE_PATHS.data / "data_updates.json", "w") as outfile:
-            json.dump(updates, outfile)
-
-    with open(PYDEFLATE_PATHS.data / "data_updates.json") as file:
-        updates = json.load(file)
-
-    updates[source] = today
-
-    with open(PYDEFLATE_PATHS.data / "data_updates.json", "w") as outfile:
-        json.dump(updates, outfile)
-
-
-def update_all_data() -> None:
-    """Run to update all underlying data."""
-
-    from pydeflate.get_data.imf_data import IMF
-    from pydeflate.get_data.oecd_data import OECD
-    from pydeflate.get_data.wb_data import WorldBank
-
-    data = {
-        "IMF WEO Data": IMF().update,
-        "OECD DAC data": OECD().update,
-        "WorldBank data": WorldBank().update,
-    }
-
-    for source, func in data.items():
-        func()
-        logger.info(f"****Successfully updated {source}****\n")