pydeflate 1.4.2__py3-none-any.whl → 2.0.0__py3-none-any.whl

pydeflate/exchange/exchangers.py

@@ -0,0 +1,147 @@
+from functools import wraps
+
+import pandas as pd
+
+from pydeflate.core.api import BaseExchange
+from pydeflate.core.source import DAC, WorldBank, IMF
+
+
+def _generate_docstring(source_name: str) -> str:
+    """Generate docstring for each decorated exchange function."""
+    return (
+        f"Exchange a DataFrame using the {source_name} rates source.\n\n"
+        f"This function applies exchange rates toa  DataFrame using the {source_name} rates.\n\n"
+        "Args:\n"
+        "    data (pd.DataFrame): The input DataFrame containing data to deflate.\n"
+        "    source_currency (str, optional): The source currency code. Defaults to 'USA'.\n"
+        "    target_currency (str, optional): The target currency code. Defaults to 'USA'.\n"
+        "    id_column (str, optional): Column with entity identifiers. Defaults to 'iso_code'.\n"
+        "    year_column (str, optional): Column with year information. Defaults to 'year'.\n"
+        "    use_source_codes (bool, optional): Use source-specific entity codes. Defaults to False.\n"
+        "    value_column (str, optional): Column with values to deflate. Defaults to 'value'.\n"
+        "    target_value_column (str, optional): Column to store deflated values. Defaults to 'value'.\n"
+        "    reversed_ (bool, optional): The reverse of an exchange conversion. Defaults to False.\n"
+        "    year_format (str | None, optional): Format of the year in `year_column`. Defaults to None.\n"
+        "    update_rates (bool, optional): Update the exchange rate data. Defaults to False.\n\n"
+        "Returns:\n"
+        "    pd.DataFrame: DataFrame with converted values in the `target_value_column`.\n"
+    )
+
+
+def _exchange(exchange_source_cls):
+    """Decorator to create exchange wrappers with specific source"""
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(
+            data: pd.DataFrame,
+            *,
+            source_currency: str = "USA",
+            target_currency: str = "USA",
+            id_column: str = "iso_code",
+            year_column: str = "year",
+            use_source_codes: bool = False,
+            value_column: str = "value",
+            target_value_column: str = "value",
+            reversed_: bool = False,
+            year_format: str | None = None,
+            update_rates: bool = False,
+        ):
+            # Validate input parameters
+            if not isinstance(data, pd.DataFrame):
+                raise ValueError("The 'data' parameter must be a pandas DataFrame.")
+
+            if id_column not in data.columns:
+                raise ValueError(
+                    f"The id_column '{id_column}' is not in the DataFrame."
+                )
+            if year_column not in data.columns:
+                raise ValueError(
+                    f"The year_column '{year_column}' is not in the DataFrame."
+                )
+            if value_column not in data.columns:
+                raise ValueError(
+                    f"The value_column '{value_column}' is not in the DataFrame."
+                )
+
+            # Copy the data to avoid modifying the original
+            to_exchange = data.copy()
+
+            # Initialize the deflator source
+            source = exchange_source_cls(update=update_rates)
+
+            # Create a deflator object
+            exchange = BaseExchange(
+                exchange_source=source,
+                source_currency=source_currency,
+                target_currency=target_currency,
+                use_source_codes=use_source_codes,
+            )
+
+            # Deflate the data
+            return exchange.exchange(
+                data=to_exchange,
+                entity_column=id_column,
+                year_column=year_column,
+                value_column=value_column,
+                target_value_column=target_value_column,
+                year_format=year_format,
+                reversed_=reversed_,
+            )
+
+        # Add the deflator source and price kind to the function
+        wrapper.__doc__ = _generate_docstring(exchange_source_cls.__name__)
+        return wrapper
+
+    return decorator
+
+
+@_exchange(DAC)
+def oecd_dac_exchange(
+    data: pd.DataFrame,
+    *,
+    source_currency: str = "USA",
+    target_currency: str = "USA",
+    id_column: str = "iso_code",
+    year_column: str = "year",
+    use_source_codes: bool = False,
+    value_column: str = "value",
+    target_value_column: str = "value",
+    reversed_: bool = False,
+    year_format: str | None = None,
+    update_rates: bool = False,
+) -> pd.DataFrame: ...
+
+
+@_exchange(WorldBank)
+def wb_exchange(
+    data: pd.DataFrame,
+    *,
+    source_currency: str = "USA",
+    target_currency: str = "USA",
+    id_column: str = "iso_code",
+    year_column: str = "year",
+    use_source_codes: bool = False,
+    value_column: str = "value",
+    target_value_column: str = "value",
+    reversed_: bool = False,
+    year_format: str | None = None,
+    update_rates: bool = False,
+) -> pd.DataFrame: ...
+
+
+@_exchange(IMF)
+def imf_exchange(
+    data: pd.DataFrame,
+    *,
+    source_currency: str = "USA",
+    target_currency: str = "USA",
+    id_column: str = "iso_code",
+    year_column: str = "year",
+    use_source_codes: bool = False,
+    value_column: str = "value",
+    target_value_column: str = "value",
+    reversed_: bool = False,
+    year_format: str | None = None,
+    update_rates: bool = False,
+) -> pd.DataFrame: ...

pydeflate/pydeflate_config.py

@@ -1,5 +1,5 @@
-from pathlib import Path
 import logging
+from pathlib import Path
 
 
 class PYDEFLATE_PATHS:
@@ -11,24 +11,33 @@ class PYDEFLATE_PATHS:
     test_data = package / "tests" / "test_files"
 
 
-# Create a root logger
-logger = logging.getLogger(__name__)
+def setup_logger(name) -> logging.Logger:
+    """Set up the logger.
+
+    Args:
+        name (str): The name of the logger.
+
+    Returns:
+        logging.Logger: The logger.
+
+    """
+    logger_ = logging.getLogger(name)
+    logger_.setLevel(logging.INFO)
 
-# Create two handlers (terminal and file)
-shell_handler = logging.StreamHandler()
+    # Only add handlers if the logger has none to avoid duplication
+    if not logger_.hasHandlers():
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(logging.INFO)
 
-# Set levels for the logger, shell and file
-logger.setLevel(logging.DEBUG)
-shell_handler.setLevel(logging.DEBUG)
+        formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s:\n %(message)s"
+        )
+        console_handler.setFormatter(formatter)
 
-# Format the outputs
-fmt_shell = "%(levelname)s [%(filename)s: %(funcName)s:] %(message)s"
+        logger_.addHandler(console_handler)
+        logger_.propagate = False
 
-# Create formatters
-shell_formatter = logging.Formatter(fmt_shell)
+    return logger_
 
-# Add formatters to handlers
-shell_handler.setFormatter(shell_formatter)
 
-# Add handlers to the logger
-logger.addHandler(shell_handler)
+logger = setup_logger("pydeflate")

pydeflate/sources/common.py

@@ -0,0 +1,278 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Literal
+
+import pandas as pd
+from hdx.location.country import Country
+
+from pydeflate.pydeflate_config import PYDEFLATE_PATHS, logger
+
+AvailableDeflators = Literal["NGDP_D", "NGDP_DL", "CPI", "PCPI", "PCPIE"]
+
+
+def check_file_age(file: Path) -> int:
+    """Check the age of a WEO file in days.
+
+    Args:
+        file (Path): The WEO parquet file to check.
+
+    Returns:
+        int: The number of days since the file was created.
+    """
+    current_date = datetime.today()
+    # Extract date from the filename (format: weo_YYYY-MM-DD.parquet)
+    file_date = datetime.strptime(file.stem.split("_")[1], "%Y-%m-%d")
+
+    # Return the difference in days between today and the file's date
+    return (current_date - file_date).days
+
+
+def enforce_pyarrow_types(df: pd.DataFrame) -> pd.DataFrame:
+    """Ensures that a DataFrame uses pyarrow dtypes."""
+    return df.convert_dtypes(dtype_backend="pyarrow")
+
+
+def today() -> str:
+    from datetime import datetime
+
+    return datetime.today().strftime("%Y-%m-%d")
+
+
+def _match_regex_to_iso3(
+    to_match: list[str], additional_mapping: dict | None
+) -> dict[str, str]:
+    """Match a list of regex strings to ISO3 country codes.
+
+    Args:
+        to_match (list): A list of regex strings to match.
+
+    Returns:
+        dict: A dictionary with the regex strings as keys and the ISO3 codes as values.
+    """
+    if additional_mapping is None:
+        additional_mapping = {}
+
+    # Create a Country object
+    country = Country()
+
+    # Match the regex strings to ISO3 country codes
+    matches = {}
+
+    for match in to_match:
+        match_ = country.get_iso3_country_code_fuzzy(match)[0]
+        matches[match] = match_
+        if match_ is None and match not in additional_mapping:
+            logger.debug(f"No ISO3 match found for {match}")
+
+    return matches | additional_mapping
+
+
+def convert_id(
+    series: pd.Series,
+    from_type: str = "regex",
+    to_type: str = "ISO3",
+    not_found: Any = None,
+    *,
+    additional_mapping: dict = None,
+) -> pd.Series:
+    """Takes a Pandas' series with country IDs and converts them into the desired type.
+
+    Args:
+        series: the Pandas series to convert
+        from_type: the classification type according to which the series is encoded.
+            For example: ISO3, ISO2, regex, DACCode.
+        to_type: the target classification type. Same options as from_type
+        not_found: what to do if the value is not found. Can pass a string or None.
+            If None, the original value is passed through.
+        additional_mapping: Optionally, a dictionary with additional mappings can be used.
+            The keys are the values to be converted and the values are the converted values.
+            The keys follow the same datatype as the original values. The values must follow
+            the same datatype as the target type.
+    """
+
+    # if from and to are the same, return without changing anything
+    if from_type == to_type:
+        return series
+
+    mapping_functions = {"regex": _match_regex_to_iso3}
+
+    # Get the unique values for mapping. This is done in order to significantly improve
+    # the performance of country_converter with very long datasets.
+    s_unique = series.unique()
+
+    # Create a correspondence dictionary
+    mapping = mapping_functions[from_type](
+        to_match=s_unique, additional_mapping=additional_mapping
+    )
+
+    return series.map(mapping).fillna(series if not_found is None else not_found)
+
+
+def add_pydeflate_iso3(
+    df: pd.DataFrame, column: str, from_type: str = "regex", fillna: Any = pd.NA
+) -> pd.DataFrame:
+    """Add a column with ISO3 country codes to a dataframe.
+
+    Args:
+        df (pd.DataFrame): The dataframe to add the column to.
+        column (str): The column containing the country codes.
+        from_type (str): The classification type of the country codes.
+        fillna (Any): The value to use when the country code is not found.
+
+    Returns:
+        pd.DataFrame: The dataframe with the added ISO3 column.
+    """
+    # Convert the country codes to ISO3
+    df["pydeflate_iso3"] = convert_id(
+        df[column].fillna(""),
+        from_type=from_type,
+        to_type="ISO3",
+        not_found=fillna,
+        additional_mapping={
+            "World": "WLD",
+            "EU Institutions": "EUI",
+            "DAC countries": "DAC",
+            "Kosovo": "XXK",
+            "G7": "G7C",
+            "Sub-Sahara Africa": "SSA",
+        },
+    )
+
+    return df
+
+
+def prefix_pydeflate_to_columns(
+    df: pd.DataFrame, prefix: str = "pydeflate_"
+) -> pd.DataFrame:
+    """Add a prefix to all columns in a DataFrame.
+
+    Args:
+        df (pd.DataFrame): The DataFrame to add the prefix to.
+        prefix (str): The prefix to add to the column names.
+
+    Returns:
+        pd.DataFrame: The DataFrame with the prefixed column names.
+    """
+    df.columns = [
+        f"{prefix}{col}" if not col.startswith(prefix) else col for col in df.columns
+    ]
+
+    return df
+
+
+def identify_base_year(df: pd.DataFrame, measure: str, year: str = "year") -> int:
+    """Identify the base year for a given measure where the value is equal to 100.
+
+    Args:
+        df (pd.DataFrame): DataFrame containing the deflator data with 'year' and the given measure.
+        measure (str): The column name for the measure to find the base year for.
+        year (str): The column name for the year.
+
+    Returns:
+        int: The base year, or None if no base year is found.
+    """
+    # Find the year where the deflator measure is exactly 100 (or very close)
+    base_year = df.loc[df[measure].round(2) == 100, year]
+
+    # Return the year if found, otherwise return None
+    return base_year.iloc[0] if not base_year.empty else None
+
+
+def compute_exchange_deflator(
+    df: pd.DataFrame,
+    base_year_measure: str | None = None,
+    exchange: str = "EXCHANGE",
+    year: str = "year",
+    grouper: list[str] = None,
+) -> pd.DataFrame:
+    """Compute the exchange rate deflator for each group of entities.
+
+    This function calculates a deflator for the exchange rate by identifying a base year
+    where the base_year_measure is 100, then normalizing exchange values to that base year.
+
+    Args:
+        df (pd.DataFrame): Input DataFrame containing columns 'year', and 'EXCHANGE'.
+        base_year_measure (str): The column name for the measure to find the base year for.
+        exchange (str): The column name for the exchange rate.
+        year (str): The column name for the year.
+        grouper (list): List of columns to group by before applying the deflator.
+
+    Returns:
+        pd.DataFrame: DataFrame with an additional column for the exchange rate deflator.
+    """
+
+    def _add_deflator(
+        group: pd.DataFrame,
+        measure: str | None = "NGDPD_D",
+        exchange: str = "EXCHANGE",
+        year: str = "year",
+    ) -> pd.DataFrame:
+        # Identify the base year for the deflator
+        if measure is not None:
+            base_year = identify_base_year(group, measure=measure, year=year)
+        else:
+            base_year = group.dropna(subset=exchange)[year].max()
+
+        # If no base year is found, return the group unchanged
+        if base_year is None or pd.isna(base_year):
+            return group
+
+        # Extract the exchange rate value for the base year
+        base_value = group.loc[group[year] == base_year, exchange].values
+
+        # If base value is found and valid, calculate the deflator
+        if base_value.size > 0 and pd.notna(base_value[0]):
+            group[f"{exchange}_D"] = round(100 * group[exchange] / base_value[0], 6)
+
+        return group
+
+    if grouper is None:
+        grouper = ["entity", "entity_code"]
+
+    # Apply the deflator computation for each group of 'entity' and 'entity_code'
+    return df.groupby(grouper, group_keys=False).apply(
+        _add_deflator, measure=base_year_measure, exchange=exchange, year=year
+    )
+
+
+def read_data(
+    file_finder_func: callable,
+    download_func: callable,
+    data_name: str,
+    update: bool = False,
+) -> pd.DataFrame:
+    """Generic function to read data from parquet files or download fresh data.
+
+    Args:
+        file_finder_func (function): Function to find existing data files in the path.
+        download_func (function): Function to download fresh data if no files are
+        found or an update is needed.
+        data_name (str): Name of the dataset for logging purposes (e.g., "WEO", "DAC").
+        update (bool): If True, forces downloading of new data even if files exist.
+
+    Returns:
+        pd.DataFrame: The latest available data.
+    """
+    # Find existing files using the provided file finder function
+    files = file_finder_func(PYDEFLATE_PATHS.data)
+
+    # If no files are found or update is requested, download new data
+    if len(files) == 0 or update:
+        download_func()
+        files = file_finder_func(PYDEFLATE_PATHS.data)
+
+    # If files are found, sort them by age and load the most recent one
+    if len(files) > 0:
+        files = sorted(files, key=check_file_age)
+        latest_file = files[0]
+
+        # Check if the latest file is older than 120 days and log a warning
+        if check_file_age(latest_file) > 120:
+            logger.warn(
+                f"The latest {data_name} data is more than 120 days old.\n"
+                f"Consider updating by setting update=True in the function call."
+            )
+
+        # Read and return the latest parquet file as a DataFrame
+        logger.info(f"Reading {data_name} data from {latest_file}")
+        return pd.read_parquet(latest_file)

pydeflate/sources/dac.py

@@ -0,0 +1,137 @@
+from pathlib import Path
+
+import pandas as pd
+from oda_reader import download_dac1
+
+from pydeflate.pydeflate_config import PYDEFLATE_PATHS
+from pydeflate.sources.common import (
+    today,
+    add_pydeflate_iso3,
+    enforce_pyarrow_types,
+    compute_exchange_deflator,
+    read_data,
+    prefix_pydeflate_to_columns,
+)
+
+
+def _find_dac_files_in_path(path: Path) -> list:
+    """Find all DAC parquet files in the specified directory.
+
+    Args:
+        path (Path): The directory path to search for DAC parquet files.
+
+    Returns:
+        list: List of DAC parquet files found in the directory.
+    """
+    return list(path.glob("dac_*.parquet"))
+
+
+def _to_units(df: pd.DataFrame) -> pd.DataFrame:
+    """Convert DAC values (in million) to units.
+
+    Args:
+        df (pd.DataFrame): Dataframe with raw observation values.
+
+    Returns:
+        pd.DataFrame: Dataframe with scaled observation values.
+    """
+    df = df.copy()
+    df["value"] = df["value"] * df["unit_multiplier"]
+    return df
+
+
+def _keep_official_definition_only(df: pd.DataFrame) -> pd.DataFrame:
+    query = (
+        "(aidtype_code == 1010 & flows_code == 1140 & year <2018 ) | "
+        "(aidtype_code == 11010 & flows_code == 1160 & year >=2018)"
+    )
+
+    return df.query(query)
+
+
+def _keep_useful_columns(df: pd.DataFrame) -> pd.DataFrame:
+    columns = ["year", "donor_code", "donor_name", "EXCHANGE", "DAC_DEFLATOR"]
+
+    return df.filter(columns)
+
+
+def _pivot_amount_type(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.filter(["year", "donor_code", "donor_name", "amounttype_code", "value"])
+    return df.pivot(
+        index=[c for c in df.columns if c not in ["amounttype_code", "value"]],
+        columns="amounttype_code",
+        values="value",
+    ).reset_index()
+
+
+def _compute_exchange(df: pd.DataFrame) -> pd.DataFrame:
+    # The values for certain providers should be 1
+    df.loc[lambda d: d.donor_code >= 20000, "N"] = df.loc[
+        lambda d: d.donor_code >= 20000, "A"
+    ]
+    df["EXCHANGE"] = round(df["N"] / df["A"], 6).fillna(1)
+    return df
+
+
+def _compute_dac_deflator(df: pd.DataFrame) -> pd.DataFrame:
+    df["DAC_DEFLATOR"] = round(100 * df["A"] / df["D"], 6)
+    return df
+
+
+def _compute_dac_gdp_deflator(df: pd.DataFrame) -> pd.DataFrame:
+    df["NGDP_D"] = round(df["EXCHANGE_D"] / 100 * df["DAC_DEFLATOR"], 5)
+
+    return df
+
+
+def _rename_columns(df: pd.DataFrame) -> pd.DataFrame:
+    return df.rename(
+        columns={
+            "donor_code": "entity_code",
+            "donor_name": "entity",
+        }
+    )
+
+
+def download_dac():
+    # Use oda_reader to get the data
+    df = download_dac1(
+        filters={"measure": ["1010", "11010"], "flow_type": ["1140", "1160"]}
+    )
+
+    # Clean the data
+    df = (
+        df.pipe(_to_units)
+        .pipe(_keep_official_definition_only)
+        .pipe(_pivot_amount_type)
+        .pipe(_compute_exchange)
+        .pipe(_compute_dac_deflator)
+        .pipe(_keep_useful_columns)
+        .pipe(add_pydeflate_iso3, column="donor_name", from_type="regex")
+        .pipe(_rename_columns)
+        .pipe(compute_exchange_deflator, base_year_measure="DAC_DEFLATOR")
+        .pipe(_compute_dac_gdp_deflator)
+        .pipe(prefix_pydeflate_to_columns)
+        .pipe(enforce_pyarrow_types)
+        .reset_index(drop=True)
+    )
+
+    # Get today's date to use as a file suffix
+    suffix = today()
+
+    # Save the data
+    df.to_parquet(PYDEFLATE_PATHS.data / f"dac_{suffix}.parquet")
+
+
+def read_dac(update: bool = False) -> pd.DataFrame:
+    """Read the latest WEO data from parquet files or download fresh data."""
+    return read_data(
+        file_finder_func=_find_dac_files_in_path,
+        download_func=download_dac,
+        data_name="DAC",
+        update=update,
+    )
+
+
+if __name__ == "__main__":
+    df = read_dac(update=True)