pydeflate 1.4.2__py3-none-any.whl → 2.0.0__py3-none-any.whl

pydeflate/sources/imf.py

@@ -0,0 +1,203 @@
+from pathlib import Path
+
+import pandas as pd
+from imf_reader import weo
+
+from pydeflate.pydeflate_config import PYDEFLATE_PATHS, logger
+from pydeflate.sources.common import (
+    today,
+    add_pydeflate_iso3,
+    enforce_pyarrow_types,
+    compute_exchange_deflator,
+    read_data,
+    prefix_pydeflate_to_columns,
+)
+
+# List of WEO indicators of interest
+WEO_INDICATORS: list[str] = [
+    "NGDP_D",  # Gross domestic product, deflator
+    "PCPI",  # Inflation, average consumer prices
+    "PCPIE",  # Inflation, end of period consumer prices
+    "PPPEX",  # Implied PPP conversion rate
+    "NGDPD",  # Gross domestic product, current prices USD
+    "NGDP",  # Gross domestic product, current prices
+]
+
+
+def _filter_indicators(df: pd.DataFrame) -> pd.DataFrame:
+    """Filter the data to include only selected WEO indicators.
+
+    Args:
+        df (pd.DataFrame): The raw dataframe containing WEO data.
+
+    Returns:
+        pd.DataFrame: Filtered dataframe with only the relevant indicators.
+    """
+    return df.loc[df.CONCEPT_CODE.isin(WEO_INDICATORS)]
+
+
+def _to_units(df: pd.DataFrame) -> pd.DataFrame:
+    """Convert OBS_VALUE using the SCALE_CODE multiplier to get proper units.
+
+    Args:
+        df (pd.DataFrame): Dataframe with raw observation values.
+
+    Returns:
+        pd.DataFrame: Dataframe with scaled observation values.
+    """
+    df = df.copy()
+    df["OBS_VALUE"] = df["OBS_VALUE"] * df["SCALE_CODE"]
+    return df
+
+
+def _clean_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """Rename columns to more readable and consistent names.
+
+    Args:
+        df (pd.DataFrame): Dataframe with original IMF column names.
+
+    Returns:
+        pd.DataFrame: Dataframe with cleaned and renamed columns.
+    """
+    df = df.rename(
+        columns={
+            "REF_AREA_CODE": "entity_code",
+            "REF_AREA_LABEL": "entity",
+            "LASTACTUALDATE": "estimates_start_after",
+            "TIME_PERIOD": "year",
+            "OBS_VALUE": "value",
+        }
+    )
+    # Standardize column names to snake_case
+    df.columns = df.columns.str.lower().str.replace(" ", "_")
+    return df
+
+
+def _keep_useful_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """Retain only the columns that are useful for further analysis.
+
+    Args:
+        df (pd.DataFrame): Dataframe with cleaned columns.
+
+    Returns:
+        pd.DataFrame: Dataframe with only useful columns.
+    """
+    cols = [
+        "year",
+        "entity_code",
+        "entity",
+        "concept_code",
+        "value",
+    ]
+    return df[cols]
+
+
+def _pivot_concept_code(df: pd.DataFrame) -> pd.DataFrame:
+    """Pivot the concept code column to get a wide format for the data.
+
+    Args:
+        df (pd.DataFrame): Dataframe with concept code column.
+
+    Returns:
+        pd.DataFrame: Dataframe with concept code pivoted to columns.
+    """
+    return df.pivot(
+        index=[c for c in df.columns if c not in ["concept_code", "value"]],
+        columns="concept_code",
+        values="value",
+    ).reset_index()
+
+
+def _compute_exchange(df: pd.DataFrame) -> pd.DataFrame:
+    """Compute the exchange rate and append it to the original DataFrame.
+
+    This function calculates the exchange rate by dividing the 'NGDP'
+     Gross domestic product in local currency) by 'NGDPD' (Gross domestic product in USD).
+    It then appends this computed exchange rate to the original DataFrame,
+    and the new exchange rate rows are labeled with the 'EXCHANGE'
+    concept code.
+
+    Args:
+        df (pd.DataFrame): Input DataFrame containing columns with
+        'concept_code' as 'NGDPD' and 'NGDP'.
+
+    Returns:
+        pd.DataFrame: DataFrame with the computed exchange rate included,
+        along with the original data.
+    """
+    # Filter rows with 'NGDPD' (GDP in USD) and 'NGDP' (GDP in local currency)
+    exchange = df.loc[lambda d: d.concept_code.isin(["NGDPD", "NGDP"])]
+
+    # Pivot the data so 'NGDPD' and 'NGDP' become separate columns
+    exchange = exchange.pipe(_pivot_concept_code)
+
+    # Remove rows that correspond to 'NGDPD' and 'NGDP' from the original DataFrame
+    df = df.loc[lambda d: ~d.concept_code.isin(["NGDPD", "NGDP"])]
+
+    # Compute the exchange rate as NGDP (local currency) divided by NGDPD (USD)
+    exchange["value"] = round(exchange["NGDP"] / exchange["NGDPD"], 7)
+
+    # Label the exchange rate with a new concept code 'EXCHANGE'
+    exchange["concept_code"] = "EXCHANGE"
+
+    # Drop the original 'NGDPD' and 'NGDP' columns as they are no longer needed
+    exchange = exchange.drop(columns=["NGDPD", "NGDP"])
+
+    # Concatenate the original DataFrame with the new exchange rate data
+    return pd.concat([df, exchange], ignore_index=True)
+
+
+def download_weo() -> None:
+    """Download the WEO data, process it, and save it to a parquet file."""
+    logger.info("Downloading the latest WEO data...")
+
+    # Fetch and process the data through a pipeline of transformations
+    df = (
+        weo.fetch_data()
+        .pipe(_filter_indicators)
+        .pipe(_to_units)
+        .pipe(_clean_columns)
+        .pipe(_keep_useful_columns)
+        .pipe(_compute_exchange)
+        .pipe(add_pydeflate_iso3, column="entity", from_type="regex")
+        .pipe(_pivot_concept_code)
+        .pipe(compute_exchange_deflator, base_year_measure="NGDP_D")
+        .pipe(prefix_pydeflate_to_columns)
+        .pipe(enforce_pyarrow_types)
+        .reset_index(drop=True)
+    )
+
+    # Get today's date to use as a file suffix
+    suffix = today()
+
+    # Save the processed dataframe to parquet format
+    df.to_parquet(PYDEFLATE_PATHS.data / f"weo_{suffix}.parquet")
+
+    logger.info(f"Saved WEO data to weo_{suffix}.parquet")
+
+
+def _find_weo_files_in_path(path: Path) -> list:
+    """Find all WEO parquet files in the specified directory.
+
+    Args:
+        path (Path): The directory path to search for WEO parquet files.
+
+    Returns:
+        list: List of WEO parquet files found in the directory.
+    """
+    return list(path.glob("weo_*.parquet"))
+
+
+def read_weo(update: bool = False) -> pd.DataFrame:
+    """Read the latest WEO data from parquet files or download fresh data."""
+    return read_data(
+        file_finder_func=_find_weo_files_in_path,
+        download_func=download_weo,
+        data_name="WEO",
+        update=update,
+    )
+
+
+if __name__ == "__main__":
+    # Download the WEO data
+    dfi = read_weo(update=True)

pydeflate/sources/world_bank.py

@@ -0,0 +1,186 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+import pandas as pd
+import wbgapi as wb
+
+from pydeflate.pydeflate_config import PYDEFLATE_PATHS, logger
+from pydeflate.sources.common import (
+    enforce_pyarrow_types,
+    today,
+    compute_exchange_deflator,
+    read_data,
+    prefix_pydeflate_to_columns,
+)
+from pydeflate.utils import emu
+
+_INDICATORS: dict = {
+    "NY.GDP.DEFL.ZS": "NGDP_D",  # GDP Deflator (Index)
+    "NY.GDP.DEFL.ZS.AD": "NGDP_DL",  # GDP Deflator linked series
+    "FP.CPI.TOTL": "CPI",  # Consumer Price Index (CPI)
+    "PA.NUS.FCRF": "EXCHANGE",  # Official Exchange Rate
+}
+
+
+def get_wb_indicator(series: str, value_name: str | None = None) -> pd.DataFrame:
+    """Fetch a World Bank indicator and transform it into a cleaned DataFrame.
+
+    Args:
+        series (str): The World Bank indicator series code.
+        value_name (str | None): The column name to assign to the series values.
+                                 If None, the series code will be used as the column name.
+
+    Returns:
+        pd.DataFrame: DataFrame with entity code, entity name, year, and the indicator values.
+    """
+    # Fetch the indicator data from World Bank API, clean and structure it
+    return (
+        wb.data.DataFrame(
+            series=series,
+            db=2,  # World Development Indicators database
+            skipBlanks=True,
+            columns="series",
+            numericTimeKeys=True,
+            labels=True,
+        )
+        .reset_index()
+        .sort_values(by=["economy", "Time"])  # Sort for easier reading
+        .drop(columns=["Time"])  # Remove unnecessary column
+        .rename(
+            columns={
+                "economy": "entity_code",
+                "Country": "entity",
+                "time": "year",
+                series: value_name or series,
+            }
+        )
+        .reset_index(drop=True)  # Drop the old index after reset
+    )
+
+
+def _eur_series_fix(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Fix the exchange rate for the Euro area countries. This is done by assigning the
+    exchange rate of the Euro to the countries in the Euro area. This is necessary
+    because the series for Euro area countries are missing EUR exchange rates.
+
+    Args:
+        df: pd.DataFrame: The DataFrame containing the World Bank data.
+
+    Returns:
+        pd.DataFrame: The DataFrame with the fixed exchange rates for the Euro area countries.
+
+    """
+    # Handle cases where EUR is represented differently in the World Bank data
+    df["entity_code"] = df["entity_code"].replace({"EMU": "EUR"})
+
+    # Find the "Euro" data. This is done given that some countries are missing
+    # exchange rates, but they are part of the Euro area.
+    eur = (
+        df.loc[lambda d: d["entity_code"] == "EUR"]
+        .dropna(subset=["EXCHANGE"])
+        .set_index("year")["EXCHANGE"]
+        .to_dict()
+    )
+
+    # Euro area countries without exchange rates
+    eur_mask = (df["entity_code"].isin(emu())) & (df["EXCHANGE"].isna())
+
+    # Assign EURO exchange rate to euro are countries from year euro adopted
+    df.loc[eur_mask, "EXCHANGE"] = df["year"].map(eur)
+
+    return df
+
+
+def _parallel_download_indicators(indicators: dict) -> list[pd.DataFrame]:
+    """Download multiple World Bank indicators in parallel.
+
+    Args:
+        indicators (dict): A dictionary of World Bank indicators to download.
+
+    Returns:
+        list[pd.DataFrame]: A list of DataFrames containing the downloaded indicators
+
+    """
+    # List to store the resulting dataframes
+    dfs = []
+
+    # Use ThreadPoolExecutor to fetch indicators in parallel
+    with ThreadPoolExecutor() as executor:
+        # Submit all tasks to the executor (downloading indicators in parallel)
+        future_to_series = {
+            executor.submit(get_wb_indicator, series, value_name): series
+            for series, value_name in indicators.items()
+        }
+
+        # Collect the results as they complete
+        for future in as_completed(future_to_series):
+            series = future_to_series[future]
+            try:
+                df_ = future.result().set_index(["year", "entity_code", "entity"])
+                dfs.append(df_)
+            except Exception as exc:
+                # Log or handle any errors that occur during the download
+                logger.warning(f"Error downloading series {series}: {exc}")
+
+    return dfs
+
+
+def download_wb() -> None:
+    """Download multiple World Bank indicators in parallel and save as a parquet file.
+
+    This function fetches all indicators defined in _INDICATORS in parallel, concatenates
+    them into a single DataFrame, and saves the result as a parquet file using today's date as a suffix.
+    """
+    logger.info("Downloading the latest World Bank data...")
+
+    indicators_data = _parallel_download_indicators(indicators=_INDICATORS)
+
+    # Concatenate all DataFrames horizontally (by columns)
+    df = pd.concat(indicators_data, axis=1).reset_index()
+
+    # cleaning
+    df = (
+        df.pipe(_eur_series_fix)
+        .pipe(compute_exchange_deflator, base_year_measure="NGDP_D")
+        .assign(pydeflate_iso3=lambda d: d.entity_code)
+        .sort_values(by=["year", "entity_code"])
+        .pipe(prefix_pydeflate_to_columns)
+        .pipe(enforce_pyarrow_types)
+        .reset_index(drop=True)
+    )
+
+    # Get today's date to use as a file suffix
+    suffix = today()
+
+    # Save the DataFrame as a parquet file
+    output_path = PYDEFLATE_PATHS.data / f"wb_{suffix}.parquet"
+    df.to_parquet(output_path)
+
+    logger.info(f"Saved World Bank data to wb_{suffix}.parquet")
+
+
+def _find_wb_files_in_path(path: Path) -> list:
+    """Find all WB parquet files in the specified directory.
+
+    Args:
+        path (Path): The directory path to search for WB parquet files.
+
+    Returns:
+        list: List of WB parquet files found in the directory.
+    """
+    return list(path.glob("wb_*.parquet"))
+
+
+def read_wb(update: bool = False) -> pd.DataFrame:
+    """Read the latest World Bank data from parquet files or download fresh data."""
+    return read_data(
+        file_finder_func=_find_wb_files_in_path,
+        download_func=download_wb,
+        data_name="World Bank",
+        update=update,
+    )
+
+
+if __name__ == "__main__":
+    df = read_wb(True)

pydeflate/utils.py

@@ -1,10 +1,10 @@
 import json
 
-import country_converter as coco
 import numpy as np
 import pandas as pd
 
-from pydeflate.pydeflate_config import PYDEFLATE_PATHS
+from pydeflate.pydeflate_config import PYDEFLATE_PATHS, logger
+from pydeflate.sources.common import enforce_pyarrow_types
 
 
 def oecd_codes() -> dict:
@@ -35,33 +35,66 @@ def clean_number(number):
 
     return float(number)
 
+def create_pydeflate_year(
+    data: pd.DataFrame, year_column: str, year_format: str | None = None
+) -> pd.DataFrame:
+    if year_format is None:
+        year_format = "ISO8601"
 
-def check_year_as_number(df: pd.DataFrame, date_column: str) -> (pd.DataFrame, bool):
-    """Check whether the date column contains an int instead of datetime.
-    This changes the column to datetime and returns a flag"""
+    data = data.copy()
 
-    if pd.api.types.is_numeric_dtype(df[date_column]):
-        df[date_column] = pd.to_datetime(df[date_column], format="%Y")
-        year_as_number = True
-    else:
-        year_as_number = False
+    data["pydeflate_year"] = pd.to_datetime(
+        data[year_column], format=year_format
+    ).dt.year
 
-    return df, year_as_number
+    return data
 
 
-def to_iso3(
-    df: pd.DataFrame,
-    codes_col: str,
-    target_col: str,
-    src_classification: str | None = None,
-    not_found: str | None = None,
+def merge_user_and_pydeflate_data(
+    data: pd.DataFrame,
+    pydeflate_data: pd.DataFrame,
+    entity_column: str,
+    ix: list[str],
 ) -> pd.DataFrame:
-    """Convert a column of country codes to iso3"""
+    return data.merge(
+        pydeflate_data,
+        how="outer",
+        left_on=["pydeflate_year", entity_column],
+        right_on=ix,
+        suffixes=("", "_pydeflate"),
+        indicator=True,
+    ).pipe(enforce_pyarrow_types)
+
+
+def get_unmatched_pydeflate_data(
+    merged_data: pd.DataFrame,
+):
+    return merged_data.loc[merged_data["_merge"] == "left_only"].filter(
+        regex="^(?!pydeflate_)(?!.*_pydeflate$)"
+    )
+
+
+def get_matched_pydeflate_data(
+    merged_data: pd.DataFrame,
+):
+    return (
+        merged_data.loc[merged_data["_merge"] != "right_only"]
+        .drop(columns="_merge")
+        .reset_index(drop=True)
+    )
+
 
-    cc = coco.CountryConverter()
+def flag_missing_pydeflate_data(unmatched_data: pd.DataFrame):
+    """Flag data which is present in the input data but missing in pydeflate's data."""
+    if unmatched_data.empty:
+        return
 
-    df[target_col] = cc.pandas_convert(
-        df[codes_col], src=src_classification, to="ISO3", not_found=not_found
+    missing = (
+        unmatched_data.drop_duplicates()
+        .dropna(axis=1)
+        .drop(columns="_merge")
+        .to_string(index=False)
     )
 
-    return df
+    # log all missing data
+    logger.info(f"Missing exchange data for:\n {missing}")

{pydeflate-1.4.2.dist-info → pydeflate-2.0.0.dist-info}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022, Jorge Rivera
+Copyright (c) 2021-2024, Jorge Rivera
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal