TiMBA-Trade 0.1.0__py3-none-any.whl

timba_trade/core/BACIDataProcessor.py

@@ -0,0 +1,111 @@
+from src.timba_trade.core.InputManager import InputManager
+from src.timba_trade.paramters.hscodes import timba_commodity_list, timba_comcode
+import src.timba_trade.paramters.paramters as p
+from src.timba_trade.utils.helper import hs_to_fao_dict
+from src.timba_trade.utils.logger import setup_logging
+
+import logging
+import pandas as pd
+
+setup_logging()
+logger = logging.getLogger(__name__)
+
+
+class BACIProcessor:
+    def __init__(self, df:pd.DataFrame, year: int = 2023,):
+        self.im = InputManager()
+        self.commodity_list = timba_commodity_list
+        self.year = year
+        self.df = df
+        
+    @staticmethod
+    def _groupby_sum(df: pd.DataFrame, id_list: list, value_list: list) -> pd.DataFrame:
+        return df.groupby(id_list)[value_list].sum().reset_index()
+
+    def prepare_baci_data(self):
+        hs_to_fao_map = hs_to_fao_dict(self.commodity_list)
+
+        df = self.df.copy()
+        df["Commodity"] = df[p.COM_COLUMN].map(hs_to_fao_map)
+
+        return df[df[p.TIME_COLUMN] == self.year].dropna(subset=["Commodity"]).reset_index(drop=True)
+
+    def create_bilateral_trade(self,df:pd.DataFrame)->pd.DataFrame:
+        counterflow = df.rename(
+            columns={
+                p.REPORTER_COLUMN: p.PARTNER_COLUMN,
+                p.PARTNER_COLUMN: p.REPORTER_COLUMN,
+                p.QUANTITY_COLUMN: "Import",
+            }
+        )
+        result = df.merge(counterflow,on=p.BACIID,how="outer")
+        result = result.rename(columns={p.QUANTITY_COLUMN: "Export"})
+
+        return self._groupby_sum(df=result,id_list=p.BACIID,value_list=p.X_M_NAMES)
+
+    def _merge_trade_totals(self,df)->pd.DataFrame:
+        exporter_totals = self._groupby_sum(df=df,id_list=p.TOTALSID,value_list=p.X_M_NAMES)
+        importer_totals = exporter_totals.rename(columns={p.REPORTER_COLUMN: p.PARTNER_COLUMN})
+
+        total = df.merge(
+            right=exporter_totals,
+            how="outer",
+            on=p.TOTALSID,
+            suffixes=(
+                "_bilateral",
+                "_total_exporter",
+            ),
+        )
+
+        total = total.merge(
+            right=importer_totals,
+            how="outer",
+            on=p.TOTALSIMPORTID,
+            suffixes=("", "_total_importer"),
+        )
+
+        return total
+
+    def add_trade_totals(self,df:pd.DataFrame,map:dict)->pd.DataFrame:
+        total = self._merge_trade_totals(df)
+
+        total = total[
+            [
+                p.TIME_COLUMN,
+                p.REPORTER_COLUMN,
+                p.PARTNER_COLUMN,
+                "Commodity",
+                "Export_bilateral",
+                "Import_bilateral",
+                "Export_total_exporter",
+                "Import_total_exporter",
+                "Export",
+                "Import",
+            ]
+        ].rename(
+            columns={
+                p.REPORTER_COLUMN: "Reporter",
+                p.PARTNER_COLUMN: "Partner",
+                "Export": "Export_total_importer",
+                "Import": "Import_total_importer",
+            }
+        )
+
+        total["Period"] = 0
+        total["Commodity"] = total["Commodity"].map(map)
+
+        return total
+
+    def main(self)->pd.DataFrame:
+        baci_data = self.prepare_baci_data()
+        baci_data = self._groupby_sum(
+            df=baci_data,
+            id_list=p.BACIID,
+            value_list=["Quantity"]
+        )
+        bilateral_trade = self.add_trade_totals(
+            df=self.create_bilateral_trade(df=baci_data),
+            map=timba_comcode,
+        )
+
+        return bilateral_trade

timba_trade/core/InputManager.py

@@ -0,0 +1,35 @@
+from src.timba_trade.paramters.paths import INPUT_PATH, BITRADEPATH
+
+import pandas as pd
+import pickle
+import gzip
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class InputManager:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def load_timba_results():
+        path = INPUT_PATH
+        pkl_files = list(path.glob("*.pkl"))
+
+        if not pkl_files:
+            raise FileNotFoundError(
+                f"   No .pkl file found at: {path.resolve()}"
+            )
+
+        file = max(pkl_files, key=lambda f: f.stat().st_mtime)
+
+        with gzip.open(file, "rb") as pkl_file:
+            obj = pickle.load(pkl_file)
+
+        return obj["data_periods"]
+
+    @staticmethod
+    def load_baci_data():
+        return pd.read_parquet(BITRADEPATH)
+

timba_trade/core/OutputManager.py

@@ -0,0 +1,30 @@
+import pandas as pd
+from pathlib import Path
+import pyarrow
+
+from src.timba_trade.paramters.paths import OUTPUT_PATH
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+class OutputManager:
+    def __init__(self):
+        pass
+
+    def save_df(self, df, file_name, max_attempts=10):
+        for i in range(max_attempts):
+            file = (file_name + ".csv") if i == 0 else f"{file_name}_({i}).csv"
+            try:
+                df.to_csv(OUTPUT_PATH / Path(file), index=False) 
+                logger.info(f"CSV file saved at: {OUTPUT_PATH / Path(file)}")
+                return file
+            except PermissionError:
+                if i == max_attempts - 1:
+                    raise
+                continue
+
+    def save_parquet(self, df, file_name):
+        file = Path(file_name + ".parquet")
+        df.to_parquet(OUTPUT_PATH / file) 
+        logger.info(f"Parquet saved at: {OUTPUT_PATH / Path(file)}")

timba_trade/core/ProcessManager.py

@@ -0,0 +1,107 @@
+from src.timba_trade.core.TiMBADataProcessor import TiMBADataProcessor
+from src.timba_trade.core.BACIDataProcessor import BACIProcessor
+from src.timba_trade.core.InputManager import InputManager
+from src.timba_trade.logic.CalculationManager import CalcManager
+from src.timba_trade.logic.RAS import RAS
+from src.timba_trade.core.OutputManager import OutputManager
+
+import time
+import pandas as pd
+import logging
+logger = logging.getLogger(__name__)
+
+
+class ProcessManager:
+
+    def __init__(self):
+        self.data = None
+        self.country_data = None
+        self.partner_data = None
+        self.countries = None
+        self.commodities = None
+        self.periods = None
+        self.dataset = None
+        self.result = None
+        self.start_time = None
+        self.im = InputManager()
+        self.om = OutputManager()
+
+    def start_process(self, process_description:str):
+        self.start_time = time.time()
+        logger.info(process_description)
+
+
+    def end_process(self, process_description):
+        if self.start_time is None:
+            raise ValueError("The process has not been started. Use `start_process` first.")
+        
+        end_time = time.time()
+        elapsed_time = round(end_time - self.start_time, 2)
+        logger.info(
+            f"{process_description} in {elapsed_time:.1f} seconds"
+        )
+
+    def load_timba_process(self, data:pd.DataFrame|None)->pd.DataFrame:
+        self.start_process("Beginn to load TiMBA data")
+        if data is None:
+            data = self.im.load_timba_results()
+        tdp = TiMBADataProcessor(data)
+        timba_data = tdp.main()
+        self.end_process("TiMBA data loaded")
+
+        return timba_data
+
+    def load_baci_process(self)->pd.DataFrame:
+        self.start_process("Beginn to load BACI data")
+        baci_data = self.im.load_baci_data()
+        bdp = BACIProcessor(df=baci_data)
+        baci_data = bdp.main()
+        self.end_process("BACI data loaded")
+
+        return baci_data
+
+    def main_process(self, td:pd.DataFrame|None)->pd.DataFrame:
+        timba_data = self.load_timba_process(data=td)
+        fao_df = CalcManager(data=timba_data).main().copy()
+        fao_df = fao_df[["Reporter","Partner","Commodity","Period","X_q_o","M_q_p","P_share_growth"]]
+
+        bt_df = self.load_baci_process()[["Reporter","Partner","Commodity","Period","Export_bilateral"]]
+        bt_df["Export_bilateral"] /= 1000
+
+        self.start_process(f"Beginn the RAS process")
+
+        fao_df = fao_df.loc[~fao_df["Commodity"].isin([89, 90, 93])]
+        bt_df = bt_df.loc[~bt_df["Commodity"].isin([89, 90, 93])]
+        
+        merge_keys = ["Reporter","Partner", "Commodity", "Period"]
+        bt_result = pd.DataFrame()
+        for period in fao_df.Period.unique():
+            logger.info(f" Period: {period}")
+            fp = fao_df[fao_df.Period==period]
+
+            bt_df = bt_df[["Reporter","Partner","Commodity","Export_bilateral"]]
+            bt_df["Period"]=period
+            bt_df = bt_df.drop_duplicates()
+            bt_df = bt_df.dropna().reset_index(drop=True)
+            bt_df = bt_df[bt_df['Reporter'] != 'RoW']
+            bt_df = bt_df[bt_df['Partner'] != 'RoW']
+
+            ras_df = fp.merge(
+                bt_df,
+                on=merge_keys,
+                how="left"
+            )
+            ras_df["Export_bilateral"] *= ras_df["P_share_growth"]
+
+            bt_df = RAS().fit(ras_df)
+            bt_result = pd.concat([bt_result,bt_df]).reset_index(drop=True)
+
+        self.om.save_parquet(
+            df=bt_result,
+            file_name="TiMBA_bilateral_results"
+        )
+
+        self.end_process(f"End of RAS process")
+
+        return bt_result
+    

timba_trade/core/TiMBADataProcessor.py

@@ -0,0 +1,217 @@
+from src.timba_trade.paramters.country_info import country_dict
+import src.timba_trade.paramters.paramters as params
+
+import pandas as pd
+import numpy as np
+import xarray as xr
+import logging
+logger = logging.getLogger(__name__)
+
+
+class TiMBADataProcessor:
+    def __init__(self, data):
+        self.data = data
+
+    def _rename_columns(self):
+        self.data = self.data.rename(columns=params.COLUMN_MAPPING)   
+
+    def _add_iso_codes(self):
+        self.data["Country"] = self.data["RegionCode"].map(country_dict)
+
+        self.data = (
+            self.data
+            .dropna(subset=["Country"])
+            .reset_index(drop=True)
+        )
+
+        self.data = self.data[
+            [
+                "Country",
+                "Commodity",
+                "domain",
+                "Period",
+                "p",
+                "q",
+            ]
+        ]
+
+    def _aggregate_domains(self):
+        self.data = self.data[
+            ~self.data["domain"].isin(params.DOMAINS_TO_REMOVE)
+        ].copy()
+
+        self.data["domain"] = self.data["domain"].replace(
+            params.DOMAIN_MAPPING
+        )
+
+        self.data = (
+            self.data
+            .groupby(
+                params.GROUP_COLUMNS,
+                as_index=False,
+                observed=True
+            )
+            .agg(
+                p=("p", "max"),
+                q=("q", "max"),
+            )
+        )
+
+    def _downcasting(self):
+        self.data["Country"] = self.data["Country"].astype("category")
+        self.data["Commodity"] = self.data["Commodity"].astype("category")
+        self.data["domain"] = self.data["domain"].astype("category")
+        self.data["p"] = self.data["p"].astype("float32")
+        self.data["q"] = self.data["q"].astype("float32")
+        self.data["Period"] = self.data["Period"].astype("int16")
+
+    def _pivot_domains(self):
+        self.data = self.data.pivot(
+            index=params.INDEX_COLUMNS,
+            columns="domain",
+            values=params.VALUE_COLUMNS,
+        )
+
+        self.data.columns = [
+            f"{domain}_{variable}"
+            for variable, domain in self.data.columns
+        ]
+
+        self.data = self.data.reset_index()
+
+    def _sort_data(self):
+        self.data = (
+            self.data
+            .sort_values(params.GROUP_COLUMNS)
+            .reset_index(drop=True)
+        )
+
+    def _prepare_timba_for_biltrade(self):
+        self.country_data = self.data[params.COUNTRY_COLUMNS].copy()
+
+        self.partner_data = self.data[params.PARTNER_COLUMNS].rename(
+            columns={"Country": "Partner"}
+        )
+
+        self.countries = np.sort(self.data["Country"].unique())
+        self.commodities = np.sort(self.data["Commodity"].unique())
+        self.periods = np.sort(self.data["Period"].unique())
+
+    def _to_dense_array(self, df: pd.DataFrame, dim_name: str, value_col: str) -> xr.DataArray:
+        series = (
+            df[[dim_name, "Commodity", "Period", value_col]]
+            .set_index([dim_name, "Commodity", "Period"])[value_col]
+            .astype(np.float32)
+        )
+
+        array = xr.DataArray.from_series(series)
+        array = array.reindex(
+            {
+                dim_name: self.countries,
+                "Commodity": self.commodities,
+                "Period": self.periods,
+            }
+        )
+
+        return array
+
+    def _to_dense_array(self, df: pd.DataFrame, dim_name: str, value_col: str) -> xr.DataArray:
+        series = (
+            df[[dim_name, "Commodity", "Period", value_col]]
+            .set_index([dim_name, "Commodity", "Period"])[value_col]
+            .astype(np.float32)
+        )
+        array = xr.DataArray.from_series(series)
+        array = array.reindex(
+            {
+                dim_name: self.countries,
+                "Commodity": self.commodities,
+                "Period": self.periods,
+            }
+        )
+        return array
+
+    @staticmethod
+    def _log_of_price_share(data):
+        with np.errstate(divide="ignore"):
+            data["P_share"] = np.log(data["P_p_o"] / data["P_p_p"])
+        return data
+
+    def _build_dataset(self):
+        self.dataset = xr.Dataset(
+            {
+                "P_p_o": self._to_dense_array(self.country_data, "Country", "P_p"),
+                "P_q_o": self._to_dense_array(self.country_data, "Country", "P_q"),
+                "M_p_o": self._to_dense_array(self.country_data, "Country", "M_p"),
+                "M_q_o": self._to_dense_array(self.country_data, "Country", "M_q"),
+                "X_p_o": self._to_dense_array(self.country_data, "Country", "X_p"),
+                "X_q_o": self._to_dense_array(self.country_data, "Country", "X_q"),
+                "P_p_p": self._to_dense_array(self.partner_data, "Partner", "P_p"),
+                "P_q_p": self._to_dense_array(self.partner_data, "Partner", "P_q"),
+                "X_p_p": self._to_dense_array(self.partner_data, "Partner", "X_p"),
+                "X_q_p": self._to_dense_array(self.partner_data, "Partner", "X_q"),
+                "M_q_p": self._to_dense_array(self.partner_data, "Partner", "M_q"),
+            }
+        )
+
+        self._log_of_price_share(data=self.dataset)
+        self._add_lags()
+
+    def _add_lags(self):
+        lagged = self.dataset.shift(Period=1)
+        lagged = lagged.rename(
+            {var: f"{var}_lag" for var in lagged.data_vars}
+        )
+
+        self.dataset = xr.merge([self.dataset, lagged])
+
+    def _process_all(self):
+        chunk_size = params.PROCESS_CHUNK_SIZE
+        n_countries = len(self.countries)
+
+        chunks = []
+
+        for start in range(0, n_countries, chunk_size):
+            end = min(start + chunk_size, n_countries)
+
+            chunk_ds = self.dataset.isel(Country=slice(start, end))
+
+            chunk_df = (
+                chunk_ds.to_dataframe()
+                .reset_index()
+            )
+
+            chunks.append(chunk_df)
+
+        self.result = pd.concat(chunks, ignore_index=True)
+
+        self.result = self.result[
+            [
+                "Country", "Partner", "Commodity", "Period",
+                "P_p_o", "P_p_o_lag",
+                "P_q_o", "P_q_o_lag", 
+                "P_p_p", "P_p_p_lag",  
+                "P_q_p", "P_q_p_lag",
+                "M_p_o", "M_p_o_lag", 
+                "M_q_o", "M_q_o_lag",
+                "M_q_p", "M_q_p_lag",
+                "X_p_o", "X_p_o_lag",  
+                "X_q_o", "X_q_o_lag",
+                "X_p_p", "X_p_p_lag",  
+                "X_q_p", "X_q_p_lag",  
+                "P_share", "P_share_lag",
+            ]
+        ]
+
+    def main(self) -> pd.DataFrame:
+        self._rename_columns()    
+        self._add_iso_codes()
+        self._aggregate_domains()
+        self._sort_data()
+        self._downcasting()
+        self._pivot_domains()
+        self._prepare_timba_for_biltrade()
+        self._build_dataset()
+        self._process_all()
+
+        return self.result

timba_trade/logic/CalculationManager.py

@@ -0,0 +1,52 @@
+import numpy as np
+import pandas as pd
+
+
+class CalcManager:
+    def __init__(self, data:pd.DataFrame):
+        self.AE = 1
+        self.data = data
+
+    def _calc_price_share(self):
+        df = self.data
+        df["P_share_growth"] = df.P_share - df.P_share_lag + 1
+        df["LD"] = df.P_q_o - df.X_q_o + df.M_q_o
+        df["LD_lag"] = df.P_q_o_lag - df.X_q_o_lag + df.M_q_o_lag
+        df["LD_growth"] = df.LD / df.LD_lag
+
+        df.rename(
+            columns={
+                "Country":"Reporter"
+            },
+            inplace=True
+        )
+
+        self.results = df[[
+            "Reporter", "Partner", "Commodity", "Period",
+            "X_q_o", "M_q_o", "X_q_p", "M_q_p",
+            "P_share_growth","LD_growth"
+        ]]
+
+    def clip_shares(self,cols:list,lb_fix:float=None,ub_fix:float=None):
+        df = self.results
+        for col in cols:
+            df[col] = df[col].replace([np.inf, -np.inf, np.nan], 1)
+            q1 = df[col].quantile(0.25)
+            q3 = df[col].quantile(0.75)
+            iqr = (q3 - q1)
+
+            lb = lb_fix if lb_fix is not None else q1 - iqr * 5
+            ub = ub_fix if ub_fix is not None else q3 + iqr * 5
+                
+            df[col] = df[col].clip(lower=lb, upper=ub)
+
+    def _combined_effect(self):
+        self.results["M_growth"] = self.results["LD_growth"] * self.results["P_share_growth"] ** (self.AE)
+        
+    def main(self)->pd.DataFrame:
+        self._calc_price_share()
+        self.clip_shares(cols=["P_share_growth"],lb_fix=0.5,ub_fix=2)
+        self.clip_shares(cols=["LD_growth"])
+        self._combined_effect()
+        #self.clip_shares(cols=["M_growth"])
+        return self.results