TiMBA-Charts 0.1.1__py3-none-any.whl

Toolbox/Input/Additional_Information/reformate_fao_data.py

@@ -0,0 +1,20 @@
+import pandas as pd
+
+fao = pd.read_csv('Forestry_subsetted_Data_reformatted.csv')
+
+h_commodity = pd.read_csv('commodity_info.csv', encoding="ISO-8859-1")
+h_commodity = h_commodity[['GFPM_Code','FAO-Code']]
+h_commodity.columns = ['GFPM_Code_Commodity','FAO_Code_commodity']
+
+h_country = pd.read_csv('country_info.csv', encoding="ISO-8859-1")
+h_country = h_country[['Country-Code', 'FAOCou-Code']]
+h_country.columns = ['GFPM_Code_Country','FAO_Code_Country']
+
+fao_commodity = fao.merge(right=h_commodity, how='left',left_on='Item_Codes',right_on='FAO_Code_commodity').dropna()
+fao_final = fao_commodity.merge(right=h_country, how='left',left_on='Area_Codes',right_on='FAO_Code_Country').dropna()
+fao_final = fao_final.reset_index(drop=True)
+
+fao_final_o5 = fao_final[fao_final.GFPM_Code_Country == 'o5']
+print(fao_final_o5)
+
+fao_final.to_csv('FAO_Data_py.csv')

Toolbox/classes/dashboard.py

@@ -0,0 +1,362 @@
+import plotly.graph_objects as go
+import dash
+import dash_bootstrap_components as dbc
+from dash import dcc, html
+from dash.dependencies import Input, Output, State
+import webbrowser
+from threading import Timer
+import pandas as pd
+import numpy as np
+from pathlib import Path
+import plotly.express as px
+import textwrap
+
+PACKAGEDIR = Path(__file__).parent.parent.absolute()
+
+class DashboardPlotter:
+
+    def __init__(self, data):
+        self.data = data
+        self.app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
+        self.start = self.data['year'].min()
+        self.end = self.data['year'].max()
+        self.color_list = [
+            '#2A4D69',  # Dunkelblau
+            '#4B8BBE',  # Hellblau
+            '#D35400',  # Dunkelorange
+            '#AAB7B8',  # Grau
+            '#9B59B6',  # Lila
+            '#2980B9',  # Blau
+            '#27AE60',  # Grün
+            '#6C757D',  # Dunkelgrau
+            '#F1C40F',  # Senfgelb
+            '#E67E22',  # Orange  
+        ]
+        self.create_layout()
+        self.create_callbacks()
+
+    def create_layout(self):
+        dropdown_style = {'height': '30px','marginBottom': '10px'}
+        self.app.layout = dbc.Container([
+            dbc.Row([
+                dbc.Col(width=5),  # Leere Spalte für den linken Rand
+                dbc.Col(
+                    html.Img(src="https://raw.githubusercontent.com/TI-Forest-Sector-Modelling/TiMBA/ToolBox_implementation_cm/images/timba_dashboard_logo.png",
+                             style={'height': '90px', 'width': 'auto'}),
+                    width=1
+                ),
+                dbc.Col(
+                    #html.H1("TiMBA Dashboard", className="text-center mb-4"), width=10
+                )
+            ]),
+            dbc.Row([
+                dbc.Col([
+                    dbc.Card([
+                        dbc.CardBody([
+                            html.H4("Filters", className="card-title"),
+                            dcc.Dropdown(id='region-dropdown',
+                                         options=[{'label': i, 'value': i} for i in sorted(self.data['ISO3'].dropna().unique())],
+                                         multi=True,
+                                         placeholder="Select Country...",
+                                         style=dropdown_style),
+                            dcc.Dropdown(id='continent-dropdown',
+                                         options=[{'label': i, 'value': i} for i in sorted(self.data['Continent'].dropna().unique())],
+                                         placeholder="Select Continent...",
+                                         multi=True,
+                                         style=dropdown_style),
+                            dcc.Dropdown(id='domain-dropdown',
+                                         options=[{'label': i, 'value': i} for i in sorted(self.data['domain'].dropna().unique())],
+                                         placeholder="Select Domain...",
+                                         multi=True,
+                                         style=dropdown_style),
+                            dcc.Dropdown(id='commodity-dropdown',
+                                         options=[{'label': i, 'value': i} for i in sorted(self.data['Commodity'].dropna().unique())],
+                                         placeholder="Select Commodity...",
+                                         multi=True,
+                                         style=dropdown_style),
+                            dcc.Dropdown(id='commodity-group-dropdown',
+                                         options=[{'label': i, 'value': i} for i in self.data['Commodity_Group'].dropna().unique().tolist()],
+                                         placeholder="Select Commodity Group...",
+                                         multi=True,
+                                         style=dropdown_style),
+                            dcc.Dropdown(id='scenario-filter',
+                                         options=[{'label': i, 'value': i} for i in self.data['Scenario'].unique()],
+                                         placeholder="Select Scenario...",
+                                         multi=True,
+                                         style=dropdown_style),
+                            html.Button("Download CSV", id="btn_csv"),
+                            dcc.Download(id="download-dataframe-csv"),
+                        ])
+                    ], className="mb-4", style={'white': 'white'}),  #filter box
+                    dbc.Card([
+                        dbc.CardBody([
+                                dcc.Graph(id='price-plot',
+                                      config={'toImageButtonOptions': {'format': 'png', 'filename': 'price_plot'}},
+                                      style={'height': '45vh','width':'46vh'})
+                    ])
+                ], style={'white': 'white'})  #price box
+            ], width=3),
+            dbc.Col([
+                dbc.Row([
+                    dbc.Col([
+                        dbc.Card([
+                            dbc.CardBody([
+                                dcc.Graph(id='quantity-plot',
+                                          config={'toImageButtonOptions': {'format': 'png', 'filename': 'quantity_plot'}},
+                                          style={'height': '84.5vh'})
+                            ])
+                        ], style={'backgroundColor': 'white'}) #mittelbox
+                    ], width=8), # Breite auf 8 reduziert
+                    dbc.Col([
+                        dbc.Card([
+                            dbc.CardBody([
+                                dcc.Graph(id='forstock-plot',  # Geändert: ID auf 'forstock-plot'
+                                          config={'toImageButtonOptions': {'format': 'png'}},
+                                          style={'height': '39vh'})
+                            ])
+                        ], style={'backgroundColor': 'white', 'marginBottom': '20px'}), # Abstand hinzugefügt
+                        dbc.Card([
+                            dbc.CardBody([
+                                html.H5("Filter for Worldmap", className="card-title"),
+                                #html.H6("Scenario Filter", className="card-title"),  # Titel für den Scenario-Filter
+                                #html.H6("Year Filter", className="card-title"),  # Titel für den Year-Filter
+                                dcc.Dropdown(
+                                    id='year-filter',
+                                    options=[{'label': i, 'value': i} for i in sorted(self.data['year'].unique())],
+                                    placeholder="Select Year...",
+                                    style=dropdown_style
+                                ),
+                                 dcc.Graph(id='world-map',  # Geändert: ID auf 'world-map'
+                                          config={'toImageButtonOptions': {'format': 'png'}},
+                                          style={'height': '31.75vh'})
+                            ])
+                        ], style={'backgroundColor': 'white'})
+                    ], width=4), # Breite auf 4 gesetzt
+                ]),
+            ], width=9)
+        ])
+    ], fluid=True, style={'backgroundColor': 'white'}) #gesamthintergrund
+
+    def create_callbacks(self):
+        @self.app.callback(
+            [Output('quantity-plot', 'figure'),
+             Output('price-plot', 'figure'),
+             Output('forstock-plot', 'figure')],
+            [Input('region-dropdown', 'value'),
+             Input('continent-dropdown', 'value'),
+             Input('domain-dropdown', 'value'),
+             Input('commodity-dropdown', 'value'),
+             Input('commodity-group-dropdown', 'value'),
+             Input('scenario-filter', 'value')]
+        )
+        def update_plots(region, continent, domain, commodity, commodity_group,scenario):
+            return self.update_plot_data(region, continent, domain, commodity, commodity_group,scenario)
+
+        @self.app.callback(
+            Output('world-map', 'figure'),
+            [Input('scenario-filter', 'value'),
+             Input('year-filter', 'value'),
+             Input('region-dropdown', 'value'),
+             Input('continent-dropdown', 'value'),
+             Input('domain-dropdown', 'value'),
+             Input('commodity-dropdown', 'value'),
+             Input('commodity-group-dropdown', 'value')]
+        )
+        def update_world_map(scenario, year, region, continent, domain, commodity, commodity_group):
+            return self.create_world_map(region, continent, domain, commodity, commodity_group, scenario, year)
+
+        @self.app.callback(
+            Output("download-dataframe-csv", "data"),
+            Input("btn_csv", "n_clicks"),
+            [State('region-dropdown', 'value'),
+            State('continent-dropdown', 'value'),
+            State('domain-dropdown', 'value'),
+            State('commodity-dropdown', 'value'),
+            State('commodity-group-dropdown', 'value'),
+            State('scenario-filter', 'value')],  # Geändert von Input zu State
+            prevent_initial_call=True
+        )
+        def func(n_clicks, region, continent, domain, commodity, commodity_group, scenario):
+            if n_clicks is None:
+                raise dash.exceptions.PreventUpdate
+            filtered_data = self.filter_data(region, continent, domain, commodity, commodity_group, scenario)
+            return dcc.send_data_frame(filtered_data.to_csv, "filtered_data.csv")
+
+
+    def filter_data(self, region, continent, domain, commodity, commodity_group, scenario):
+        filtered_data = self.data
+        if region and isinstance(region, list):
+            filtered_data = filtered_data[filtered_data['ISO3'].isin(region)]
+        if continent and isinstance(continent, list):
+            filtered_data = filtered_data[filtered_data['Continent'].isin(continent)]
+        if domain and isinstance(domain, list):
+            filtered_data = filtered_data[filtered_data['domain'].isin(domain)]
+        if commodity and isinstance(commodity, list):
+            filtered_data = filtered_data[filtered_data['Commodity'].isin(commodity)]
+        if commodity_group and isinstance(commodity_group, list):
+            filtered_data = filtered_data[filtered_data['Commodity_Group'].isin(commodity_group)]
+        if scenario and isinstance(scenario, list):
+            filtered_data = filtered_data[filtered_data['Scenario'].isin(scenario)]
+        filtered_data = self.remove_extreme_outliers(df=filtered_data, col='price')
+        return filtered_data
+
+    def update_plot_data(self, region, continent, domain, commodity, commodity_group, scenario):
+        graphic_template='plotly_white'#'plotly_dark'#'plotly_white'
+        filtered_data = self.filter_data(region, continent, domain, commodity, commodity_group, scenario)
+
+        # Quantity plot
+        grouped_data_quantity = filtered_data.groupby(['year', 'Scenario']).sum().reset_index()
+        grouped_data_quantity = grouped_data_quantity[(grouped_data_quantity["year"] >= self.start) & (grouped_data_quantity["year"] <= self.end)]
+        fig_quantity = go.Figure()
+        for i, scenario in enumerate(grouped_data_quantity['Scenario'].unique()):
+            subset = grouped_data_quantity[grouped_data_quantity['Scenario'] == scenario]
+            color = self.color_list[i % len(self.color_list)]
+            dash = 'solid' if scenario in ['Historic Data'] else 'dash'
+            fig_quantity.add_trace(go.Scatter(x=subset['year'], y=subset['quantity'], mode='lines',
+                                              name=f'{scenario}', line=dict(color=color, dash=dash)))
+        title_quantity = self.generate_title(region, continent, domain, commodity, commodity_group)
+        title= "Quantity by Period and Scenario for " + title_quantity
+        fig_quantity.update_layout(
+            title='<br>'.join(textwrap.wrap(title, width=90)),
+            xaxis_title='Year',
+            yaxis_title='Quantity',
+            yaxis=dict(rangemode='nonnegative', zeroline=True, zerolinewidth=2, zerolinecolor='LightGrey'),
+            legend_title='Scenario',
+            legend=dict(orientation="h", yanchor="top", y=-0.1, xanchor="center", x=0.5),
+            margin=dict(l=35, r=35, t=60, b=90),
+            hovermode='x unified',
+            template=graphic_template
+        )
+
+        # Price plot
+        grouped_data_price = filtered_data.groupby(['year', 'Scenario']).mean().reset_index()
+        max_year = grouped_data_price['year'].max() + 0.5
+
+        fig_price = go.Figure()
+        for i, scenario in enumerate(grouped_data_price['Scenario'].unique()):
+            subset = grouped_data_price[grouped_data_price['Scenario'] == scenario]
+            color = self.color_list[i % len(self.color_list)]
+            fig_price.add_trace(go.Bar(x=subset['price'], y=subset['year'], orientation='h',
+                                       name=f'{scenario}', marker_color=color))
+            
+        title_price = f'Price by Period and Scenario'
+        fig_price.update_layout(
+            title=title_price,
+            xaxis_title='Price',
+            yaxis_title='Year',
+            yaxis=dict(range=[2020-0.5, max_year]),
+            legend_title='Scenario',
+            template=graphic_template,
+            showlegend=False,
+            #legend=dict(orientation="h", yanchor="top", y=-0.25, xanchor="center", x=0.5),
+            margin=dict(l=35, r=60, t=50, b=5),
+            barmode='group'
+        )
+
+        # ForStock plot
+        grouped_data_stock = filtered_data.drop(columns=['domain', 'price','quantity','CommodityCode','Commodity','Commodity_Group'])
+        grouped_data_stock = grouped_data_stock.drop_duplicates().reset_index(drop=True)
+        grouped_data_stock = grouped_data_stock.groupby(['year', 'Scenario']).agg({
+            'ForStock': 'sum',
+            }).reset_index()
+        grouped_data_stock = grouped_data_stock[grouped_data_stock.Scenario!='Historic Data']
+        fig_stock = go.Figure()
+        for i, scenario in enumerate(grouped_data_stock['Scenario'].unique()):
+            subset = grouped_data_stock[grouped_data_stock['Scenario'] == scenario]
+            color = self.color_list[i+1 % len(self.color_list)]
+            fig_stock.add_trace(go.Bar(x=subset['year'], y=subset['ForStock'],
+                                    name=f'{scenario}', marker_color=color))
+            
+        min_year = 2020 - 1
+        max_year = grouped_data_stock['year'].max() + 0.5
+
+        min_val = grouped_data_stock['ForStock'].min() * 0.9
+        max_val = grouped_data_stock['ForStock'].max() * 1.1
+
+        fig_stock.update_layout(
+            title='Forest Stock by Year and Scenario',
+            xaxis_title='Year',
+            xaxis=dict(range=[min_year, max_year]),
+            yaxis=dict(range=[min_val, max_val]),
+            yaxis_title='ForStock',
+            template=graphic_template,
+            showlegend=False,
+            #legend=dict(orientation="h", yanchor="top", y=-0.35, xanchor="center", x=0.5),
+            margin=dict(l=50, r=50, t=40, b=5),
+            barmode='group'
+        )
+
+        return fig_quantity, fig_price, fig_stock
+
+    def generate_title(self, region, continent, domain, commodity, commodity_group):
+        title_parts = []
+        if region:
+            title_parts.append(f"{region}")
+        if continent:
+            title_parts.append(f"{continent}")
+        if domain:
+            title_parts.append(f"{domain}")
+        if commodity:
+            title_parts.append(f"{commodity}")
+        if commodity_group:
+            title_parts.append(f"{commodity_group}")
+        title = ", ".join(title_parts) if title_parts else "all data"
+        clean_title = title.replace("'", "").replace("[", "").replace("]", "")
+        return clean_title
+
+    def create_world_map(self, region, continent, domain, commodity, commodity_group, scenario=None, year=None):
+        filtered_data = self.filter_data(region, continent, domain, commodity, commodity_group, scenario)
+        if year:
+            filtered_data = filtered_data[filtered_data['year']==year]
+        country_data = filtered_data.groupby('ISO3')['quantity'].sum().reset_index()
+        country_data = country_data[country_data['quantity']>=0.001].reset_index()
+
+        fig = px.choropleth(
+            country_data,
+            locations="ISO3",
+            color="quantity",
+            hover_name="ISO3",
+            color_continuous_scale="Greens"
+        )
+
+        title = 'Worldmap for '+ self.generate_title(region, continent, domain, commodity, commodity_group)
+        fig.update_layout(
+            #title= '<br>'.join(textwrap.wrap(title, width=43)),
+            geo=dict(
+                showcoastlines=True,
+                coastlinecolor="LightGray",
+                showocean=False,
+                oceancolor="LightBlue",
+                projection_type='natural earth',
+                # Grenzen bestimmen
+                lonaxis_range=[-360, 360],  # Längengradbereich
+                lataxis_range=[-55, 55],   # Breitengradbereich
+            ),
+            margin=dict(l=1, r=1, t=1, b=1),  # Ränder minimieren
+            coloraxis_showscale=False  # Legende entfernen
+        )
+
+        return fig
+    
+    def get_last_historic_year(self):
+        historic_data = self.data[self.data['Scenario'] == 'Historic Data']
+        if not historic_data.empty:
+            return historic_data['year'].max()
+        else:
+            return self.data['year'].max()
+    
+    def remove_extreme_outliers(self,df:pd.DataFrame,col:str,threshhold:float=50):        
+        Q1 = df[col].quantile(0.25)
+        Q3 = df[col].quantile(0.75)
+        IQR = Q3 - Q1
+        outlier_threshold = threshhold * IQR
+        df.loc[df[col] >= outlier_threshold, col] = np.nan
+        return df
+    
+    def open_browser(self):
+        webbrowser.open_new("http://localhost:8050")
+
+
+    def run(self):
+        Timer(1, self.open_browser).start()
+        self.app.run(host='localhost', debug=False, dev_tools_ui=False, dev_tools_hot_reload=False, port=8050)

Toolbox/classes/import_data.py

@@ -0,0 +1,243 @@
+import pickle
+import pandas as pd
+import numpy as np
+from pathlib import Path
+import os
+from enum import Enum
+import pickle
+import gzip
+import Toolbox.parameters.paths as toolbox_paths
+import Toolbox.parameters.default_parameters as toolbox_parameters
+
+class import_pkl_data:
+    def __init__(self, num_files_to_read:int=10,
+                 SCENARIOPATH:Path= toolbox_paths.SCINPUTPATH,
+                 ADDINFOPATH:Path= toolbox_paths.AIINPUTPATH):
+        self.num_files_to_read = num_files_to_read
+        self.SCENARIOPATH = SCENARIOPATH
+        self.ADDINFOPATH = ADDINFOPATH
+
+    def open_pickle(self, src_filepath: str):
+        """open pkl file
+        :param src_filepath: source path for pkl file
+        :return: object from pkl file
+        """
+        import pickle
+        with open(src_filepath, "rb") as pkl_file:
+            obj = pickle.load(pkl_file)
+        return obj
+
+    def read_country_data(self):
+        """read data additional information for country data
+        :return: country data
+        """
+        country_data = pd.read_csv(self.ADDINFOPATH / toolbox_paths.COUNTRYINFO, encoding = "ISO-8859-1")
+        country_data = country_data[["Country-Code", "ContinentNew", "Country","ISO-Code"]]
+        country_data.columns = ["RegionCode","Continent", "Country","ISO3"]
+        country_data.Country = country_data.Country.astype("category")
+        country_data.Continent = country_data.Continent.astype("category")
+        country_data.ISO3 = country_data.ISO3.astype("category")
+        return country_data
+    
+    def read_commodity_data(self):
+        """read data additional information for commodity data
+        :return: commodity data
+        """
+        commodity_data = pd.read_csv(self.ADDINFOPATH / toolbox_paths.COMMODITYINFO , encoding = "ISO-8859-1")
+        commodity_data = commodity_data[["Commodity","CommodityCode","Commodity_Group"]]
+        commodity_data.Commodity = commodity_data.Commodity.astype("category")
+        commodity_data.CommodityCode = commodity_data.CommodityCode.astype("category")
+        commodity_data.Commodity_Group = commodity_data.Commodity_Group.astype("category")
+        return commodity_data
+    
+    def read_historic_data(self):
+        data = pd.read_csv(self.ADDINFOPATH / toolbox_paths.HISTINFO)
+        data = self.downcasting(data)
+        return data
+        
+    def downcasting(self, data: pd.DataFrame):
+        data.RegionCode = data.RegionCode.astype("category")
+        data.CommodityCode = data.CommodityCode.astype("category")
+        data.domain = data.domain.astype("category")
+        data.price = data.price.astype("float32")
+        data.quantity = data.quantity.astype("float32")
+        data.Period = data.Period.astype("int16")
+        data.year = data.year.astype("int16")
+        data.Scenario = data.Scenario.astype("category")
+        data.Model = data.Model.astype("category")
+        return data
+    
+    def add_consumption(self, data):
+        data["quantity"] = (data["quantity_ManufactureCost"] +
+                            data["quantity_Supply"] -
+                            data["quantity_TransportationExport"] +
+                            data["quantity_TransportationImport"])
+        data.loc[data["quantity"] < 0, "quantity"] = 0
+        data["price"] = (((data["quantity_ManufactureCost"] * data["price_ManufactureCost"]) +
+                          (data["quantity_Supply"] * data["price_Supply"]) -
+                          (data["quantity_TransportationExport"] * data["price_TransportationExport"]) +
+                          (data["quantity_TransportationImport"]* data["price_TransportationImport"]))/
+                          data["quantity"])
+        data["price"] = 0
+        data["domain"] = "Consumption"
+        return data
+    
+    def add_net_exports(self, data):
+        data["quantity"] =  (data["quantity_TransportationExport"] -
+                            data["quantity_TransportationImport"])
+        data["price"] =  data["price_TransportationExport"]
+        data["domain"] = "Net Exports"
+        return data
+    
+    def add_net_imports(self, data):
+        data["quantity"] =  (data["quantity_TransportationImport"] -
+                            data["quantity_TransportationExport"])
+        data["price"] = data["price_TransportationImport"]
+        data["domain"] = "Net Imports"
+        return data
+    
+    def add_production(self, data):
+        data["quantity"] =  (data["quantity_ManufactureCost"] + data["quantity_Supply"])
+        data["price"] = (((data["quantity_ManufactureCost"] * data["price_ManufactureCost"]) +
+                          (data["quantity_Supply"] * data["price_Supply"])) / data["quantity"])
+        # if data["price"].mean() <=0:
+        #     data["price"] = 0
+        data["domain"] = "Production"
+        return data
+    
+    def concat_calc_domains(self,origin_data:pd.DataFrame,calc_data:pd.DataFrame):
+        calc_data = calc_data[['RegionCode','CommodityCode','Period','year','domain','price','quantity']].reset_index(drop=True) 
+        result_df = pd.concat([origin_data, calc_data], axis=0).reset_index(drop=True) 
+        return result_df
+
+    def add_calculated_domains(self,data:pd.DataFrame):
+        pivoted_price = data["data_periods"].pivot(index=["RegionCode", "CommodityCode", "Period", "year"], 
+                         columns="domain", 
+                         values="price").add_prefix("price_")
+        pivoted_quantity = data["data_periods"].pivot(index=["RegionCode", "CommodityCode", "Period", "year"], 
+                                    columns="domain", 
+                                    values="quantity").add_prefix("quantity_")
+        pivoted_df = pd.concat([pivoted_price, pivoted_quantity], axis=1).reset_index()    
+
+        calculated_functions = [#"add_consumption",
+                                "add_net_exports",
+                                "add_net_imports",
+                                #"add_production",
+                                ]
+        for method_name in calculated_functions:
+            calc_df = getattr(self, method_name)(data=pivoted_df)
+            data["data_periods"] = self.concat_calc_domains(origin_data=data["data_periods"], calc_data=calc_df)
+
+        return data["data_periods"]
+
+    def concat_scenarios(self, data: pd.DataFrame, sc_name:str, data_prev: pd.DataFrame, ID: int):
+        """concat_scenarios, add scenario name from pkl file to data frames
+        :param data: dictionary of the data container
+        :param sc_name: scenario name from file name in dictionary
+        """    
+        data["data_periods"] = self.add_calculated_domains(data=data)
+        try:
+            for key in data: #loop through all data from datacontainer
+                data[key][toolbox_parameters.column_name_scenario] = sc_name
+                data[key][toolbox_parameters.column_name_model] = toolbox_parameters.model_name
+                #data[key][parameters.column_name_id.value] = ID
+                if data_prev != []:
+                    data[key] = pd.concat([data_prev[key], data[key]], axis=0)
+        except KeyError:
+            pass
+                
+    def combined_data(self):
+        """loop trough all input files in input directory
+        """
+        scenario_path = self.SCENARIOPATH
+        num_files_to_read = self.num_files_to_read
+        pkl_files = [
+            Path(scenario_path) / file
+            for file in os.listdir(scenario_path)
+            if file.endswith(".pkl")
+        ]
+        sorted_files = sorted(pkl_files, key=lambda x: x.stat().st_mtime, reverse=True)
+        newest_files = sorted_files[:num_files_to_read]
+
+        data = []
+        data_prev = []
+        ID = 1
+        for scenario_files in newest_files:
+            src_filepath = scenario_path / scenario_files
+            print(src_filepath)
+            scenario_name = str(scenario_files)[str(scenario_files).rfind(toolbox_parameters.seperator_scenario_name)+3
+                                        :-4]
+            try:
+                with gzip.open(src_filepath,'rb') as f:
+                    if type(f) == gzip.GzipFile:
+                        data = pickle.load(f)
+                        data['data_periods'] = data['data_periods'][['RegionCode','CommodityCode','Period','year','domain','price','quantity']]
+                self.concat_scenarios(data=data, sc_name=scenario_name, data_prev=data_prev, ID=ID)
+            except gzip.BadGzipFile:
+                pass
+            except pickle.UnpicklingError:
+                pass
+            except PermissionError:
+                pass
+            except ValueError:
+                pass
+
+            data_prev = data
+            ID += 1
+        
+        data_prev["data_periods"] = self.downcasting(data_prev["data_periods"])
+        try:
+            data = self.read_historic_data()
+        except FileNotFoundError:
+            data = pd.DataFrame()
+        country_data = self.read_country_data()
+        commodity_data = self.read_commodity_data()
+        forest_data = data_prev['Forest']
+        forest_data = forest_data[['Scenario','RegionCode','Period','ForStock','ForArea']]
+        forest_data = forest_data.drop_duplicates(subset=['Scenario', 'RegionCode', 'Period'], keep='first')
+        data_prev["data_periods"] = pd.merge(data_prev["data_periods"], forest_data, how='left', on=['Scenario','RegionCode','Period'])
+        data_prev["data_periods"] = pd.concat([data_prev["data_periods"], data], axis=0)
+        data_prev["data_periods"] = pd.merge(data_prev["data_periods"], country_data, on="RegionCode", how="left")
+        data_prev["data_periods"] = pd.merge(data_prev["data_periods"], commodity_data, on="CommodityCode", how="left")
+        data_prev["data_periods"]["domain"] = data_prev["data_periods"]["domain"].replace({
+            'ManufactureCost': 'Manufacturing',
+            'TransportationExport': 'Export',
+            'TransportationImport': 'Import',
+            })
+        data_prev["data_periods"] = data_prev["data_periods"][['Model','Scenario','RegionCode','Continent','Country','ISO3',
+                                                               'CommodityCode','Commodity','Commodity_Group','Period','year',
+                                                               'domain','price','quantity',
+                                                               'ForStock','ForArea',
+                                                               ]]
+        return data_prev
+
+    def read_forest_data_gfpm(self, country_data:pd.DataFrame):
+        for_data_gfpm = pd.read_csv(self.ADDINFOPATH / toolbox_paths.FORESTINFO, encoding = "ISO-8859-1")
+        
+        rearranged_for_data = pd.melt(for_data_gfpm, id_vars=['domain','Country'], var_name='Year',value_name='for')
+        rearranged_for_data = rearranged_for_data.dropna()
+        rearranged_for_data['Year'] = rearranged_for_data['Year'].astype(int)
+
+        foreststock = pd.DataFrame()        
+        for domain in rearranged_for_data.domain.unique():
+            rearranged_for_data_domain = rearranged_for_data[rearranged_for_data['domain'] == domain].reset_index(drop=True)
+            if domain == 'ForArea':
+                rearranged_for_data_domain['ForStock'] = foreststock
+            else: 
+                foreststock = rearranged_for_data_domain['for']
+        forest_data = rearranged_for_data_domain[['Country', 'Year', 'for', 'ForStock']]
+        forest_data.columns = ['Country', 'Year', 'ForArea', 'ForStock']
+        forest_data = pd.merge(forest_data, country_data, on= 'Country')
+
+        period_mapping = {2017: 0, 2020: 1, 2025: 2, 2030: 3, 2035: 4, 2040: 5, 2045: 6, 2050: 7, 2055: 8, 2060: 9, 2065: 10}
+        forest_data['Period'] = forest_data['Year'].map(period_mapping)
+
+        forest_gfpm = forest_data[['RegionCode', 'Period', 'ForStock', 'ForArea']]
+        forest_gfpm[toolbox_parameters.column_name_scenario]= 'world500'
+        forest_data['Model'] = 'GFPM'
+        return forest_gfpm
+    
+if __name__ == "__main__":
+    import_pkl = import_pkl_data()
+    data = import_pkl.combined_data()
+    print(data['data_periods'])