mrio-toolbox 1.0.0__py3-none-any.whl → 1.1.2__py3-none-any.whl

extractors/gloria/gloria_extractor.py

@@ -0,0 +1,202 @@
+"""
+Extractor for GLORIA data.
+
+This extractor loads GLORIA raw data files and converts them to NetCDF
+files.
+  
+Supports GLORIA version 059
+https://ielab.info/labs/ielab-gloria 
+
+Created on Fr Dez 20, 2024
+@author: wirth, based on code of beaufils
+
+"""
+
+import os
+import logging
+import numpy as np
+import pandas as pd
+from mrio_toolbox import MRIO
+from mrio_toolbox.utils.savers._to_nc import save_to_nc
+
+log = logging.getLogger(__name__)
+
+def extract_gloria(
+    year, 
+    source,
+    markup = 1,
+    parts = "all", 
+    precision=32):
+    """
+    Extract GLORIA data. 
+
+    Loads GLORIA tables and labels and store them as NetCDF for further use with 
+    the mrio_toolbox library. Currrently, this extractor supports loading T, Y, 
+    VA, Q, and QY tables. 
+    
+    Put all tables (including emission satellite accounts) as well as the 
+    'GLORIA_ReadMe_059a.xlsx' file in the same source folder. 
+    
+
+    Parameters
+    ----------
+    year : str
+        Data year to load.
+    parts : str
+        Data blocks to load:
+            basic : T, Y
+            all : T, Y, VA, Q, QY
+    markup : int
+        Version of prices to load. Available versions: 
+            1 : basic prices
+            2 : trade margins
+            3 : transport margins
+            4 : taxes on products
+            5 : subsidies on products
+    source : path-like
+        Path to folder where raw data is stored
+    precision : int
+        Floating point precision in bits. Default is 32. 
+        This introduces some rounding error for large numbers. 
+    """
+
+    #Check source path
+    source = source + f"/GLORIA_MRIOs_59_{year}"
+    if not os.path.exists(source):
+        log.error(f"{os.path.abspath(source)} does not exist.")
+        raise NotADirectoryError(f"{os.path.abspath(source)} does not exist.")
+
+    # Gloria comes with 164 regions (160 countries + rest of americas, 
+    # rest of europe, rest of africa, rest of asia-pacific) and 120 sectors. 
+
+    if parts == "all":
+        parts = ["T","Y","V","TQ","YQ"]
+    elif parts == "basic":
+        parts = ["T","Y", "V"]
+    
+    tables = {}
+    
+    if precision == 32:
+        log.info("Data precision is 32 bits")
+        dt = np.float32
+    elif precision == 64:
+        log.info("Data precision is 64 bits")
+        dt = np.float64
+    
+    log.info("Loading Gloria labels...")
+    labels = {} 
+    countries = pd.read_excel(
+        io = os.path.join(source, "GLORIA_ReadMe_059a.xlsx"),
+        sheet_name = "Regions")
+    countries = countries["Region_acronyms"].tolist()
+    sectors = pd.read_excel(
+        io = os.path.join(source, "GLORIA_ReadMe_059a.xlsx"),
+        sheet_name = "Sectors")
+    sectors = sectors["Sector_names"].tolist()
+    va_and_y_labs = pd.read_excel(
+        io = os.path.join(source, "GLORIA_ReadMe_059a.xlsx"),
+        sheet_name = "Value added and final demand")
+    va_labs= va_and_y_labs["Value_added_names"].tolist()
+    y_labs = va_and_y_labs["Final_demand_names"].tolist()  
+    q_labs = pd.read_excel(
+        io = os.path.join(source, "GLORIA_ReadMe_059a.xlsx"),
+        sheet_name = "Satellites")
+    q_labs["combined"] = q_labs["Sat_head_indicator"] + " - " + q_labs["Sat_indicator"] + " - " + q_labs["Sat_unit"]
+    q_labs = q_labs["combined"].tolist()
+
+    labels["countries"] = countries
+    labels["sectors"] = sectors
+    labels["y_labs"] = y_labs
+    labels["q_labs"] = q_labs
+    labels["va_labs"] = va_labs
+    log.info("Loaded Gloria labels")
+    
+    log.info("Loading Gloria tables, this can take a while...")
+    for part in parts:
+        if part == "T" or part == "Y":
+            path = os.path.join(source, f'20240111_120secMother_AllCountries_002_{part}-Results_{year}_059_Markup00{markup}(full).csv')
+        elif part == "V":
+            path = os.path.join(source, f'20240419_120secMother_AllCountries_002_{part}-Results_{year}_059_Markup001(full).csv')
+        elif part == "TQ" or part == "YQ":
+            path = os.path.join(source, f'20240417_120secMother_AllCountries_002_{part}-Results_{year}_059_Markup00{markup}(full).csv')
+        log.info(f"Loading {part} table...")
+        tables[part] = load_and_transform_to_IO_structure(path, part, dt)
+        log.info(f"Loaded {part} table")
+        
+    
+    # build an MRIO object from labels and tables
+    m = MRIO()
+    m.add_dimensions(labels)
+    
+    m.parts["T"] = m.new_part(name="t",
+        data= tables["T"],
+        dimensions = [["countries","sectors"],["countries", "sectors"]])
+    log.info("Added T table")
+    
+    m.parts["Y"] = m.new_part(name="y",
+        data= tables["Y"],
+        dimensions = [["countries","sectors"],["countries", "y_labs"]])
+    log.info("Added Y table")
+    
+    m.parts["VA"] = m.new_part(name="va",
+        data= tables["V"],
+        dimensions = ["va_labs",["countries","sectors"]])
+    log.info("Added VA table")
+    
+    if parts == "all": 
+        m.parts["Q"] = m.new_part(name="q",
+            data= tables["TQ"],
+            dimensions = ["q_labs",["countries","sectors"]])
+        log.info("Added Q table")
+        
+        m.parts["QY"] = m.new_part(name="qy",
+            data= tables["YQ"],
+            dimensions = ["q_labs",["countries","y_labs"]])    
+        log.info("Added QY table")
+
+    m.name = f"gloria_{year}_markup00{markup}"
+    return m
+
+def load_and_transform_to_IO_structure(path, part, dt):
+    c = 164 # number of countries
+    s = 120 # number of sectors
+    
+    table = np.loadtxt(path, dtype=dt, delimiter=',')
+    
+    rows = np.arange(table.shape[0])
+    columns = np.arange(table.shape[1])
+    
+    if part == "T": 
+        selected_rows = (rows // s) % 2 == 1 # Starts with 120 off, then 120 on
+        selected_columns = (columns // s) % 2 == 0 # starts with 120 on, then 120 off
+    elif part == "Y":
+        selected_rows = (rows // s) % 2 == 1 
+        selected_columns = columns
+    elif part == "V": 
+        selected_rows = rows
+        selected_columns = (columns // s) % 2 == 0
+    elif part == "TQ":
+        selected_rows = rows
+        selected_columns = (columns // s) % 2 == 0
+    elif part == "YQ":
+        selected_rows = rows
+        selected_columns = columns
+               
+    table = table[selected_rows][:, selected_columns]
+    
+    if part == "V":
+        # Stack the entries to transform the pseudo-diagonalized 984x19680 shape into a 6x19680 shape
+        
+        block_height = 6
+        block_width = 120 
+        blocks = []
+
+        for i in range (0, int(table.shape[0]/block_height)):
+            block = table[i*block_height:(i+1)*block_height,
+                           i*block_width:(i+1)*block_width]
+            blocks.append(block)
+            
+        table = np.hstack(blocks)
+       
+    return table
+

extractors/gtap11/__init__.py

@@ -0,0 +1,7 @@
+"""
+This module contains the extractor for raw GTAP 11 data and the IO builder to transform it into an MRIO object.
+"""
+from .extraction.extractor import extract_gtap11
+from .gtap_mrio import build_io
+
+__all__ = ["extract_gtap11","build_io"]

extractors/gtap11/extraction/__init__.py

@@ -0,0 +1,3 @@
+"""
+This module contains the GTAP 11 extractor. 
+"""

extractors/gtap11/extraction/extractor.py

@@ -0,0 +1,129 @@
+"""
+Extracts the data from the .har file and saves it as .npy files.
+"""
+
+import os
+from mrio_toolbox.extractors.gtap11.extraction.harpy_files.har_file import HarFileObj
+
+import xarray as xr
+import logging
+import mrio_toolbox.extractors.gtap11.gtap_mrio.mrio_builder as mrio_builder
+
+log = logging.getLogger(__name__)
+
+def extract_gtap11(
+        year = 2017,
+        source = "",
+        files = "all",
+        name = "gtap",
+        build_io = True,
+        save_raw = True
+):
+    """
+    Extract GTAP data from .har files and save it as .nc file.
+
+    Parameters
+    ----------
+    source : str, optional
+        Location of the source files, by default the current directory
+    save_raw : str, optional
+        Where to save the raw files, by default the current directory
+    files : list, optional
+        List of files to extract, by default, all files in the source directory
+    name : str, optional
+        Name under which the files are saved, by default "gtap"
+    build_io : bool, optional
+        Whether to build the input-output table, by default False
+
+    Raises
+    ------
+    NotADirectoryError
+        Exception raised when the source directory does not exist
+    FileNotFoundError
+        Exception raised when the destination directory does not contain any .har files
+        If only some files are missing, a warning is issued but the extraction continues
+    """
+    #Check source path
+    if not os.path.exists(source):
+        log.error(f"{os.path.abspath(source)} does not exist.")
+        raise NotADirectoryError(f"{os.path.abspath(source)} does not exist.")
+
+    log.info(f"Extracting GTAP data from {os.path.abspath(source)}")
+    
+    #List available har files
+    har_files = [f for f in os.listdir(source) if f.endswith(".har")]
+    if len(har_files) == 0:
+        log.error(f"No .har files found in {os.path.abspath(source)}")
+        raise FileNotFoundError(f"No .har files found in {os.path.abspath(source)}")
+    log.info(f"Found {len(har_files)} .har files in {os.path.abspath(source)}")
+
+    if isinstance(files, str) and files == "all":
+        files = har_files
+    
+    ds = xr.Dataset()
+    for file in files:
+        if file not in har_files:
+            log.warning(f"{file} not found in {os.path.abspath(source)}")
+            continue
+        log.info(f" Extracting {file}")
+        filename = os.path.join(source, file)
+        data = HarFileObj(filename = filename)
+        variables = data.getHeaderArrayNames()
+        for variable in variables:
+            log.info(f"     Extracting {variable}")
+            ds = convert_variable(data, variable, ds)
+
+    if save_raw:
+        log.info(f"Saving {name}_raw.nc")
+        if save_raw is True:
+            #By default, save in the source folder
+            save_raw = source
+        ds.to_netcdf(os.path.join(save_raw, f"{name}_raw.nc"))
+    if build_io:
+        log.info("Building input-output table")
+        mrio = mrio_builder.build_io(ds)
+        mrio.name = f"{name}_{year}"
+        return mrio
+
+def convert_variable(file, variable, ds):
+    """
+    Convert a variable from a .har file to a xarray DataArray.
+
+    Data descriptor variables are stored as attributes of the dataset.
+
+    Parameters
+    ----------
+    file : harpy.HarFileObj
+        Representation of the .har file
+    variable : str
+        Name of the variable to extract
+    ds : xarray.Dataset
+        Dataset to which the variable is added
+
+    Returns
+    -------
+    ds : xarray.Dataset
+        Updated dataset
+    """
+    data = file[variable]
+    coords = dict()
+    dims = []
+    for dim in data.sets.dims:
+        if dim.name is None:
+            #Intercepts descriptive variables
+            log.info(f"     {variable} is a descriptive variable")
+            ds.attrs[variable] = str(data.array)
+            return ds
+        dims.append(dim.name)
+        coords[dim.name] = dim.dim_desc
+    ds[variable] = xr.DataArray(
+        data.array,
+        coords = coords,
+        dims = dims,
+        attrs = {
+            "long_name": data.long_name,
+            "name" : variable
+        }
+    )
+    return ds 
+    

extractors/gtap11/extraction/harpy_files/__init__.py

@@ -0,0 +1,6 @@
+"""
+The code in this module is copied from the HARPY package, which you can visit on github:
+https://github.com/GEMPACKsoftware/HARPY/tree/master
+
+Both the HARPY library and the MRIO toolbox library are licenced under the GPL 3 license.
+"""

extractors/gtap11/extraction/harpy_files/_header_sets.py

@@ -0,0 +1,279 @@
+"""
+Created on Jun 29 14:46:48 2018
+
+"""
+
+import numpy as np
+from typing import List, Union
+
+class _HeaderSet:
+    """
+    This class is used to represent sets associated with header arrays.
+    """
+
+#    Status is unknown elements but set, element index, known set elements, no set just numeric
+    _valid_status = ["u", "e", "k", "n"]
+    _genSetID     = 0
+
+    def __init__(self, name: 'Union[str,None]',
+                 status: str,
+                 dim_desc: 'Union[List[str],str,None]',
+                 dim_size: int):
+
+        self.name = name
+        self.status = status
+        self.dim_desc = dim_desc
+        if not dim_desc is None:
+            if any([len(el) > 12 for el in dim_desc]):
+                raise ValueError("Set Element too long (maximum 12 Characters for set Elements)")
+        self.elemPosDict={} if self.dim_desc is None else dict(zip( [elem.strip().lower() for elem in dim_desc], range(0,len(self.dim_desc))))
+        self.dim_size = dim_size
+
+    def transform_index(self,index):
+        if isinstance(index,(str,int)):
+            return self.name_to_ind(index), None
+
+        elif isinstance(index,slice):
+            newslice=self.convertSlice(index)
+            npIndList=list(range(self.dim_size))[newslice]
+            SetName=self._newname() if not all(p is None for p in [newslice.start,newslice.stop,newslice.step]) else self.name
+            if self.dim_desc:
+                return npIndList, _HeaderSet(SetName, self.status, self.dim_desc[newslice], len(npIndList))
+            else:
+                return npIndList, _HeaderSet(SetName, self.status, dim_desc=None, dim_size=len(npIndList))
+
+
+        elif isinstance(index,list):
+            useElem=self.status in ["e","k"]
+            setElList=[] if useElem else None
+            npIndList=[]
+            for ind in index:
+                if isinstance(ind, (str,int) ):
+                    idx=self.name_to_ind(ind)
+                    npIndList.append(idx)
+                    if useElem: setElList.append(self.dim_desc[idx])
+                elif isinstance(ind,slice):
+                    newslice = self.convertSlice(ind)
+                    npIndList.append(list(range(self.dim_size))[newslice])
+                    if useElem: setElList.extend(self.dim_desc[newslice])
+                else:
+                    raise TypeError("Only slice, str, int allowed in list indexing")
+            if useElem:
+                if len(set(setElList)) != len(setElList):
+                    raise ValueError("Indexing leads to duplicate set elements which is not permitted")
+                if setElList != self.dim_desc:
+                    return npIndList, _HeaderSet(self._newname(), self.status, setElList, len(npIndList))
+                else:
+                    return npIndList, self
+            else:
+                return npIndList, _HeaderSet(self._newname(), self.status, None, len(npIndList))
+
+
+    def convertSlice(self,index):
+        if not isinstance(index.step, int) and not index.step is None:
+            raise ValueError("step in slice has to be integer")
+        start=self.name_to_ind(index.start)
+        start= None if start==0 else start
+        stop = self.name_to_ind(index.stop)
+        stop =  None if stop==self.dim_size else stop
+        step= None if index.step == 1 else index.step
+        return slice(start, stop, step)
+
+    def name_to_ind(self,idx):
+        if idx is None:
+            return None
+        elif isinstance(idx,str):
+            if idx.strip().lower() in self.elemPosDict:
+                return self.elemPosDict[idx.strip().lower()]
+            else:
+                raise ValueError("Element not in set")
+        elif isinstance(idx,int):
+            if idx >= self.dim_size:
+                raise ValueError("Index Out Of bounds")
+            return idx
+
+    def _newname(self):
+        _HeaderSet._genSetID+=1
+        return "S@"+str(_HeaderSet._genSetID)
+
+
+class _HeaderDims:
+
+    def __init__(self, setList):
+        self._dims=setList
+
+    @staticmethod
+    def fromShape(shape):
+        setList=[_HeaderSet(None, 'n', None, dim) for dim in shape]
+        return _HeaderDims(setList)
+
+    @staticmethod
+    def fromSetShape(sets, setElDict, shape):
+        setObjList=[]
+        lowerDict=dict(zip([key.strip().lower() for key in setElDict.keys()], setElDict.keys() ))
+        for idim, setName in enumerate(sets):
+            lowSet=setName.strip().lower()
+            if lowSet in lowerDict:
+                setObjList.append(_HeaderSet(setName,'k',setElDict[lowerDict[lowSet]],shape[idim]))
+            else:
+                setObjList.append(_HeaderSet(setName, 'u', None, shape[idim]))
+        return _HeaderDims(setObjList)
+
+
+    @property
+    def dims(self) -> List[_HeaderSet]:
+        return self._dims
+
+    @dims.setter
+    def dims(self, obj) -> None:
+        self._dims = obj
+
+    def ndim(self):
+        """
+        Number of dimensions
+        """
+        return len(self._dims)
+
+    def defined(self):
+        """
+        Tells whether dimensensions have sets defined or are just array like
+        """
+        return not all([dim.name is None for dim in self._dims])
+
+    @property
+    def setNames(self):
+        return [dim.name for dim in self.dims]
+    
+    @setNames.setter
+    def setNames(self, sNames):
+        if not isinstance(sNames,list): raise TypeError("set Names needs to be given as a list of strings")
+        if len(sNames) != len(self.dims) : raise ValueError("wrong length of set List. Header is rank "+str(len(self.dims))+ "but received list size "+ len(sNames))
+        for name in sNames:
+            if not isinstance(name,str): raise TypeError("set Names contains a non string object: "+str(name))
+            if len(name.strip()) > 12 : raise ValueError("Set names are limited to 12 characters. received '"+name+"'")
+        for newName, dim in zip(sNames,self.dims):
+            dim.name=newName.strip()
+
+    @property
+    def setElements(self):
+        return [dim.dim_desc for dim in self.dims]
+
+    @property
+    def shape(self):
+        return tuple([sets.dim_size for sets in self._dims])
+
+    def __str__(self):
+        outputstr=""
+        for setDim in self._dims:
+            if setDim.status in "keu":
+                outputstr+="   " + setDim.name.ljust(12) + ": \n"
+            else:
+                outputstr+="   "+"Not Specified"
+            if setDim.status in "ke":
+                outputstr+="      " +", ".join(setDim.dim_desc) + "\n"
+        return outputstr
+
+
+
+    def compatible_shape(self,other):
+        return self.shape == other
+
+    def matchSets(self,sets=None, shape:tuple=None):
+        if sets is None and shape is None : raise KeyError("Only one argument allowed")
+        newSets = []
+        if not sets is None:
+            # Try to match the shape of the dimensions
+            iset=len(self.dims)-1; jset=len(sets.dims)-1
+            while iset >=0 and jset >=0:
+                if jset < 0 :
+                    newSets.append(self.dims[iset])
+                    iset -=1
+                elif iset < 0 :
+                    newSets.append(sets.dims[jset])
+                    jset -=1
+                if self.dims[iset].dim_size == sets.dims[jset].dim_size or self.dims[iset].dim_size == 1 or sets.dims[jset].dim_size == 1:
+                    if self.dims[iset].status != 'n':
+                        newSets.append(self.dims[iset])
+                    else:
+                        newSets.append(sets.dims[jset])
+                    iset-= 1 ; jset -=1
+            newSets.reverse()
+        elif not shape is None:
+            iset = len(self.dims) - 1; jset=len(shape)-1
+            while iset >=0 and jset >=0:
+                if jset < 0 :
+                    newSets.append(self.dims[iset])
+                    iset -=1
+                elif iset < 0 :
+                    newSets.append(_HeaderSet(None , 'n' , None, shape[jset]))
+                    jset -=1
+                if self.dims[iset].dim_size == shape[jset] or self.dims[iset].dim_size == 1 or shape[jset] == 1:
+                    newSets.append(self.dims[iset])
+                    iset-= 1 ; jset -=1
+            newSets.reverse()
+        else:
+            return KeyError("Either sets o shape have to be defined")
+
+        return _HeaderDims(newSets)
+
+
+    def transform_index(self,index_tuple):
+        if not isinstance(index_tuple,tuple):
+            index_tuple=(index_tuple,)
+
+        trueLen=len([x for x in index_tuple if x is not None])
+        if trueLen != self.ndim() and not Ellipsis in index_tuple:
+            raise ValueError("Rank mismatch in indexing")
+        if index_tuple.count(Ellipsis)>1:
+            raise ValueError("Only single Ellipsis (...) allowed in indexing")
+
+        thisIndex=[]
+        for ind in index_tuple:
+            if ind == Ellipsis:
+                for i in range(0,self.ndim()-trueLen+1):
+                    thisIndex.append(slice(None,None,None))
+            elif isinstance(ind,(list,str,int,slice)) or ind is None:
+                thisIndex.append(ind)
+            else:
+                raise TypeError("Only ...,list,str,int,slice and None allowed as indices")
+
+        npIndex=[]
+        newSets=[]
+
+        iset=0
+        for index in thisIndex:
+            if index is None:
+                npInd=np.newaxis
+                newSet=_HeaderSet(None , 'n' , None, 1)
+            else:
+                setDim=self._dims[iset]
+                npInd, newSet = setDim.transform_index(index)
+                iset+=1
+            npIndex.append(npInd)
+            newSets.append(newSet)
+
+        rankIndex=tuple([slice(None) if isinstance(ind,list) or ind is None else 0 for ind in npIndex])
+        newSets = [setDim for ri, setDim in zip(rankIndex,newSets) if ri != 0]
+        return self._makeNPIndex(npIndex), rankIndex, _HeaderDims(newSets)
+
+
+    @staticmethod
+    def _makeNPIndex(indexList):
+        newinds = []
+        for i, item in enumerate(indexList):
+            if isinstance(item, list):
+                newinds.append(item)
+            elif isinstance(item,int):
+                newinds.append([item])
+
+        numpyInd = list(np.ix_(*newinds))
+        newinds=[]
+        for item in indexList:
+            if not item is None:
+                newinds.append(numpyInd.pop(0))
+            else:
+                newinds.append(None)
+
+        return tuple(newinds)
+
+