mrio-toolbox 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

mrio_toolbox/extractors/gloria/gloria_extractor.py

@@ -1,202 +0,0 @@
-"""
-Extractor for GLORIA data.
-
-This extractor loads GLORIA raw data files and converts them to NetCDF
-files.
-  
-Supports GLORIA version 059
-https://ielab.info/labs/ielab-gloria 
-
-Created on Fr Dez 20, 2024
-@author: wirth, based on code of beaufils
-
-"""
-
-import os
-import logging
-import numpy as np
-import pandas as pd
-from mrio_toolbox import MRIO
-from mrio_toolbox.utils.savers._to_nc import save_to_nc
-
-log = logging.getLogger(__name__)
-
-def extract_gloria(
-    year, 
-    source,
-    markup = 1,
-    parts = "all", 
-    precision=32):
-    """
-    Extract GLORIA data. 
-
-    Loads GLORIA tables and labels and store them as NetCDF for further use with 
-    the mrio_toolbox library. Currrently, this extractor supports loading T, Y, 
-    VA, Q, and QY tables. 
-    
-    Put all tables (including emission satellite accounts) as well as the 
-    'GLORIA_ReadMe_059a.xlsx' file in the same source folder. 
-    
-
-    Parameters
-    ----------
-    year : str
-        Data year to load.
-    parts : str
-        Data blocks to load:
-            basic : T, Y
-            all : T, Y, VA, Q, QY
-    markup : int
-        Version of prices to load. Available versions: 
-            1 : basic prices
-            2 : trade margins
-            3 : transport margins
-            4 : taxes on products
-            5 : subsidies on products
-    source : path-like
-        Path to folder where raw data is stored
-    precision : int
-        Floating point precision in bits. Default is 32. 
-        This introduces some rounding error for large numbers. 
-    """
-
-    #Check source path
-    source = source + f"/GLORIA_MRIOs_59_{year}"
-    if not os.path.exists(source):
-        log.error(f"{os.path.abspath(source)} does not exist.")
-        raise NotADirectoryError(f"{os.path.abspath(source)} does not exist.")
-
-    # Gloria comes with 164 regions (160 countries + rest of americas, 
-    # rest of europe, rest of africa, rest of asia-pacific) and 120 sectors. 
-
-    if parts == "all":
-        parts = ["T","Y","V","TQ","YQ"]
-    elif parts == "basic":
-        parts = ["T","Y", "V"]
-    
-    tables = {}
-    
-    if precision == 32:
-        log.info("Data precision is 32 bits")
-        dt = np.float32
-    elif precision == 64:
-        log.info("Data precision is 64 bits")
-        dt = np.float64
-    
-    log.info("Loading Gloria labels...")
-    labels = {} 
-    countries = pd.read_excel(
-        io = os.path.join(source, "GLORIA_ReadMe_059a.xlsx"),
-        sheet_name = "Regions")
-    countries = countries["Region_acronyms"].tolist()
-    sectors = pd.read_excel(
-        io = os.path.join(source, "GLORIA_ReadMe_059a.xlsx"),
-        sheet_name = "Sectors")
-    sectors = sectors["Sector_names"].tolist()
-    va_and_y_labs = pd.read_excel(
-        io = os.path.join(source, "GLORIA_ReadMe_059a.xlsx"),
-        sheet_name = "Value added and final demand")
-    va_labs= va_and_y_labs["Value_added_names"].tolist()
-    y_labs = va_and_y_labs["Final_demand_names"].tolist()  
-    q_labs = pd.read_excel(
-        io = os.path.join(source, "GLORIA_ReadMe_059a.xlsx"),
-        sheet_name = "Satellites")
-    q_labs["combined"] = q_labs["Sat_head_indicator"] + " - " + q_labs["Sat_indicator"] + " - " + q_labs["Sat_unit"]
-    q_labs = q_labs["combined"].tolist()
-
-    labels["countries"] = countries
-    labels["sectors"] = sectors
-    labels["y_labs"] = y_labs
-    labels["q_labs"] = q_labs
-    labels["va_labs"] = va_labs
-    log.info("Loaded Gloria labels")
-    
-    log.info("Loading Gloria tables, this can take a while...")
-    for part in parts:
-        if part == "T" or part == "Y":
-            path = os.path.join(source, f'20240111_120secMother_AllCountries_002_{part}-Results_{year}_059_Markup00{markup}(full).csv')
-        elif part == "V":
-            path = os.path.join(source, f'20240419_120secMother_AllCountries_002_{part}-Results_{year}_059_Markup001(full).csv')
-        elif part == "TQ" or part == "YQ":
-            path = os.path.join(source, f'20240417_120secMother_AllCountries_002_{part}-Results_{year}_059_Markup00{markup}(full).csv')
-        log.info(f"Loading {part} table...")
-        tables[part] = load_and_transform_to_IO_structure(path, part, dt)
-        log.info(f"Loaded {part} table")
-        
-    
-    # build an MRIO object from labels and tables
-    m = MRIO()
-    m.add_dimensions(labels)
-    
-    m.parts["T"] = m.new_part(name="t",
-        data= tables["T"],
-        dimensions = [["countries","sectors"],["countries", "sectors"]])
-    log.info("Added T table")
-    
-    m.parts["Y"] = m.new_part(name="y",
-        data= tables["Y"],
-        dimensions = [["countries","sectors"],["countries", "y_labs"]])
-    log.info("Added Y table")
-    
-    m.parts["VA"] = m.new_part(name="va",
-        data= tables["V"],
-        dimensions = ["va_labs",["countries","sectors"]])
-    log.info("Added VA table")
-    
-    if parts == "all": 
-        m.parts["Q"] = m.new_part(name="q",
-            data= tables["TQ"],
-            dimensions = ["q_labs",["countries","sectors"]])
-        log.info("Added Q table")
-        
-        m.parts["QY"] = m.new_part(name="qy",
-            data= tables["YQ"],
-            dimensions = ["q_labs",["countries","y_labs"]])    
-        log.info("Added QY table")
-
-    m.name = f"gloria_{year}_markup00{markup}"
-    return m
-
-def load_and_transform_to_IO_structure(path, part, dt):
-    c = 164 # number of countries
-    s = 120 # number of sectors
-    
-    table = np.loadtxt(path, dtype=dt, delimiter=',')
-    
-    rows = np.arange(table.shape[0])
-    columns = np.arange(table.shape[1])
-    
-    if part == "T": 
-        selected_rows = (rows // s) % 2 == 1 # Starts with 120 off, then 120 on
-        selected_columns = (columns // s) % 2 == 0 # starts with 120 on, then 120 off
-    elif part == "Y":
-        selected_rows = (rows // s) % 2 == 1 
-        selected_columns = columns
-    elif part == "V": 
-        selected_rows = rows
-        selected_columns = (columns // s) % 2 == 0
-    elif part == "TQ":
-        selected_rows = rows
-        selected_columns = (columns // s) % 2 == 0
-    elif part == "YQ":
-        selected_rows = rows
-        selected_columns = columns
-               
-    table = table[selected_rows][:, selected_columns]
-    
-    if part == "V":
-        # Stack the entries to transform the pseudo-diagonalized 984x19680 shape into a 6x19680 shape
-        
-        block_height = 6
-        block_width = 120 
-        blocks = []
-
-        for i in range (0, int(table.shape[0]/block_height)):
-            block = table[i*block_height:(i+1)*block_height,
-                           i*block_width:(i+1)*block_width]
-            blocks.append(block)
-            
-        table = np.hstack(blocks)
-       
-    return table
-

mrio_toolbox/extractors/gtap11/__init__.py

@@ -1,7 +0,0 @@
-"""
-This module contains the extractor for raw GTAP 11 data and the IO builder to transform it into an MRIO object.
-"""
-from .extraction.extractor import extract_gtap11
-from .gtap_mrio import build_io
-
-__all__ = ["extract_gtap11","build_io"]

mrio_toolbox/extractors/gtap11/extraction/__init__.py

@@ -1,3 +0,0 @@
-"""
-This module contains the GTAP 11 extractor. 
-"""

mrio_toolbox/extractors/gtap11/extraction/extractor.py

@@ -1,129 +0,0 @@
-"""
-Extracts the data from the .har file and saves it as .npy files.
-"""
-
-import os
-from mrio_toolbox.extractors.gtap11.extraction.harpy_files.har_file import HarFileObj
-
-import xarray as xr
-import logging
-import mrio_toolbox.extractors.gtap11.gtap_mrio.mrio_builder as mrio_builder
-
-log = logging.getLogger(__name__)
-
-def extract_gtap11(
-        year = 2017,
-        source = "",
-        files = "all",
-        name = "gtap",
-        build_io = True,
-        save_raw = True
-):
-    """
-    Extract GTAP data from .har files and save it as .nc file.
-
-    Parameters
-    ----------
-    source : str, optional
-        Location of the source files, by default the current directory
-    save_raw : str, optional
-        Where to save the raw files, by default the current directory
-    files : list, optional
-        List of files to extract, by default, all files in the source directory
-    name : str, optional
-        Name under which the files are saved, by default "gtap"
-    build_io : bool, optional
-        Whether to build the input-output table, by default False
-
-    Raises
-    ------
-    NotADirectoryError
-        Exception raised when the source directory does not exist
-    FileNotFoundError
-        Exception raised when the destination directory does not contain any .har files
-        If only some files are missing, a warning is issued but the extraction continues
-    """
-    #Check source path
-    if not os.path.exists(source):
-        log.error(f"{os.path.abspath(source)} does not exist.")
-        raise NotADirectoryError(f"{os.path.abspath(source)} does not exist.")
-
-    log.info(f"Extracting GTAP data from {os.path.abspath(source)}")
-    
-    #List available har files
-    har_files = [f for f in os.listdir(source) if f.endswith(".har")]
-    if len(har_files) == 0:
-        log.error(f"No .har files found in {os.path.abspath(source)}")
-        raise FileNotFoundError(f"No .har files found in {os.path.abspath(source)}")
-    log.info(f"Found {len(har_files)} .har files in {os.path.abspath(source)}")
-
-    if isinstance(files, str) and files == "all":
-        files = har_files
-    
-    ds = xr.Dataset()
-    for file in files:
-        if file not in har_files:
-            log.warning(f"{file} not found in {os.path.abspath(source)}")
-            continue
-        log.info(f" Extracting {file}")
-        filename = os.path.join(source, file)
-        data = HarFileObj(filename = filename)
-        variables = data.getHeaderArrayNames()
-        for variable in variables:
-            log.info(f"     Extracting {variable}")
-            ds = convert_variable(data, variable, ds)
-
-    if save_raw:
-        log.info(f"Saving {name}_raw.nc")
-        if save_raw is True:
-            #By default, save in the source folder
-            save_raw = source
-        ds.to_netcdf(os.path.join(save_raw, f"{name}_raw.nc"))
-    if build_io:
-        log.info("Building input-output table")
-        mrio = mrio_builder.build_io(ds)
-        mrio.name = f"{name}_{year}"
-        return mrio
-
-def convert_variable(file, variable, ds):
-    """
-    Convert a variable from a .har file to a xarray DataArray.
-
-    Data descriptor variables are stored as attributes of the dataset.
-
-    Parameters
-    ----------
-    file : harpy.HarFileObj
-        Representation of the .har file
-    variable : str
-        Name of the variable to extract
-    ds : xarray.Dataset
-        Dataset to which the variable is added
-
-    Returns
-    -------
-    ds : xarray.Dataset
-        Updated dataset
-    """
-    data = file[variable]
-    coords = dict()
-    dims = []
-    for dim in data.sets.dims:
-        if dim.name is None:
-            #Intercepts descriptive variables
-            log.info(f"     {variable} is a descriptive variable")
-            ds.attrs[variable] = str(data.array)
-            return ds
-        dims.append(dim.name)
-        coords[dim.name] = dim.dim_desc
-    ds[variable] = xr.DataArray(
-        data.array,
-        coords = coords,
-        dims = dims,
-        attrs = {
-            "long_name": data.long_name,
-            "name" : variable
-        }
-    )
-    return ds 
-    

mrio_toolbox/extractors/gtap11/extraction/harpy_files/__init__.py

@@ -1,6 +0,0 @@
-"""
-The code in this module is copied from the HARPY package, which you can visit on github:
-https://github.com/GEMPACKsoftware/HARPY/tree/master
-
-Both the HARPY library and the MRIO toolbox library are licenced under the GPL 3 license.
-"""

mrio_toolbox/extractors/gtap11/extraction/harpy_files/_header_sets.py

@@ -1,279 +0,0 @@
-"""
-Created on Jun 29 14:46:48 2018
-
-"""
-
-import numpy as np
-from typing import List, Union
-
-class _HeaderSet:
-    """
-    This class is used to represent sets associated with header arrays.
-    """
-
-#    Status is unknown elements but set, element index, known set elements, no set just numeric
-    _valid_status = ["u", "e", "k", "n"]
-    _genSetID     = 0
-
-    def __init__(self, name: 'Union[str,None]',
-                 status: str,
-                 dim_desc: 'Union[List[str],str,None]',
-                 dim_size: int):
-
-        self.name = name
-        self.status = status
-        self.dim_desc = dim_desc
-        if not dim_desc is None:
-            if any([len(el) > 12 for el in dim_desc]):
-                raise ValueError("Set Element too long (maximum 12 Characters for set Elements)")
-        self.elemPosDict={} if self.dim_desc is None else dict(zip( [elem.strip().lower() for elem in dim_desc], range(0,len(self.dim_desc))))
-        self.dim_size = dim_size
-
-    def transform_index(self,index):
-        if isinstance(index,(str,int)):
-            return self.name_to_ind(index), None
-
-        elif isinstance(index,slice):
-            newslice=self.convertSlice(index)
-            npIndList=list(range(self.dim_size))[newslice]
-            SetName=self._newname() if not all(p is None for p in [newslice.start,newslice.stop,newslice.step]) else self.name
-            if self.dim_desc:
-                return npIndList, _HeaderSet(SetName, self.status, self.dim_desc[newslice], len(npIndList))
-            else:
-                return npIndList, _HeaderSet(SetName, self.status, dim_desc=None, dim_size=len(npIndList))
-
-
-        elif isinstance(index,list):
-            useElem=self.status in ["e","k"]
-            setElList=[] if useElem else None
-            npIndList=[]
-            for ind in index:
-                if isinstance(ind, (str,int) ):
-                    idx=self.name_to_ind(ind)
-                    npIndList.append(idx)
-                    if useElem: setElList.append(self.dim_desc[idx])
-                elif isinstance(ind,slice):
-                    newslice = self.convertSlice(ind)
-                    npIndList.append(list(range(self.dim_size))[newslice])
-                    if useElem: setElList.extend(self.dim_desc[newslice])
-                else:
-                    raise TypeError("Only slice, str, int allowed in list indexing")
-            if useElem:
-                if len(set(setElList)) != len(setElList):
-                    raise ValueError("Indexing leads to duplicate set elements which is not permitted")
-                if setElList != self.dim_desc:
-                    return npIndList, _HeaderSet(self._newname(), self.status, setElList, len(npIndList))
-                else:
-                    return npIndList, self
-            else:
-                return npIndList, _HeaderSet(self._newname(), self.status, None, len(npIndList))
-
-
-    def convertSlice(self,index):
-        if not isinstance(index.step, int) and not index.step is None:
-            raise ValueError("step in slice has to be integer")
-        start=self.name_to_ind(index.start)
-        start= None if start==0 else start
-        stop = self.name_to_ind(index.stop)
-        stop =  None if stop==self.dim_size else stop
-        step= None if index.step == 1 else index.step
-        return slice(start, stop, step)
-
-    def name_to_ind(self,idx):
-        if idx is None:
-            return None
-        elif isinstance(idx,str):
-            if idx.strip().lower() in self.elemPosDict:
-                return self.elemPosDict[idx.strip().lower()]
-            else:
-                raise ValueError("Element not in set")
-        elif isinstance(idx,int):
-            if idx >= self.dim_size:
-                raise ValueError("Index Out Of bounds")
-            return idx
-
-    def _newname(self):
-        _HeaderSet._genSetID+=1
-        return "S@"+str(_HeaderSet._genSetID)
-
-
-class _HeaderDims:
-
-    def __init__(self, setList):
-        self._dims=setList
-
-    @staticmethod
-    def fromShape(shape):
-        setList=[_HeaderSet(None, 'n', None, dim) for dim in shape]
-        return _HeaderDims(setList)
-
-    @staticmethod
-    def fromSetShape(sets, setElDict, shape):
-        setObjList=[]
-        lowerDict=dict(zip([key.strip().lower() for key in setElDict.keys()], setElDict.keys() ))
-        for idim, setName in enumerate(sets):
-            lowSet=setName.strip().lower()
-            if lowSet in lowerDict:
-                setObjList.append(_HeaderSet(setName,'k',setElDict[lowerDict[lowSet]],shape[idim]))
-            else:
-                setObjList.append(_HeaderSet(setName, 'u', None, shape[idim]))
-        return _HeaderDims(setObjList)
-
-
-    @property
-    def dims(self) -> List[_HeaderSet]:
-        return self._dims
-
-    @dims.setter
-    def dims(self, obj) -> None:
-        self._dims = obj
-
-    def ndim(self):
-        """
-        Number of dimensions
-        """
-        return len(self._dims)
-
-    def defined(self):
-        """
-        Tells whether dimensensions have sets defined or are just array like
-        """
-        return not all([dim.name is None for dim in self._dims])
-
-    @property
-    def setNames(self):
-        return [dim.name for dim in self.dims]
-    
-    @setNames.setter
-    def setNames(self, sNames):
-        if not isinstance(sNames,list): raise TypeError("set Names needs to be given as a list of strings")
-        if len(sNames) != len(self.dims) : raise ValueError("wrong length of set List. Header is rank "+str(len(self.dims))+ "but received list size "+ len(sNames))
-        for name in sNames:
-            if not isinstance(name,str): raise TypeError("set Names contains a non string object: "+str(name))
-            if len(name.strip()) > 12 : raise ValueError("Set names are limited to 12 characters. received '"+name+"'")
-        for newName, dim in zip(sNames,self.dims):
-            dim.name=newName.strip()
-
-    @property
-    def setElements(self):
-        return [dim.dim_desc for dim in self.dims]
-
-    @property
-    def shape(self):
-        return tuple([sets.dim_size for sets in self._dims])
-
-    def __str__(self):
-        outputstr=""
-        for setDim in self._dims:
-            if setDim.status in "keu":
-                outputstr+="   " + setDim.name.ljust(12) + ": \n"
-            else:
-                outputstr+="   "+"Not Specified"
-            if setDim.status in "ke":
-                outputstr+="      " +", ".join(setDim.dim_desc) + "\n"
-        return outputstr
-
-
-
-    def compatible_shape(self,other):
-        return self.shape == other
-
-    def matchSets(self,sets=None, shape:tuple=None):
-        if sets is None and shape is None : raise KeyError("Only one argument allowed")
-        newSets = []
-        if not sets is None:
-            # Try to match the shape of the dimensions
-            iset=len(self.dims)-1; jset=len(sets.dims)-1
-            while iset >=0 and jset >=0:
-                if jset < 0 :
-                    newSets.append(self.dims[iset])
-                    iset -=1
-                elif iset < 0 :
-                    newSets.append(sets.dims[jset])
-                    jset -=1
-                if self.dims[iset].dim_size == sets.dims[jset].dim_size or self.dims[iset].dim_size == 1 or sets.dims[jset].dim_size == 1:
-                    if self.dims[iset].status != 'n':
-                        newSets.append(self.dims[iset])
-                    else:
-                        newSets.append(sets.dims[jset])
-                    iset-= 1 ; jset -=1
-            newSets.reverse()
-        elif not shape is None:
-            iset = len(self.dims) - 1; jset=len(shape)-1
-            while iset >=0 and jset >=0:
-                if jset < 0 :
-                    newSets.append(self.dims[iset])
-                    iset -=1
-                elif iset < 0 :
-                    newSets.append(_HeaderSet(None , 'n' , None, shape[jset]))
-                    jset -=1
-                if self.dims[iset].dim_size == shape[jset] or self.dims[iset].dim_size == 1 or shape[jset] == 1:
-                    newSets.append(self.dims[iset])
-                    iset-= 1 ; jset -=1
-            newSets.reverse()
-        else:
-            return KeyError("Either sets o shape have to be defined")
-
-        return _HeaderDims(newSets)
-
-
-    def transform_index(self,index_tuple):
-        if not isinstance(index_tuple,tuple):
-            index_tuple=(index_tuple,)
-
-        trueLen=len([x for x in index_tuple if x is not None])
-        if trueLen != self.ndim() and not Ellipsis in index_tuple:
-            raise ValueError("Rank mismatch in indexing")
-        if index_tuple.count(Ellipsis)>1:
-            raise ValueError("Only single Ellipsis (...) allowed in indexing")
-
-        thisIndex=[]
-        for ind in index_tuple:
-            if ind == Ellipsis:
-                for i in range(0,self.ndim()-trueLen+1):
-                    thisIndex.append(slice(None,None,None))
-            elif isinstance(ind,(list,str,int,slice)) or ind is None:
-                thisIndex.append(ind)
-            else:
-                raise TypeError("Only ...,list,str,int,slice and None allowed as indices")
-
-        npIndex=[]
-        newSets=[]
-
-        iset=0
-        for index in thisIndex:
-            if index is None:
-                npInd=np.newaxis
-                newSet=_HeaderSet(None , 'n' , None, 1)
-            else:
-                setDim=self._dims[iset]
-                npInd, newSet = setDim.transform_index(index)
-                iset+=1
-            npIndex.append(npInd)
-            newSets.append(newSet)
-
-        rankIndex=tuple([slice(None) if isinstance(ind,list) or ind is None else 0 for ind in npIndex])
-        newSets = [setDim for ri, setDim in zip(rankIndex,newSets) if ri != 0]
-        return self._makeNPIndex(npIndex), rankIndex, _HeaderDims(newSets)
-
-
-    @staticmethod
-    def _makeNPIndex(indexList):
-        newinds = []
-        for i, item in enumerate(indexList):
-            if isinstance(item, list):
-                newinds.append(item)
-            elif isinstance(item,int):
-                newinds.append([item])
-
-        numpyInd = list(np.ix_(*newinds))
-        newinds=[]
-        for item in indexList:
-            if not item is None:
-                newinds.append(numpyInd.pop(0))
-            else:
-                newinds.append(None)
-
-        return tuple(newinds)
-
-