mrio-toolbox 1.1.2__py3-none-any.whl → 1.1.3__py3-none-any.whl

utils/loaders/_loader.py

@@ -1,312 +0,0 @@
-"""
-Central loading module for the mrio_toolbox package.
-
-This module contains the central loading function for the mrio_toolbox package.
-Depending on the loading mode, the function will call the appropriate loader.
-"""
-
-import os
-import logging
-import yaml
-
-log = logging.Logger(__name__)
-
-class Loader:
-    """
-    Parent class for loaders in the MRIO toolbox.
-
-    The `Loader` class provides a base implementation for loading MRIO data. 
-    It includes methods for extracting metadata, updating settings, and managing 
-    groupings and labels. Specific loaders can inherit from this class to implement 
-    format-specific loading functionality.
-
-    Instance variables
-    ------------------
-    metadata : dict
-        Metadata associated with the loader.
-    labels : dict
-        Labels for the axes of the MRIO data.
-    groupings : dict
-        Groupings for the labels, defining higher-level aggregations.
-    file : str or None
-        Path to the file being loaded.
-    loader_kwargs : dict
-        Additional parameters for the loader.
-
-    Methods
-    -------
-    extract_basic_info(**kwargs):
-        Extract basic information such as path, labels, and groupings.
-    update_settings(**settings):
-        Update the loader settings with new parameters.
-    load_mrio():
-        Create an MRIO container based on the current parameters.
-    load_part(**kwargs):
-        Load an MRIO Part based on new or existing parameters.
-    set_groupings(groupings):
-        Update the groupings attribute of the loader.
-    update_attributes(**kwargs):
-        Update the current attributes of the loader.
-    load_groupings(file, dimension=None, path=None):
-        Load groupings from a file.
-    set_labels(labels):
-        Update the labels attribute of the loader.
-    available_parts(**kwargs):
-        Return the available parts in the MRIO data.
-    check_instructions(**kwargs):
-        Interpret the file argument for loading a part and check for instruction consistency.
-
-    Notes
-    -----
-    This class is intended to be used as a base class for specific loaders. 
-    It provides general functionality for managing metadata, labels, and groupings, 
-    but does not implement actual data loading.
-    """
-    def __init__(
-            self
-            ):
-        """
-        Initialize a Loader object.
-        
-        Notes
-        -----
-        Loaders are created with format-specific parameters. They hold metadata and methods to load MRIO data.
-        A loader is created using the base class if no specific loader is required,
-        i.e., if the data is directly loaded from dict, pandas or xarray.
-        In that case, the loader will fail when used,
-        triggering the creation of a specific loader.
-        """
-        self.load_mrio()
-
-    def extract_basic_info(self,**kwargs):
-        """
-        Extract basic information from the loader.
-
-        The function will extract the path, labels and groupings from the loader.
-        """
-        self.loader_kwargs = kwargs.pop("loader_kwargs",dict())
-        self.file = kwargs.get("file",None)
-        self.groupings = kwargs.get("groupings",dict())
-        self.labels = kwargs.get("labels",dict())
-        #Remaining kwargs are metadata
-        self.metadata = kwargs
-        if isinstance(self.groupings,str):
-            self.groupings = self.load_groupings(self.groupings)
-
-    def update_settings(self,**settings):
-        """
-        Update the loader settings with new parameters
-        """
-        self.loader_kwargs.update(
-            settings.pop("loader_kwargs",dict())
-        )
-        self.groupings.update(
-            settings.pop("groupings",dict())
-        )
-        self.labels.update(
-            settings.pop("labels",dict())
-        )
-        self.metadata.update(
-            settings.pop("metadata",dict())
-        )
-        self.metadata.update(settings)
-
-
-    def load_mrio(
-            self
-    ):
-        """
-        Create an MRIO container based on the new parameters
-
-        Returns
-        -------
-        dict
-            Dictionary of MRIO metadata
-        """
-        self.metadata = dict()
-        self.labels = dict()
-        self.groupings = dict()
-        self.file = None
-        pass
-
-    def load_part(
-            self,
-            **kwargs
-    ):
-        """
-        Load an MRIO Part based on new or existing parameters
-
-        Returns
-        -------
-        dict
-            Dictionary containing the Part data
-        """
-        raise FileNotFoundError("No proper loader was initialised.\n"+\
-        "The loader needs to be reloaded with new instructions.")
-
-    def set_groupings(self,groupings):
-        """
-        Update the groupings attribute of the loader
-
-        Parameters
-        ----------
-        groupings : dict of dict of str
-            Aggregation on labels
-        """
-        self.groupings = groupings
-    
-    def update_attributes(self,**kwargs):
-        """
-        Update the current attributes of the loader.
-
-        The function will update the groupings, paths, labels and metadata attributes.
-        """
-        if "groupings" in kwargs:
-            log.debug("Update groupings")
-            self.groupings = kwargs.pop("groupings",self.groupings)
-            
-        self.extract_path(update=True,**kwargs)
-
-        if "labels" in kwargs:
-            log.debug("Update labels")
-            self.format_labels(kwargs.pop("labels"))
-
-        for kwarg in kwargs:
-            log.debug(f"Override parameter {kwarg} with explicit parameter {kwargs[kwarg]}")
-            self.metadata[kwarg] = kwargs[kwarg]
-            
-    def load_groupings(self,
-                       file,
-                       dimension=None,
-                       path=None):
-        """Load groupings from a file
-        
-        Parameters
-        ---------- 
-        file : str
-            Name of the file to load
-        dimension : str, optional
-            Name of the dimension to load groupings for.
-            By default (None), the file is interpreted as a preset
-            of groupings on different dimension.
-        path : path-like, optional
-            Path where the file is stored. 
-            By default, the groupings are from the settings dir
-            in the working dir.
-        """
-        def _check_groupings(groupings,dimension):
-            """Check whether the groupings are consistent with the labels"""
-            for key in groupings.keys():
-                for item in groupings[key]:
-                    if item not in self.labels[dimension]:
-                        log.warning(
-                            f"Item {item} not found in {dimension} labels"
-                            )
-                        groupings[key].remove(item)
-                if len(groupings[key])==0:
-                    log.warning(f"Group {key} is empty")
-                    groupings.pop(key)
-            return groupings
-    
-        def load_grouping(file,level,path):
-            """Load a single grouping file"""
-            path = os.path.join(path,level)
-            with open(os.path.join(path,file+'.txt')) as f:
-                group = f.read().splitlines()
-            return {file:group}
-        
-        if path is None:
-            path = os.path.join("parameters","groupings")
-
-        #If no dimension is specified, interpret as a preset
-        output = dict()
-        if isinstance(file,str):
-            log.info("Load groupings set from "+path+file)
-            with open(os.path.join(path,file)) as f:
-                groupings = yaml.safe_load(f)
-        elif isinstance(file,dict):
-            groupings = file
-        output = self.groupings
-        
-        if dimension is None:
-            dimensions = list(groupings.keys())
-            output = dict()
-        for level in dimensions:
-            if isinstance(groupings[level],dict):
-                #Case the preset explicitly defines a grouping
-                groupings[level] = _check_groupings(
-                    groupings[level],level
-                    )
-                output[level] = groupings[level]
-                continue
-            if isinstance(groupings[level],str):
-                groupings[level] = [groupings[level]]
-            if isinstance(groupings[level],list):
-                #Otherwise, interpret as a list of groupings
-                output[level] = dict()
-                covered = []
-                for item in groupings[level]:
-                    #Load all groupings
-                    groups= load_grouping(
-                        item,level,path
-                    )
-                    if any([group in covered for group in groups]):
-                        duplicate = [
-                            group for group in groups if group in covered
-                            ]
-                        log.warning("The following items are covered in "+\
-                                    "multiple groupings: "+duplicate)
-                    covered += groups
-                    output[level][item] = groups
-        return output
-    
-    def set_labels(self,labels):
-        """
-        Update the labels attribute of the loader
-
-        Parameters
-        ----------
-        labels : dict of str:list of str
-            Labels of the axes
-        """
-        self.labels = labels
-
-    def available_parts(self,**kwargs):
-        """
-        Return the available parts in the MRIO data
-        """
-        if self.file is None:
-            raise FileNotFoundError("No file was provided.")
-
-    def check_instructions(self,**kwargs):
-        """
-        Interpret the file argument for loading a part.
-
-        This method solves the ambiguity between data files and optional
-        .yaml instructions.
-        If the file argument refers to an instruction file, it is compared
-        to the current instructions.
-        If the data file or instruction file differ from the ones currently loaded,
-        an exception is raised to force a reload.
-
-        Parameters
-        ----------
-        file : path-like
-            User-provided file path
-        kwargs : additional arguments
-
-        Raises
-        ------
-        FileNotFoundError
-            If the loader needs to be reloaded with new instructions.
-        
-        """
-        #The 'instructions' attribute is used to check if the loader needs to be reloaded
-        #It contains the reference to the potential yaml file used to load the data
-        new_instructions = kwargs.get("instructions",None)
-        ref_instructions = self.metadata.get("instructions",None)
-        if new_instructions is not None and ref_instructions != new_instructions:
-            #If the instructions differ from the current ones,
-            #trigger a reload of the loader
-            log.error("The loader needs to be reloaded with new instructions.")
-            raise FileNotFoundError("The loader needs to be reloaded with new instructions.")

utils/loaders/_loader_factory.py

@@ -1,96 +0,0 @@
-"""
-Initialize the appropriate loader based on the provided parameters.
-"""
-import os
-import yaml
-from mrio_toolbox.utils.loaders._nc_loader import NetCDF_Loader
-from mrio_toolbox.utils.loaders._parameter_loader import Parameter_Loader
-from mrio_toolbox.utils.loaders._pandas_loader import Pandas_Loader
-from mrio_toolbox.utils.loaders._loader import Loader
-import logging
-
-log = logging.getLogger(__name__)
-
-def make_loader(**kwargs):
-    """
-    Initialize the appropriate loader based on the provided parameters.
-
-    If a file or data_file is provided, 
-    the function will attempt to determine the appropriate loader based on the file extension.
-
-    Namely:
-    - .nc files are loaded using the NetCDF_Loader
-    - .yaml files are interpreted as loading instructions
-    
-    All non-netCDF files are loaded using the Parameter_Loader.
-    """
-    file = kwargs.get("file",None)
-    if file is not None:
-        file = os.path.abspath(file) # Avoid issue with UNIX/windows path
-    extension = kwargs.get("extension",None)
-
-    if extension is None:
-        if file is None:
-            log.info("No file or extension provided.")
-            log.info("An empty loader will be created.")
-            return Loader()
-        extension = os.path.splitext(file)[1]
-        if extension == "":
-            log.error("File extension missing.")
-            raise ValueError("File extension missing.")
-        
-    if extension == "":
-        log.error("File extension missing.")
-        raise ValueError("File extension missing.")
-    if extension == ".nc":
-        return NetCDF_Loader(**kwargs)
-    if extension in [".yaml",".yml"]:
-        return load_from_yaml(**kwargs)
-    if extension in [".npy",".txt"]:
-        return Parameter_Loader(**kwargs)
-    if extension in [".csv"]:
-        if "loader_kwargs" in kwargs:
-            pandas = kwargs["loader_kwargs"].pop(
-                "pandas",False
-                )
-            if pandas:
-                return Pandas_Loader(**kwargs)
-        return Parameter_Loader(**kwargs)
-    if extension == ".xlsx":
-        return Pandas_Loader(**kwargs)
-    log.error(f"File extension {extension} not supported.")
-
-def load_from_yaml(**kwargs):	
-    """
-    Create a loader based on yaml file instructions.
-
-    Parameters
-    ----------
-    file : path-like
-        Full path to the .yaml file
-    """
-    instructions = kwargs.pop("file")
-    log.info("Get loading instructions from: "+instructions)
-    with open(instructions) as f: 
-        parameters = yaml.safe_load(f)
-    for kwarg in kwargs:
-        #Override parameters with kwargs
-        log.debug(f"Override file parameter {kwarg} with explicit parameter {kwargs[kwarg]}")
-        parameters[kwarg] = kwargs[kwarg]
-
-    # Error handling
-    if "path" not in parameters.keys(): 
-        if "file" not in parameters.keys(): 
-            log.info("No path provided, using current working directory instead")
-            parameters["path"] = os.getcwd()
-    elif not os.path.isdir(parameters["path"]):
-        log.error("Provided path is not a directory")
-        raise ValueError("Provided path is not a directory")
-    
-
-    return make_loader(instructions=instructions,**parameters)
-
-
-    
-
-    

utils/loaders/_nc_loader.py

@@ -1,184 +0,0 @@
-"""
-Provides the NetCDF_Loader class for loading MRIO data from netCDF files.
-"""
-from mrio_toolbox.utils.loaders._loader import Loader
-from mrio_toolbox.utils import converters
-import xarray as xr
-
-import logging
-import pandas as pd
-
-log = logging.getLogger(__name__)
-
-class NetCDF_Loader(Loader):
-    """
-    Class for loading MRIO data from a netCDF file.
-
-    The `NetCDF_Loader` class extends the base `Loader` class to provide 
-    functionality for loading MRIO data stored in netCDF format. It uses the 
-    xarray library to load the data and extract metadata, labels, and groupings.
-
-    Instance variables
-    ------------------
-    data : xarray.Dataset
-        The loaded netCDF data stored as an xarray Dataset.
-    _available_parts : list
-        List of available parts in the MRIO data.
-    metadata : dict
-        Metadata extracted from the netCDF file.
-    labels : dict
-        Labels for the axes of the MRIO data.
-    groupings : dict
-        Groupings for the labels, defining higher-level aggregations.
-    file : str or None
-        Path to the netCDF file being loaded.
-    loader_kwargs : dict
-        Additional parameters passed to the xarray loader.
-
-    Methods
-    -------
-    load_mrio(file=None, **kwargs):
-        Load a netCDF file into memory and extract metadata.
-    load_part(file=None, **kwargs):
-        Load a specific part of the MRIO table.
-    get_file(file=None, **kwargs):
-        Get the file to load, updating the current file if necessary.
-    available_parts(**kwargs):
-        Return a list of available parts in the MRIO table.
-    """
-    
-    def __init__(
-            self,
-            **kwargs
-            ):
-        """
-        Initialize a NetCDF_Loader object.
-
-        Parameters
-        ----------
-        loader_kwargs : dict, optional
-            Parameters passed to the xarray loader.
-        file : path-like
-            Full path to the netCDF file.
-        groupings : dict, optional
-            Aggregation on labels
-        **kwargs : dict
-            Metadata for the MRIO data.
-            MRIO metadata are passed to associated parts.
-
-        """
-        self.extract_basic_info(**kwargs)
-        super().__init__()
-        self.update_settings(**kwargs)
-        
-    def load_mrio(
-            self,
-            file = None,
-            **kwargs
-    ):
-        """
-        Load a netcdf file in the memory.
-
-        This procedure is based on the xarray library.
-        The xarray dataset is stored in the data attribute.
-        The loader also extracts all metadata from the file.
-
-        Parameters
-        ----------
-        file : path-like, optional
-            Full path to the file.
-            If left empty, the file currently initialised is used.
-
-        Raises
-        ------
-        ValueError
-            If the file is not provided.
-        """
-        
-        if file is None:
-            file = self.file
-
-        if file is None:
-            raise ValueError("No file provided.")
-        
-        log.info(f"Load MRIO data from {file}")
-        self.data = xr.open_dataset(file, **self.loader_kwargs)
-        mrio_data,list_of_parts = converters.xarray.make_mrio(self.data)
-        self._available_parts = list_of_parts
-        self.update_settings(**mrio_data["data"])
-
-
-    def load_part(
-            self,
-            file = None,
-            **kwargs
-    ):
-        """
-        Load a part of the MRIO table.
-
-        Parameters
-        ----------
-        name : str
-            Name of the variable to load
-        file : path, optional
-            Full path to the data.
-            If left empty, the current xarray Dataset is used.
-
-        Returns
-        -------
-        dict
-            Data required to create a Part object
-        """
-        self.get_file(file,**kwargs) #Update the file if needed
-        return converters.xarray.make_part(
-            self.data,**kwargs
-            )
-    
-    def get_file(self, file=None, **kwargs):
-        """
-        Get the file to load.
-
-        Parameters
-        ----------
-        file : path-like, optional
-            User-defined path to the file, by default None
-
-        Returns
-        -------
-        path-like
-            Path to the file to load from
-
-        Raises
-        ------
-        ValueError
-            If no file is provided nor currently loaded
-        
-        """
-        self.check_instructions(**kwargs)
-        #Check if new instructions are provided
-
-        if file is None and self.file is None:
-            raise ValueError("No file provided.")
-        
-        instructions = self.metadata.get("instructions",None)
-
-        if file != self.file and file != instructions:
-            #If the file is different from the one currently loaded, the current data is replaced
-            self.load_mrio(file)
-
-        return file
-
-    def available_parts(
-            self,**kwargs
-    ):
-        """
-        Return a list of available parts in the MRIO table.
-
-        Returns
-        -------
-        list
-            List of available parts
-        """
-        return self._available_parts
-
-    

utils/loaders/_np_loader.py

@@ -1,112 +0,0 @@
-"""
-Routine for loading MRIO Parts from .npy and .csv files
-"""
-
-import os
-import numpy as np
-import pandas as pd
-import logging
-import yaml
-
-log = logging.getLogger(__name__)
-
-def load_file(file,extension=None,pandas=False,**kwargs):
-    """
-    Load data from a .npy, .txt, .xlsx or .csv file.
-
-    Parameters
-    ----------
-    file : path-like
-        Full path to the file
-    kwargs : dict
-        Additional parameters for the loaders
-
-    Returns
-    -------
-    data : np.array
-        Numerical data
-    
-    Raises
-    ------
-    FileNotFoundError
-        If the file is not found in the specified path
-    ValueError
-        If the file extension is not supported
-    """
-    if extension is None:
-        extension = os.path.splitext(file)[1]
-    elif os.path.splitext(file)[1] == "":
-        file = file+extension
-    elif os.path.splitext(file)[1] != extension:
-        raise FileNotFoundError(f"File {file} does not match the provided extension {extension}.")
-    if extension == "":
-        log.info("No extension provided. Trying .npy, .csv and .txt.")
-        for loader in [load_npy,load_csv,load_txt,load_xlsx]:
-            try:
-                return loader(file)
-            except FileNotFoundError:
-                pass
-        log.error(f"File {file} not found with extensions .npy, .csv or .txt.")
-        raise FileNotFoundError(f"File {file} not found in the specified path.")
-    if extension not in [".npy",".csv",".txt",".xlsx",".yaml"]:
-        log.error(f"File extension {extension} not supported.")
-        raise ValueError(f"File extension {extension} not supported.\nSupported extensions: .npy, .csv, .txt")
-    if extension == ".npy":
-        return load_npy(file,**kwargs)
-    if extension == ".csv":
-        return load_csv(file,pandas=pandas,**kwargs)
-    if extension == ".txt":
-        return load_txt(file,**kwargs)
-    if extension == ".xlsx":
-        return load_xlsx(file,**kwargs)
-    if extension == ".yaml":
-        return load_yaml(file,**kwargs)
-    
-def load_yaml(file,**kwargs):
-    if os.path.splitext(file)[1] == "":
-        file = file+".yaml"
-    with open(file,"r") as f:
-        return yaml.safe_load(f)
-
-def load_npy(file,**kwargs):
-    if os.path.splitext(file)[1] == "":
-        file = file+".npy"
-    return np.load(file,**kwargs)
-
-def load_csv(file,pandas=False,**kwargs):
-    """
-    Read a .csv file using pandas or numpy.
-
-    If pandas, the file is read using pandas,
-    such that labels are automatically extracted.
-    Otherwise, the file is read using numpy and labels are loaded from another file.
-    """
-    if os.path.splitext(file)[1] == "":
-        file = file+".csv"
-    delimiter = kwargs.get("delimiter",",")
-    if pandas:
-        #Remove header if not provided
-        #This is to avoid issues with the label autodetection
-        kwargs["header"] = kwargs.get("header",None)
-        return pd.read_csv(file,
-                           **kwargs)
-    return np.loadtxt(file,delimiter=delimiter,**kwargs)
-
-def load_txt(file,**kwargs):
-    if os.path.splitext(file)[1] == "":
-        file = file+".txt"
-    delimiter = kwargs.get("delimiter","\t")
-    try:
-        return np.loadtxt(file,delimiter=delimiter,**kwargs)
-    except ValueError:
-        #If the basic loading fails, it's probably a label file
-        return np.loadtxt(file,dtype=str,delimiter=delimiter,**kwargs).tolist()
-
-def load_xlsx(file, **kwargs):
-    if os.path.splitext(file)[1] == "":
-        file = file+".xlsx"
-    #Remove header if not provided
-    #This is to avoid issues with the label autodetection
-    kwargs["header"] = kwargs.get("header",None)
-    return pd.read_excel(file,
-                         **kwargs)