mrio-toolbox 1.0.0__py3-none-any.whl

mrio_toolbox/utils/loaders/_loader_factory.py

@@ -0,0 +1,75 @@
+import os
+import yaml
+from mrio_toolbox.utils.loaders._nc_loader import NetCDF_Loader
+from mrio_toolbox.utils.loaders._parameter_loader import Parameter_Loader
+from mrio_toolbox.utils.loaders._pandas_loader import Pandas_Loader
+from mrio_toolbox.utils.loaders._loader import Loader
+import logging
+
+log = logging.getLogger(__name__)
+
+def make_loader(**kwargs):
+    """
+    Initialize the appropriate loader based on the provided parameters.
+
+    If a file or data_file is provided, 
+    the function will attempt to determine the appropriate loader based on the file extension.
+
+    Namely:
+    - .nc files are loaded using the NetCDF_Loader
+    - .yaml files are interpreted as loading instructions
+    
+    All non-netCDF files are loaded using the Parameter_Loader.
+    """
+    file = kwargs.get("file",None)
+    extension = kwargs.get("extension",None)
+
+    if extension is None:
+        if file is None:
+            log.info("No file or extension provided.")
+            log.info("An empty loader will be created.")
+            return Loader()
+        extension = os.path.splitext(file)[1]
+        if extension == "":
+            log.error("File extension missing.")
+            raise ValueError("File extension missing.")
+        
+    if extension == "":
+        log.error("File extension missing.")
+        raise ValueError("File extension missing.")
+    if extension == ".nc":
+        return NetCDF_Loader(**kwargs)
+    if extension in [".yaml",".yml"]:
+        return load_from_yaml(**kwargs)
+    if extension in [".npy",".txt"]:
+        return Parameter_Loader(**kwargs)
+    if extension in [".csv"]:
+        if "loader_kwargs" in kwargs:
+            pandas = kwargs["loader_kwargs"].pop(
+                "pandas",False
+                )
+            if pandas:
+                return Pandas_Loader(**kwargs)
+        return Parameter_Loader(**kwargs)
+    if extension == ".xlsx":
+        return Pandas_Loader(**kwargs)
+    log.error(f"File extension {extension} not supported.")
+
+def load_from_yaml(**kwargs):	
+    """
+    Create a loader based on yaml file instructions.
+
+    Parameters
+    ----------
+    file : path-like
+        Full path to the .yaml file
+    """
+    instructions = kwargs.pop("file")
+    log.info("Get loading instructions from: "+instructions)
+    with open(instructions) as f: 
+        parameters = yaml.safe_load(f)
+    for kwarg in kwargs:
+        #Override parameters with kwargs
+        log.debug(f"Override file parameter {kwarg} with explicit parameter {kwargs[kwarg]}")
+        parameters[kwarg] = kwargs[kwarg]
+    return make_loader(instructions=instructions,**parameters)

mrio_toolbox/utils/loaders/_nc_loader.py

@@ -0,0 +1,148 @@
+from mrio_toolbox.utils.loaders._loader import Loader
+from mrio_toolbox.utils import converters
+import xarray as xr
+
+import logging
+import pandas as pd
+
+log = logging.getLogger(__name__)
+
+class NetCDF_Loader(Loader):
+    """
+    Class for loading MRIO data from a netCDF file.
+    """
+    def __init__(
+            self,
+            **kwargs
+            ):
+        """
+        Loader for MRIO data in netCDF format.
+
+        Parameters
+        ----------
+        loader_kwargs : dict, optional
+            Parameters passed to the xarray loader.
+        file : path-like
+            Full path to the netCDF file.
+        groupings : dict, optional
+            Aggregation on labels
+        **kwargs : dict
+            Metadata for the MRIO data.
+            MRIO metadata are passed to associated parts.
+
+        """
+        self.extract_basic_info(**kwargs)
+        super().__init__()
+        self.update_settings(**kwargs)
+        
+    def load_mrio(
+            self,
+            file = None,
+            **kwargs
+    ):
+        """
+        Load a netcdf file in the memory.
+
+        This procedure is based on the xarray library.
+        The xarray dataset is stored in the data attribute.
+        The loader also extracts all metadata from the file.
+
+        Parameters
+        ----------
+        file : path-like, optional
+            Full path to the file.
+            If left empty, the file currently initialised is used.
+
+        Raises
+        ------
+        ValueError
+            If the file is not provided.
+        """
+        
+        if file is None:
+            file = self.file
+
+        if file is None:
+            raise ValueError("No file provided.")
+        
+        log.info(f"Load MRIO data from {file}")
+        self.data = xr.open_dataset(file, **self.loader_kwargs)
+        mrio_data,list_of_parts = converters.xarray.make_mrio(self.data)
+        self._available_parts = list_of_parts
+        self.update_settings(**mrio_data["data"])
+
+
+    def load_part(
+            self,
+            file = None,
+            **kwargs
+    ):
+        """
+        Load a part of the MRIO table.
+
+        Parameters
+        ----------
+        name : str
+            Name of the variable to load
+        file : path, optional
+            Full path to the data.
+            If left empty, the current xarray Dataset is used.
+
+        Returns
+        -------
+        dict
+            Data required to create a Part object
+        """
+        self.get_file(file,**kwargs) #Update the file if needed
+        return converters.xarray.make_part(
+            self.data,**kwargs
+            )
+    
+    def get_file(self, file=None, **kwargs):
+        """
+        Get the file to load.
+
+        Parameters
+        ----------
+        file : path-like, optional
+            User-defined path to the file, by default None
+
+        Returns
+        -------
+        path-like
+            Path to the file to load from
+
+        Raises
+        ------
+        ValueError
+            If no file is provided nor currently loaded
+        
+        """
+        self.check_instructions(**kwargs)
+        #Check if new instructions are provided
+
+        if file is None and self.file is None:
+            raise ValueError("No file provided.")
+        
+        instructions = self.metadata.get("instructions",None)
+
+        if file != self.file and file != instructions:
+            #If the file is different from the one currently loaded, the current data is replaced
+            self.load_mrio(file)
+
+        return file
+
+    def available_parts(
+            self,**kwargs
+    ):
+        """
+        Return a list of available parts in the MRIO table.
+
+        Returns
+        -------
+        list
+            List of available parts
+        """
+        return self._available_parts
+
+    

mrio_toolbox/utils/loaders/_np_loader.py

@@ -0,0 +1,112 @@
+"""
+Routine for loading MRIO Parts from .npy and .csv files
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import logging
+import yaml
+
+log = logging.getLogger(__name__)
+
+def load_file(file,extension=None,pandas=False,**kwargs):
+    """
+    Load data from a .npy, .txt, .xlsx or .csv file.
+
+    Parameters
+    ----------
+    file : path-like
+        Full path to the file
+    kwargs : dict
+        Additional parameters for the loaders
+
+    Returns
+    -------
+    data : np.array
+        Numerical data
+    
+    Raises
+    ------
+    FileNotFoundError
+        If the file is not found in the specified path
+    ValueError
+        If the file extension is not supported
+    """
+    if extension is None:
+        extension = os.path.splitext(file)[1]
+    elif os.path.splitext(file)[1] == "":
+        file = file+extension
+    elif os.path.splitext(file)[1] != extension:
+        raise FileNotFoundError(f"File {file} does not match the provided extension {extension}.")
+    if extension == "":
+        log.info("No extension provided. Trying .npy, .csv and .txt.")
+        for loader in [load_npy,load_csv,load_txt,load_xlsx]:
+            try:
+                return loader(file)
+            except FileNotFoundError:
+                pass
+        log.error(f"File {file} not found with extensions .npy, .csv or .txt.")
+        raise FileNotFoundError(f"File {file} not found in the specified path.")
+    if extension not in [".npy",".csv",".txt",".xlsx",".yaml"]:
+        log.error(f"File extension {extension} not supported.")
+        raise ValueError(f"File extension {extension} not supported.\nSupported extensions: .npy, .csv, .txt")
+    if extension == ".npy":
+        return load_npy(file,**kwargs)
+    if extension == ".csv":
+        return load_csv(file,pandas=pandas,**kwargs)
+    if extension == ".txt":
+        return load_txt(file,**kwargs)
+    if extension == ".xlsx":
+        return load_xlsx(file,**kwargs)
+    if extension == ".yaml":
+        return load_yaml(file,**kwargs)
+    
+def load_yaml(file,**kwargs):
+    if os.path.splitext(file)[1] == "":
+        file = file+".yaml"
+    with open(file,"r") as f:
+        return yaml.safe_load(f)
+
+def load_npy(file,**kwargs):
+    if os.path.splitext(file)[1] == "":
+        file = file+".npy"
+    return np.load(file,**kwargs)
+
+def load_csv(file,pandas=False,**kwargs):
+    """
+    Read a .csv file using pandas or numpy.
+
+    If pandas, the file is read using pandas,
+    such that labels are automatically extracted.
+    Otherwise, the file is read using numpy and labels are loaded from another file.
+    """
+    if os.path.splitext(file)[1] == "":
+        file = file+".csv"
+    delimiter = kwargs.get("delimiter",",")
+    if pandas:
+        #Remove header if not provided
+        #This is to avoid issues with the label autodetection
+        kwargs["header"] = kwargs.get("header",None)
+        return pd.read_csv(file,
+                           **kwargs)
+    return np.loadtxt(file,delimiter=delimiter,**kwargs)
+
+def load_txt(file,**kwargs):
+    if os.path.splitext(file)[1] == "":
+        file = file+".txt"
+    delimiter = kwargs.get("delimiter","\t")
+    try:
+        return np.loadtxt(file,delimiter=delimiter,**kwargs)
+    except ValueError:
+        #If the basic loading fails, it's probably a label file
+        return np.loadtxt(file,dtype=str,delimiter=delimiter,**kwargs).tolist()
+
+def load_xlsx(file, **kwargs):
+    if os.path.splitext(file)[1] == "":
+        file = file+".xlsx"
+    #Remove header if not provided
+    #This is to avoid issues with the label autodetection
+    kwargs["header"] = kwargs.get("header",None)
+    return pd.read_excel(file,
+                         **kwargs)

mrio_toolbox/utils/loaders/_pandas_loader.py

@@ -0,0 +1,102 @@
+"""Routines for loading from Excel"""
+
+from mrio_toolbox.utils.loaders._np_loader import load_file
+from mrio_toolbox.utils.loaders._parameter_loader import Parameter_Loader
+from mrio_toolbox.utils import converters
+import os
+import logging
+
+log = logging.getLogger(__name__)
+
+class Pandas_Loader(Parameter_Loader):
+    """
+    Class for loading MRIO data through Pandas.
+    """
+    def __init__(
+            self,
+            **kwargs
+            ):
+        """
+        Loader for MRIO data through pandas.
+
+        Used for loading data from .xlsx and .csv files.
+
+        Parameters
+        ----------
+        loader_kwargs : dict, optional
+            Parameters passed to the underlying loader.
+               - .xlsx: pandas.read_excel
+               - .csv: pandas.read_csv
+        groupings : dict, optional
+            Aggregation on labels
+        labels : dict, optional
+            Explicit dictionary of labels.
+        dimensions : list of int, optional
+            List of label names.
+        path : str, optional
+            Path to the data
+            The following paths are recognized:
+            - path
+            - mrio_path
+            - file
+            - data_path
+            - table/year/version
+        labels_path : str, optional
+            Path to the labels files
+        parts : dict, optional
+            Parts to load, with specific settings
+        **kwargs : dict
+            Metadata for the MRIO data.
+            MRIO metadata are passed to associated parts.
+
+        """
+        super().__init__(**kwargs)
+
+    def load_part(
+            self,
+            **kwargs
+    ):
+        """
+        Load a Part from explicit parameters.
+
+        Parameters provided as arguments overload the corresponding instance attributes.
+
+        Returns
+        -------
+        dict
+            Data for creating the Part object
+
+        Raises
+        ------
+        FileNotFoundError
+            If no file nor name argument is provided
+        """
+        #Update loader parameters
+        self.update_attributes(**kwargs)
+
+        file = self.get_file(**kwargs)
+        
+        loader_kwargs = kwargs.pop("loader_kwargs",self.loader_kwargs)
+
+        name = kwargs.pop("name",os.path.splitext(os.path.basename(file))[0])
+
+        log.info(f"Load part {name} from {file}")
+        
+        if name in self.part_settings:
+            #Load preset settings
+            part_settings = self.part_settings[name]
+            kwargs.update(part_settings)
+
+        autodetect_labels = True
+        if any(key in loader_kwargs for key in ["index_col", "header"]):
+            #If labels are explicitly provided, do not autodetect
+            autodetect_labels = False
+
+        
+        return converters.pandas.make_part(load_file(file,
+                                     **loader_kwargs,
+                                     extension=self.extension,
+                                     pandas=True),
+                                     name=name,
+                                     label_detection=autodetect_labels,
+                                     **kwargs)