mrio-toolbox 1.0.0__py3-none-any.whl

mrio_toolbox/utils/loaders/_parameter_loader.py

@@ -0,0 +1,341 @@
+"""
+Routine for loading MRIO tables from explicit parameters
+"""
+
+import os
+from mrio_toolbox.utils.loaders._loader import Loader
+from mrio_toolbox.utils.loaders._np_loader import load_file
+import pathlib
+import logging
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+class Parameter_Loader(Loader):
+    """
+    Class for loading MRIO data from explicit parameters.
+    """
+    def __init__(
+            self,
+            **kwargs
+            ):
+        """
+        Loader for MRIO data in non-netCDF formats.
+
+        Parameters
+        ----------
+        loader_kwargs : dict, optional
+            Parameters passed to the underlying loader.
+               - .npy: numpy.load
+               - .csv, .txt: numpy.loadtxt
+        groupings : dict, optional
+            Aggregation on labels
+        labels : dict, optional
+            Explicit dictionary of labels.
+        dimensions : list of int, optional
+            List of label names.
+        path : str, optional
+            Path to the data
+            The following paths are recognized:
+            - path
+            - mrio_path
+            - file
+            - data_path
+            - table/year/version
+        labels_path : str, optional
+            Path to the labels files
+        parts : dict, optional
+            Parts to load, with specific settings
+        **kwargs : dict
+            Metadata for the MRIO data.
+            MRIO metadata are passed to associated parts.
+
+        """
+        self.extract_basic_info(**kwargs)
+        self.extract_path(update=True,**kwargs)
+        self.labels = dict()
+
+        try:
+            log.debug("Try bulk labels loading.")
+            self.labels = load_file(
+                os.path.join(
+                   self.metadata["path"],"labels.yaml"
+                   )
+                    )
+        except FileNotFoundError:
+            log.debug("No labels found in the path.")
+            labels = kwargs.pop("labels",None)
+            if labels is None:
+                self.metadata["dimensions"] = kwargs.get("dimensions",None)
+                labels = self.metadata["dimensions"]
+            self.labels = dict()
+            self.format_labels(labels)
+
+        try:
+            self.groupings = load_file(
+                os.path.join(
+                    self.metadata["path"],"groupings.yaml"
+                    )
+                    )
+        except FileNotFoundError:
+            self.groupings = kwargs.pop("groupings",dict())
+
+        self.extension = kwargs.get("extension",None)
+        
+        self.part_settings = kwargs.get("parts",dict())
+        super().__init__()
+
+    def available_parts(self,extension=None):
+        """
+        List the available parts in the current path.
+
+        Parameters
+        ----------
+        extension : str, optional
+            Extension of the files to look for.
+            If not provided, all files are listed.
+
+        Returns
+        -------
+        list
+            List of available parts
+        """
+        if extension is None:
+            extension = self.extension
+            
+        if extension is None:
+            return os.listdir(self.path)
+        files = os.listdir(self.path)
+        parts = [
+            Path(file).stem for file in files if file.endswith(extension)
+            ]
+        return parts
+    
+    def extract_path(self,update=False,**kwargs):
+        """
+        Extract the path from the kwargs.
+
+        Valid formats are:
+        - path
+        - mrio_path
+        - file
+        - data_path
+        - table/year/version
+        In absence of explicit path, the current directory is used.
+
+        Parameters
+        ----------
+        update : bool, optional
+            Whether to update the path attribute.
+            If a path is already set, it is not overridden.
+        """
+        if "path" in kwargs:
+            self.path = kwargs.pop("path")
+        elif "mrio_path" in kwargs:
+            self.path = kwargs.pop("mrio_path")
+        elif "data_path" in kwargs and update:
+            self.path = kwargs.pop("data_path")
+        elif "file" in kwargs and update:
+            self.path = pathlib.Path(kwargs.pop("file")).parent
+        elif "path" in self.__dict__.keys() and not update:
+            log.debug("No path provided.")
+            self.path = "."
+
+        if "table" in kwargs and "year" in kwargs and "version" in kwargs:
+            self.path = os.path.join(
+                self.path,
+                kwargs.pop("table"),
+                str(kwargs.pop("year")),
+                kwargs.pop("version"))
+        
+        self.labels_path = kwargs.get("labels_path",
+                                      self.__dict__.get("labels_path",self.path)
+                                      )
+
+        #Store paths in metadata
+        self.metadata["path"] = self.path
+        self.metadata["labels_path"] = self.labels_path
+
+    def format_labels(self,labels):
+        """
+        Treat the label information
+
+        If labels are provided as dict, they are kept as is.
+        If labels are provided as string, they are loaded from the labels_path folder.
+        The labels are stored as a dict of lists.
+        """
+        if labels is None:
+            log.debug("No labels provided.")
+            return
+        if isinstance(labels,dict):
+            self.labels = labels
+        if isinstance(labels,str):
+            labels = [labels]
+            
+        for label in labels:
+            if isinstance(label,list):
+                for sublabel in label:
+                    self.format_labels(sublabel)
+            elif label not in self.labels.keys():
+                log.debug("Load labels: "+label)
+                self.labels[label] = load_file(
+                    os.path.join(self.labels_path,label),dtype=str
+                )
+                
+    def load_mrio(
+            self,
+            **kwargs
+    ):
+        """
+        Load MRIO data from explicit parameters.
+
+        If parameters are provided, they overload the corresponding instance attributes.
+        """
+        self.update_attributes(**kwargs)
+    
+    def get_file(self,**kwargs):
+        """
+        Get the file to load.
+
+        Parameters
+        ----------
+        file : path-like, optional
+            User-defined path to the file, by default None
+
+        Returns
+        -------
+        path-like
+            Path to the file to load from
+
+        Raises
+        ------
+        ValueError
+            If no file is provided nor currently loaded
+        """
+        self.check_instructions(**kwargs)
+        #Check if new instructions are provided
+        
+        instructions = self.metadata.get("instructions",None)
+
+        #Find file
+        if "file" in kwargs and kwargs.get("file")!=instructions:
+            #Ignore the file argument if it is the same as the one in the instructions
+            return kwargs.pop("file")
+        if "file_name" in kwargs:
+            return os.path.join(self.path,kwargs.pop("file_name"))
+        if "name" in kwargs:
+            return os.path.join(self.path,kwargs.pop("name"))
+        if self.file is None:
+            log.error("No file provided: please provide a full file or a file name.")
+            raise ValueError("No file provided: please provide a full file or a file name.")
+        return self.file
+        
+
+    def load_part(
+            self,
+            **kwargs
+    ):
+        """
+        Load a Part from explicit parameters.
+
+        Parameters provided as arguments overload the corresponding instance attributes.
+
+        Returns
+        -------
+        dict
+            Data for creating the Part object
+
+        Raises
+        ------
+        FileNotFoundError
+            If no file nor name argument is provided
+        """
+        #Initialize Part specific parameters
+        part_data = {
+            "metadata" : dict()
+        }
+
+        #Update loader parameters
+        self.update_attributes(**kwargs)
+
+        file = self.get_file(**kwargs)
+
+        
+        loader_kwargs = kwargs.pop("loader_kwargs",self.loader_kwargs)
+
+        name = kwargs.pop("name",os.path.splitext(os.path.basename(file))[0])
+
+        log.info(f"Load part {name} from {file}")
+        
+        if name in self.part_settings:
+            #Load preset settings
+            part_settings = self.part_settings[name]
+            kwargs.update(part_settings)
+        
+        part_data["data"] = load_file(file,
+                                      extension=self.extension,**loader_kwargs)
+
+        labels = []
+        dimensions = kwargs.get("dimensions",
+                                self.metadata.get("dimensions",
+                                part_data["data"].shape)
+        )
+        if dimensions is None:
+            dimensions = part_data["data"].shape
+        for dim in dimensions:
+            labels.append(self._get_labels(dim))
+
+        part_data["metadata"] = self.metadata
+        part_data["name"] = name
+        part_data["metadata"]["path"] = self.path
+        part_data["metadata"]["loader_kwargs"] = loader_kwargs
+        part_data["labels"] = labels
+        part_data["groupings"] = kwargs.get("groupings",self.groupings)       
+        return part_data
+    
+    def _get_labels(self,l):
+        """Find the labels fitting an axis with a given shape
+            
+        If no fitting label is found, data are labelled numerically
+
+        Parameters
+        ----------
+        l : int, list or str
+            Length of the data dimension or name of the dimensions.
+
+        Returns
+        -------
+        dict of str:list of str
+            Labels of the axis.
+
+        """
+        if isinstance(l,list):
+            output = dict()
+            try:
+                for label in l:
+                    output.update(self._get_labels(label))
+                return output
+            except IndexError:
+                return {"0":l}
+        if isinstance(l,str):
+            if l not in self.labels.keys():
+                self.format_labels(l)
+            return {l:self.labels[l]}
+        if not isinstance(l,int):
+            log.error(f"Invalid dimension type {type(l)}")
+            raise TypeError(f"Invalid dimension type {type(l)}")
+        if l==1:
+            return {0:"all"}
+        log.debug("Try to infer label from axis of length "+str(l))
+        for label in self.labels:
+            #Look whether a basic label fits the axis
+            if l == len(self.labels[label]):
+                log.debug(f"Label {label} fits axis of length {l}")
+                return {label:self.labels[label]}
+        for grouping in self.groupings:
+            #Look whether a grouped label fits the axis
+            if l == len(self.groupings[grouping]):
+                log.debug(f"Label {label} fits axis of length {l}")
+                return {grouping:list(self.groupings[grouping]).keys()}
+        log.warning("No label found for axis of length "+str(l))
+        return {"0":[i for i in range(l)]}

mrio_toolbox/utils/savers/__init__.py

@@ -0,0 +1,8 @@
+from mrio_toolbox.utils.savers._to_folder import save_mrio_to_folder, save_part_to_folder
+from mrio_toolbox.utils.savers._to_nc import save_to_nc
+
+__all__ = [
+    "save_mrio_to_folder",
+    "save_part_to_folder",
+    "save_to_nc"
+]

mrio_toolbox/utils/savers/_path_checker.py

@@ -0,0 +1,19 @@
+import os
+
+def check_path(path):
+    """
+    Extend the name path to avoid overwriting existing files.
+
+    Parameters
+    ----------
+    path : str
+        Path currently selected
+    """
+    if os.path.exists(path):
+        i=0
+        new_path = f"{path}_{i}"
+        while os.path.exists(new_path):
+            new_path = f"{path}_{i}"
+            i+=1
+        return new_path
+    return path

mrio_toolbox/utils/savers/_to_folder.py

@@ -0,0 +1,160 @@
+import os
+from mrio_toolbox.utils.savers._path_checker import check_path
+import numpy as np
+import yaml
+import pandas as pd
+
+import logging
+
+log = logging.getLogger(__name__)
+
+def save_mrio_to_folder(obj,
+                        path,
+                        name=None,
+                        extension=".npy",
+                        overwrite=False,
+                        **kwargs):
+    """
+    Save an MRIO instance in a folder
+
+    Parameters
+    ----------
+    path : str
+        Path to the folder to save the MRIO instance into.
+    extension : str, optional
+        Extension of the files to save the MRIO instance into.
+        The default is "npy".
+    overwrite : bool, optional
+        Whether to overwrite the existing files. The default is False.
+        If False, the version name is iterated until a non-existing
+        file name is found.
+    kwargs : dict
+        Additional arguments to pass to the saver.
+    """
+    if name is None:
+        name = os.path.basename(path)+".yaml"
+    if not os.path.isdir(path):
+        os.mkdir(path)
+    elif not overwrite:
+        os.mkdir(check_path(path))
+    log.info(f"Saving MRIO instance {name} to folder {path}") 
+    loading_instructions = dict()
+    loading_instructions.update(obj.metadata)
+    loading_instructions["path"] = path
+    parts_instructions = dict()
+    for part in obj.parts:
+        save_part_to_folder(
+            obj.parts[part],
+            path,
+            extension=extension,
+            overwrite=overwrite,
+            include_labels=False,
+            write_instructions=False,
+            **kwargs
+        )
+        #Save part metadata
+        parts_instructions[part] = dict()
+        parts_instructions[part]["dimensions"] = obj.parts[part].get_dimensions()
+        parts_instructions[part]["metadata"] = obj.parts[part].metadata
+    write_labels(path,obj.labels)
+    loading_instructions["parts"] = parts_instructions
+    loading_instructions["extension"] = extension
+    with open(os.path.join(path+".yaml"),"w") as file:
+        yaml.dump(loading_instructions,file)
+
+def write_labels(path,labels):
+    """
+    Save labels in a folder
+
+    Parameters
+    ----------
+    path : str
+        Path to the folder to save the labels into.
+    extension : str, optional
+        Extension of the files to save the labels into.
+        The default is "txt".
+    overwrite : bool, optional
+        Whether to overwrite the existing files. The default is False.
+        If False, the version name is iterated until a non-existing
+        file name is found.
+    kwargs : dict
+        Additional arguments to pass to the saver.
+    """
+    with open(os.path.join(path,"labels.yaml"),"w") as file:
+        yaml.dump(labels,file)
+
+def save_part_to_folder(obj,
+                        path,
+                        name=None,
+                        extension=".npy",
+                        save_labels=True,
+                        write_instructions=True,
+                        overwrite=False,
+                        include_labels=True,
+                        **kwargs):
+    """
+    Save a Part instance in a folder
+
+    Parameters
+    ----------
+    obj : Part
+        Part instance to save
+    path : str
+        Path to the folder to save the Part instance into.
+    extension : str, optional
+        Extension of the files to save the Part instance into.
+        The default is ".npy".
+    save_labels : bool, optional
+        Whether to save the labels. The default is True.
+    save_instructions : bool, optional
+        Whether to save the instructions. The default is True.
+    overwrite : bool, optional
+        Whether to overwrite the existing files. The default is False.
+        If False, the version name is iterated until a non-existing
+        file name is found.
+    include_labels: bool, optional
+        Whether to include the labels in the file. The default is True.
+        This is only relevant for .csv and .xlsx files.
+        If False, the labels are saved in a separate file.
+    kwargs : dict
+        Additional arguments to pass to the saver.
+    """
+    if name is None:
+        name = obj.name
+    log.info(f"Saving Part instance {name} to folder {path} with extension {extension}")
+    if not os.path.isdir(path):
+            os.mkdir(path)
+    elif not overwrite:
+        os.mkdir(check_path(path))
+    if save_labels:
+        write_labels(path,obj.labels)
+    parts_instructions = dict()
+    if write_instructions:
+        parts_instructions["dimensions"] = obj.get_dimensions()
+        parts_instructions["metadata"] = obj.metadata
+    if extension == ".npy":
+        np.save(os.path.join(path,name+extension),obj.data,**kwargs)
+    elif extension == ".csv":
+        delimiter = kwargs.pop("delimiter",",")
+        if include_labels:
+            obj.to_pandas().to_csv(os.path.join(path,name+extension),
+                                   **kwargs)
+            parts_instructions["index_col"] = [i for i in range(len(obj.axes[0].dims))]
+            if len(obj.axes) > 1:
+                parts_instructions["header"] = [i for i in range(len(obj.axes[1].dimensions))]
+        else:
+            np.savetxt(os.path.join(path,name+extension),obj.data,
+                    delimiter=delimiter,**kwargs)
+    elif extension == ".txt":
+        delimiter = kwargs.pop("delimiter","\t")
+        np.savetxt(os.path.join(path,name+extension),obj.data,
+                   delimiter=delimiter,**kwargs)
+    elif extension == ".xlsx":
+        obj.to_pandas().to_excel(os.path.join(path,name+extension),
+                                 **kwargs)
+    else:
+        raise NotImplementedError(f"Extension {extension} not supported")
+    if write_instructions:
+        parts_instructions["file"] = os.path.join(path,name+extension)
+        with open(os.path.join(path,name+".yaml"),"w") as file:
+            yaml.dump(parts_instructions,file)

mrio_toolbox/utils/savers/_to_nc.py

@@ -0,0 +1,52 @@
+import os
+import yaml
+import pandas as pd
+from mrio_toolbox.utils.savers._path_checker import check_path
+import logging
+
+log = logging.getLogger(__name__)
+
+def save_to_nc(obj,path,overwrite=False,write_instructions=False,**kwargs):
+    """
+    Save an MRIO or Path instance in a .nc file
+
+    Parameters
+    ----------
+    path : str
+        Path to the .nc file to save the MRIO instance into.
+    
+    **kwargs : dict
+        Additional arguments to pass to the saver.
+    """
+    log.info(f"Saving {obj.__class__.__name__} instance to {path}")
+    ds = obj.to_xarray()
+    for index in ds.indexes:
+        if isinstance(ds.indexes[index],pd.MultiIndex):
+            import cf_xarray as cfxr
+            #Compress MultiIndex data as it is not supported by xarray
+            ds = cfxr.encode_multi_index_as_compress(ds,index)
+
+    #Remove dict attrs (not supported for serialization)
+    attrs = list(ds.attrs.keys())
+    for attr in attrs:
+        if isinstance(ds.attrs[attr],dict):
+            log.warning(f"Attribute {attr} is a dict. It will not be saved.")
+            ds.attrs.pop(attr)
+    
+    for var in ds.data_vars:
+        attrs = list(ds[var].attrs.keys())
+        for attr in attrs:
+            if isinstance(ds[var].attrs[attr],dict):
+                log.warning(f"Attribute {attr} of {var} is a dict. It will not be saved.")
+                ds[var].attrs.pop(attr)
+
+    if not overwrite:
+        base_path = os.path.splitext(path)[0]
+        path = check_path(base_path)
+    ds.to_netcdf(path+".nc",**kwargs)
+    if write_instructions:
+        instructions = {
+            "file": path+".nc"
+        }
+        with open(path+".yaml","w") as file:
+            yaml.dump(instructions,file)