dsipts 1.1.5__py3-none-any.whl

dsipts/__init__.py

@@ -0,0 +1,48 @@
+# src/dispts/__init__.py
+
+from .data_management.monash import Monash, get_freq
+from .data_management.public_datasets import read_public_dataset
+from .data_structure.data_structure import TimeSeries, Categorical
+from .data_structure.utils import extend_time_df, beauty_string
+
+from .models.RNN import RNN
+from .models.LinearTS import LinearTS
+from .models.Persistent import Persistent
+from .models.D3VAE import D3VAE
+from .models.DilatedConv import DilatedConv
+from .models.TFT import TFT
+from .models.Informer import Informer
+from .models.VVA import VVA
+from .models.VQVAEA import VQVAEA
+from .models.CrossFormer import CrossFormer
+from .models.Autoformer import Autoformer
+from .models.PatchTST import PatchTST
+from .models.Diffusion import Diffusion
+from .models.DilatedConvED import DilatedConvED
+from .models.TIDE import TIDE
+from .models.ITransformer import ITransformer
+from .models.TimeXER import TimeXER
+from .models.TTM import TTM
+from .models.Samformer import Samformer
+from .models.Duet import Duet
+
+try:
+    import lightning.pytorch as pl
+    from .models.base_v2 import Base
+    OLD_PL = False
+except ImportError:
+    import pytorch_lightning as pl
+    from .models.base import Base
+    OLD_PL = True
+
+__all__ = [
+    # Data Management
+    "Monash", "get_freq", "read_public_dataset",
+    # Data Structure
+    "TimeSeries", "Categorical", "extend_time_df", "beauty_string",
+    # Models
+    "RNN", "LinearTS", "Persistent", "D3VAE", "DilatedConv", "TFT",
+    "Informer", "VVA", "VQVAEA", "CrossFormer", "Autoformer", "PatchTST",
+    "Diffusion", "DilatedConvED", "TIDE", "ITransformer", "TimeXER",
+    "TTM", "Samformer", "Duet", "Base"
+]

dsipts/data_management/monash.py

@@ -0,0 +1,338 @@
+import pandas as pd
+import requests
+from bs4 import BeautifulSoup as bs
+import pickle 
+import os
+import shutil
+from datetime import datetime
+from distutils.util import strtobool
+from typing import Union
+import logging
+
+# Converts the contents in a .tsf file into a dataframe and returns it along with other meta-data of the dataset: frequency, horizon, whether the dataset contains missing values and whether the series have equal lengths
+#
+# Parameters
+# full_file_path_and_name - complete .tsf file path
+# replace_missing_vals_with - a term to indicate the missing values in series in the returning dataframe
+# value_column_name - Any name that is preferred to have as the name of the column containing series values in the returning dataframe
+def convert_tsf_to_dataframe(
+    full_file_path_and_name:str,
+    replace_missing_vals_with:str="NaN",
+    value_column_name:str="series_value",
+)-> pd.DataFrame:
+    """I copied this function from the repo
+
+
+    Args:
+        full_file_path_and_name (str): path
+        replace_missing_vals_with (str, optional): replace not valid numbers. Defaults to "NaN".
+        value_column_name (str, optional):. Defaults to "series_value". 
+
+    Raises:
+        Exception:  see https://forecastingdata.org/ for more information
+       
+
+    Returns:
+        pd.DataFrame: the selected timseries
+    """
+    col_names = []
+    col_types = []
+    all_data = {}
+    line_count = 0
+    frequency = None
+    forecast_horizon = None
+    contain_missing_values = None
+    contain_equal_length = None
+    found_data_tag = False
+    found_data_section = False
+    started_reading_data_section = False
+
+    with open(full_file_path_and_name, "r", encoding="cp1252") as file:
+        for line in file:
+            # Strip white space from start/end of line
+            line = line.strip()
+
+            if line:
+                if line.startswith("@"):  # Read meta-data
+                    if not line.startswith("@data"):
+                        line_content = line.split(" ")
+                        if line.startswith("@attribute"):
+                            if (
+                                len(line_content) != 3
+                            ):  # Attributes have both name and type
+                                raise Exception("Invalid meta-data specification.")
+
+                            col_names.append(line_content[1])
+                            col_types.append(line_content[2])
+                        else:
+                            if (
+                                len(line_content) != 2
+                            ):  # Other meta-data have only values
+                                raise Exception("Invalid meta-data specification.")
+
+                            if line.startswith("@frequency"):
+                                frequency = line_content[1]
+                            elif line.startswith("@horizon"):
+                                forecast_horizon = int(line_content[1])
+                            elif line.startswith("@missing"):
+                                contain_missing_values = bool(
+                                    strtobool(line_content[1])
+                                )
+                            elif line.startswith("@equallength"):
+                                contain_equal_length = bool(strtobool(line_content[1]))
+
+                    else:
+                        if len(col_names) == 0:
+                            raise Exception(
+                                "Missing attribute section. Attribute section must come before data."
+                            )
+
+                        found_data_tag = True
+                elif not line.startswith("#"):
+                    if len(col_names) == 0:
+                        raise Exception(
+                            "Missing attribute section. Attribute section must come before data."
+                        )
+                    elif not found_data_tag:
+                        raise Exception("Missing @data tag.")
+                    else:
+                        if not started_reading_data_section:
+                            started_reading_data_section = True
+                            found_data_section = True
+                            all_series = []
+
+                            for col in col_names:
+                                all_data[col] = []
+
+                        full_info = line.split(":")
+
+                        if len(full_info) != (len(col_names) + 1):
+                            raise Exception("Missing attributes/values in series.")
+
+                        series = full_info[len(full_info) - 1]
+                        series = series.split(",")
+
+                        if len(series) == 0:
+                            raise Exception(
+                                "A given series should contains a set of comma separated numeric values. At least one numeric value should be there in a series. Missing values should be indicated with ? symbol"
+                            )
+
+                        numeric_series = []
+
+                        for val in series:
+                            if val == "?":
+                                numeric_series.append(replace_missing_vals_with)
+                            else:
+                                numeric_series.append(float(val))
+
+                        if numeric_series.count(replace_missing_vals_with) == len(
+                            numeric_series
+                        ):
+                            raise Exception(
+                                "All series values are missing. A given series should contains a set of comma separated numeric values. At least one numeric value should be there in a series."
+                            )
+
+                        all_series.append(pd.Series(numeric_series).array)
+
+                        for i in range(len(col_names)):
+                            att_val = None
+                            if col_types[i] == "numeric":
+                                att_val = int(full_info[i])
+                            elif col_types[i] == "string":
+                                att_val = str(full_info[i])
+                            elif col_types[i] == "date":
+                                att_val = datetime.strptime(
+                                    full_info[i], "%Y-%m-%d %H-%M-%S"
+                                )
+                            else:
+                                raise Exception(
+                                    "Invalid attribute type."
+                                )  # Currently, the code supports only numeric, string and date types. Extend this as required.
+
+                            if att_val is None:
+                                raise Exception("Invalid attribute value.")
+                            else:
+                                all_data[col_names[i]].append(att_val)
+
+                line_count = line_count + 1
+
+        if line_count == 0:
+            raise Exception("Empty file.")
+        if len(col_names) == 0:
+            raise Exception("Missing attribute section.")
+        if not found_data_section:
+            raise Exception("Missing series information under data section.")
+
+        all_data[value_column_name] = all_series
+        loaded_data = pd.DataFrame(all_data)
+
+        return (
+            loaded_data,
+            frequency,
+            forecast_horizon,
+            contain_missing_values,
+            contain_equal_length,
+        )
+        
+        
+def get_freq(freq)->str:
+    """Get the frequency based on the string reported. I don't think there are all the possibilities here
+
+    Args:
+        freq (str): string coming from 
+
+    Returns:
+        str: pandas frequency format
+    """
+    if freq =='10_minutes':
+        return '600s'
+    elif freq == 'hourly':
+        return 'H'
+    else:
+        return 'D'     
+        
+        
+class Monash():
+    
+    def __init__(self,filename:str,baseUrl:str ='https://forecastingdata.org/', rebuild:bool =False):
+        """Class for downloading datasets listed here https://forecastingdata.org/ 
+
+        Args:
+            filename (str):  name of the class, used for saving
+            baseUrl (str, optional): url to the source page. Defaults to 'https://forecastingdata.org/'.
+            rebuild (bool, optional):  if true the table will be loaded from the webpage otherwise it will be loaded from the saved file. Defaults to False.
+        """
+        self.baseUrl = baseUrl
+        self.downloaded = {}
+        if rebuild is False:
+            logging.info(filename)
+            if os.path.exists(filename+'.pkl'):
+                self.load(filename)
+            else:
+                self.get_table(baseUrl)
+                self.save(filename)
+        else:
+            self.get_table(baseUrl)
+            self.save(filename)
+
+    def get_table(self, baseUrl):
+        """   get table
+        
+        :meta private:
+        """
+        with requests.Session() as s:
+            r = s.get(baseUrl)
+        soup = bs(r.content)
+        header = []
+        for x in soup.find("table", {"class": "responsive-table sortable"}).find('thead').find_all('th'):
+            header.append(x.text)
+        header
+
+        tot= []
+        for row in soup.find("table", {"class": "responsive-table sortable"}).find('tbody').find_all('tr'):
+            row_data = []
+            links = {}
+            for i,column in enumerate(row.find_all('td')):
+                tmp_links = column.find_all('a')
+                if len(tmp_links)>0:
+
+                    for x in tmp_links:
+                        if 'zenodo' in x['href']:
+                            links[x.text] = x['href']
+                            i_links = i
+                row_data.append(column.text)
+            for dataset in links:
+                row_to_insert = {}
+                for j, head in enumerate(header):
+                    if j!=i_links:
+                        row_to_insert[header[j]] = row_data[j]
+                    else:
+                        row_to_insert['freq'] = dataset
+                        row_to_insert[header[j]] = links[dataset]
+                tot.append(row_to_insert)
+
+        tot = pd.DataFrame(tot)
+        tot['id']  = tot.Download.apply(lambda x:int(x.split('/')[-1]))
+        self.table = tot.copy()
+   
+    def save(self, filename:str)-> None:
+        """Save the monarch structure
+
+        Args:
+            filename (str): name of the file to generate
+        """
+        logging.info('Saving')
+        with open(f'{filename}.pkl','wb') as f:
+            params =  self.__dict__.copy()
+            #for k in ['data','data_train','data_test','data_validation']:
+            #    if k in params.keys():
+            #        _ = params.pop(k)
+            pickle.dump(params,f)
+    def load(self, filename:str)-> None:
+        """Load a monarch structure
+
+        Args:
+            filename (str): filename to load
+        """
+        logging.info('Loading')
+        with open(filename+'.pkl','rb') as f:
+            params = pickle.load(f)
+            for p in params:
+                setattr(self,p, params[p])    
+   
+    def download_dataset(self,path: str,id:int ,rebuild=False)->None:
+        """download a specific dataset 
+
+        Args:
+            path (str): path in which save the data
+            id (int): id of the dataset
+            rebuild (bool, optional): if true the dataset will be re-downloaded. Defaults to False.
+        """
+            
+        if os.path.exists(path):
+            pass
+        else:
+            os.mkdir(path)
+        if os.path.exists(os.path.join(path,str(id))):
+            if rebuild:
+                file = self._download(url = self.table.Download[self.table.id== id].values[0], path = os.path.join(path,str(id)))
+                self.downloaded[id] = f'{path}/{id}/{file}'
+            else:
+                pass
+        else:
+            file = self._download(url = self.table.Download[self.table.id== id].values[0] , path = os.path.join(path,str(id)))
+            self.downloaded[id] = f'{path}/{id}/{file}'
+            
+    def _download(self,url, path)->str:
+        """ get data
+        
+        :meta private:
+        """
+        with requests.Session() as s:
+            r = s.get(url)
+            soup = bs(r.content)
+
+            url = soup.find("link", {"type": "application/zip"})['href']
+            logging.info(url)
+            with open(path+'.zip', "wb") as f:
+                f.write(s.get(url).content)
+        
+        shutil.unpack_archive(path+'.zip', path)
+        os.remove(path+'.zip')
+        return os.listdir(path)[0]
+       
+    def generate_dataset(self, id:int)-> Union[None, pd.DataFrame]:
+        """Parse the id-th dataset in a convient format and return a pandas dataset
+
+        Args:
+            id (int): id of the dataset
+
+        Returns:
+            None or pd.DataFrame: dataframe
+        """
+
+        if id not in self.downloaded.keys():
+            logging.error('please call first download dataset')
+            return None
+        else:
+            return convert_tsf_to_dataframe(self.downloaded[id])

dsipts/data_management/public_datasets.py

@@ -0,0 +1,162 @@
+import pandas as pd
+import os
+import numpy as np
+from typing import List, Tuple
+import logging
+import requests
+from bs4 import BeautifulSoup as bs
+
+def build_venice(path:str,url='https://www.comune.venezia.it/it/content/archivio-storico-livello-marea-venezia-1')->None:
+    
+      
+    with requests.Session() as s:
+        r = s.get(url)
+    soup = bs(r.content)
+
+    print('CARE THE STRUCTURE OF THE SITE CAN BE CHANGED')
+
+    def cast_string(x):
+        if np.isfinite(x) is False:
+            return x
+        if x<10:
+            return f'0{int(x)}:00'
+        else:
+            return f'{int(x)}:00'
+        
+    def cast_month(x):
+        try:
+            return x.replace('gen','01').replace('feb','02').replace('mar','03').replace('apr','04').replace('mag','05').replace('giu','06').replace('lug','07').replace('ago','08').replace('set','09').replace('ott','10').replace('nov','11').replace('dic','12')
+        except:
+            return x
+        
+    def remove_float(table,column):
+        if table[column].dtype in [int,float]:
+            table[column] = table[column].apply(lambda x:cast_string(x))
+        else:
+            pass
+        
+    def remove_str(table,column):
+        table[column] = table[column].apply(lambda x:cast_month(x))
+            
+    def normalize(table):
+        columns = table.columns
+        table = table[~table.isna().all(axis=1)]
+        if 'Data_ora(solare)' in columns:
+            table['time'] = table['Data_ora(solare)'] 
+        
+        elif 'GIORNO' in columns and 'ORA solare' in columns:
+            remove_float(table,'ORA solare')
+            table['time'] = table['GIORNO'] +' '+ table['ORA solare'] 
+        
+        elif 'data' in columns and 'ora solare' in columns:
+            remove_float(table,'ora solare')
+            table['time'] =table['data'] +' '+ table['ora solare'] 
+        
+        elif 'Data' in columns and 'Ora solare' in columns:
+            remove_str(table,'Data')
+            remove_float(table,'Ora solare')
+            table['time'] = table['Data'] +' '+ table['Ora solare'] 
+        elif 'GIORNO' in columns and 'ORA' in columns:
+            remove_str(table,'GIORNO')
+            remove_float(table,'ORA')
+            table['time'] = table['GIORNO'] +' '+ table['ORA'] 
+        
+        else:
+            import pdb
+            pdb.set_trace()
+       
+        for c in columns:
+            if 'Salute' in c:
+                table['y'] = table[c].values
+                if 'cm' in c:
+                    table['y']/=100
+        res = table[['time','y']].dropna()
+        try:
+            res['time'] = pd.to_datetime(res['time'],format='mixed')
+        except:
+            import pdb
+            pdb.set_trace()
+        return res
+    tot= []
+    for row in soup.find_all("table")[1].find('tbody').find_all('tr'):
+        for i,column in enumerate(row.find_all('td')):
+            tmp_links = column.find_all('a')
+            if len(tmp_links)>0:
+                for x in tmp_links:
+                    if 'orari' in x['href']:
+                        tmp =  pd.read_csv('https://www.comune.venezia.it/'+x['href'],sep=';', parse_dates=True)
+                        tot.append(normalize(tmp))
+              
+    
+    res = pd.concat(tot)
+    res.sort_values(by='time',inplace=True)
+    res.to_csv(f'{path}/venice.csv',index=False)
+
+
+
+def read_public_dataset(path:str,dataset:str)->Tuple[pd.DataFrame,List[str]]:
+    """
+    Returns the public dataset chosen. Pleas download the dataset from here https://drive.google.com/drive/folders/1ZOYpTUa82_jCcxIdTmyr0LXQfvaM9vIy or ask to agobbi@fbk.eu. 
+
+
+    Args:
+        path (str): path to data
+        dataset (str): dataset (one of 'electricity','etth1','etth2','ettm1','ettm2','exchange_rate','illness','traffic','weather')
+
+    Returns:
+        Tuple[pd.DataFrame,List[str]]: The target variable is *y* and the time index is *time* and the list of the covariates
+    """
+    
+    
+
+    if os.path.isdir(path):
+        pass
+    else:
+        logging.info('I will try to create the folder')
+        os.mkdir(path)
+        
+    files = os.listdir(path)
+    if 'all_six_datasets' in files:
+        pass
+    else:
+        logging.error('Please dowload the zip file form here and unzip it https://drive.google.com/drive/folders/1ZOYpTUa82_jCcxIdTmyr0LXQfvaM9vIy')
+        return None,None
+    
+    
+    if dataset not in ['electricity','etth1','etth2','ettm1','ettm2','exchange_rate','illness','traffic','weather','venice']:
+        logging.error(f"Dataset {dataset} not available, use one among ['electricity','etth1','etth2','ettm1','ettm2','exchange_rate','illness','traffic','weather','venice']")
+        return None,None
+
+    if dataset=='electricity':
+        dataset = pd.read_csv(os.path.join(path,'all_six_datasets/electricity/electricity.csv'),sep=',',na_values=-9999)
+    elif dataset=='etth1':
+        dataset = pd.read_csv(os.path.join(path,'all_six_datasets/ETT-small/ETTh1.csv'),sep=',',na_values=-9999)  
+    elif dataset=='etth2':
+        dataset = pd.read_csv(os.path.join(path,'all_six_datasets/ETT-small/ETTh2.csv'),sep=',',na_values=-9999)
+    elif dataset=='ettm1':
+        dataset = pd.read_csv(os.path.join(path,'all_six_datasets/ETT-small/ETTm1.csv'),sep=',',na_values=-9999)
+    elif dataset=='ettm2':
+        dataset = pd.read_csv(os.path.join(path,'all_six_datasets/ETT-small/ETTm2.csv'),sep=',',na_values=-9999)
+    elif dataset=='exchange_rate':
+        dataset = pd.read_csv(os.path.join(path,'all_six_datasets/exchange_rate/exchange_rate.csv'),sep=',',na_values=-9999)
+    elif dataset=='illness':
+        dataset = pd.read_csv(os.path.join(path,'all_six_datasets/illness/national_illness.csv'),sep=',',na_values=-9999)
+    elif dataset=='traffic':
+        dataset = pd.read_csv(os.path.join(path,'all_six_datasets/traffic/traffic.csv'),sep=',',na_values=-9999) 
+    elif dataset=='weather':
+        dataset = pd.read_csv(os.path.join(path,'all_six_datasets/weather/weather.csv'),sep=',',na_values=-9999) 
+    elif dataset=='venice':
+        if os.path.isfile(os.path.join(path,'venice.csv')):
+            dataset = pd.read_csv(os.path.join(path,'venice.csv')) 
+        else:
+            logging.info('I WILL TRY TO DOWNLOAD IT, if there are errors please have a look to `build_venice` function')
+            build_venice(path,url='https://www.comune.venezia.it/it/content/archivio-storico-livello-marea-venezia-1')
+            dataset = pd.read_csv(os.path.join(path,'venice.csv')) 
+    else:
+        logging.error(f'Dataset {dataset} not found')
+        return None, None
+    dataset.rename(columns={'date':'time','OT':'y'},inplace=True)
+    dataset.time = pd.to_datetime(dataset.time)
+    logging.info(f'Dataset loaded with shape {dataset.shape}')
+    
+    return dataset, list(set(dataset.columns).difference(set(['time','y'])))