wolfhece 2.1.85__py3-none-any.whl → 2.1.87__py3-none-any.whl

wolfhece/irm_qdf.py

@@ -10,13 +10,16 @@ copying or distribution of this file, via any medium, is strictly prohibited.
 
 #Code INS des communes belges
 import numpy as np
-from os import path,mkdir
+from os import path, mkdir
+from pathlib import Path
 from time import sleep
 import pandas as pd
 import matplotlib.pyplot as plt
 from scipy.optimize import minimize,curve_fit
 from scipy.stats import gumbel_r,genextreme
 from typing import Literal
+from tqdm import tqdm
+import logging
 
 from .ins import Localities
 from .PyTranslate import _
@@ -72,6 +75,7 @@ durationsd=np.array([1,2,3,4,5,7,10,15,20,25,30],np.float64)*24.*60.
 durations = np.concatenate([durations,durationsd])
 
 class MontanaIRM():
+    """ Classe pour la gestion des relations de Montana pour les précipitations """
 
     def __init__(self,coeff:pd.DataFrame,time_bounds=None) -> None:
 
@@ -82,7 +86,12 @@ class MontanaIRM():
 
         self.coeff=coeff
 
-    def get_ab(self,dur,T):
+    def get_ab(self, dur, T):
+        """ Get the Montana coefficients for a given duration and return period
+
+        :param dur: the duration
+        :param T: the return period
+        """
 
         curcoeff = self.coeff.loc[float(T)]
         if dur<self.time_bounds[0]:
@@ -97,24 +106,48 @@ class MontanaIRM():
 
         return a,b
 
-    def get_meanrain(self,dur,T,ab=None):
+    def get_meanrain(self, dur, T, ab= None):
+        """ Get the mean rain for a given duration and return period
+
+        :param dur: the duration
+        :param T: the return period
+        :param ab: the Montana coefficients
+        """
+
         if ab is None:
             ab = self.get_ab(dur,T)
         return ab[0]*dur**(-ab[1])
 
-    def get_instantrain(self,dur,T,ab=None):
+    def get_instantrain(self, dur, T, ab= None):
+        """ Get the instantaneous rain for a given duration and return period
+
+        :param dur: the duration
+        :param T: the return period
+        :param ab: the Montana coefficients
+        """
         if ab is None:
             ab = self.get_ab(dur,T)
         meani=self.get_meanrain(dur,T,ab)
         return (1.-ab[1])*meani
 
-    def get_Q(self,dur,T):
+    def get_Q(self, dur, T):
+        """ Get the quantity of rain for a given duration and return period
+
+        :param dur: the duration
+        :param T: the return period
+        """
+
         rain = self.get_meanrain(dur,T)
         return rain*dur/60. #to obtains [mm.h^-1] as dur is in [min]
 
-    def get_hyeto(self,durmax,T,r=0.5):
-        """ :param r: Decentration coefficient
+    def get_hyeto(self, durmax, T, r= 0.5):
+        """ Get the hyetogram for a given return period
+
+        :param durmax: the maximum duration of the hyetogram
+        :param T: the return period
+        :param r: Decentration coefficient
         """
+
         x = np.arange(10,durmax,1,dtype=np.float64)
         # y = [self.get_instantrain(curx,T) for curx in x]
 
@@ -148,7 +181,13 @@ class MontanaIRM():
 
         return x_hyeto,y_hyeto
 
-    def plot_hyeto(self,durmax,T,r=0.5):
+    def plot_hyeto(self, durmax, T, r= 0.5):
+        """ Plot the hyetogram for a given return period
+
+        :param durmax: the maximum duration of the hyetogram
+        :param T: the return period
+        :param r: Decentration coefficient
+        """
         x,y = self.get_hyeto(durmax,T,r)
 
         fig,ax = plt.subplots(1,1,figsize=[15,10])
@@ -160,7 +199,13 @@ class MontanaIRM():
 
         return fig,ax
 
-    def plot_hyetos(self,durmax,r=0.5):
+    def plot_hyetos(self, durmax, r= 0.5):
+        """ Plot the hyetograms for all return periods
+
+        :param durmax: the maximum duration of the hyetograms
+        :param r: Decentration coefficient
+        """
+
         fig,ax = plt.subplots(1,1,figsize=[15,10])
 
         for curT in RT:
@@ -183,6 +228,7 @@ class Qdf_IRM():
     Pour importer les fichiers depuis le site web de l'IRM meteo.be
     from wolfhece.irm_qdf import Qdf_IRM
     qdf = Qdf_IRM(force_import=True)
+    qdf =
 
     Il est possible de spécifier le répertoire de stockage des fichiers Excel
     Par défaut, il s'agit d'un sous-répertoire 'irm' du répertoire courant qui sera créé s'il n'exsiste pas
@@ -209,25 +255,58 @@ class Qdf_IRM():
 
     """
 
-    def __init__(self, store_path= 'irm', code:int= 0, name= '', force_import= False, ins:Literal['2018', '2019', '2025', 2018, 2019, 2025] = 2019) -> None:
+    def __init__(self, store_path= 'irm',
+                 code:int= 0, name= '',
+                 force_import= False,
+                 ins:Literal['2018', '2019', '2025', 2018, 2019, 2025] = 2018,
+                 localities:Localities = None) -> None:
+
+        if localities is None:
+
+            assert int(ins) in [2018, 2019, 2025], _('Bad INS - Retry !')
+
+            self.myloc = Localities(ins)
+        else:
+            self.myloc = localities
 
-        self.myloc = Localities(ins)
         self.store = store_path
 
         if force_import:
-            self.importfromwebsite(store_path)
+            self.importfromwebsite(store_path, ins=ins)
 
         if code !=0:
-            self.ins_read_excel(code=str(code))
-            self.fit_all()
+            if self.ins_read_excel(code=str(code)):
+                self.fit_all()
+                self._code = code
+                self._name = self.myloc.get_namefromINS(code)
+            else:
+                logging.debug(f"INS code {code} not found in the store")
         elif name!='':
-            self.ins_read_excel(name=name)
-            self.fit_all()
+            if self.ins_read_excel(name=name):
+                self.fit_all()
+                self._name = name
+                self._code = self.myloc.get_INSfromname(name)
+            else:
+                logging.debug(f"Name {name} not found in the store")
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def code(self):
+        return self._code
 
-        self.montanacoeff=None
-        pass
+    @property
+    def code_name(self):
+        return str(self._code) + '-' + self._name
+
+    @property
+    def name_code(self):
+        return self._name + '-' + str(self._code)
 
     def export_allmontana2xls(self):
+        """ Export all Montana coefficients to an Excel file """
 
         newdf = []
 
@@ -246,24 +325,29 @@ class Qdf_IRM():
         newdf.to_excel("allmontana.xlsx")
 
 
-    def importfromwebsite(self,store_path='irm',verbose=False,waitingtime=.01):
+    @classmethod
+    def importfromwebsite(cls, store_path= 'irm', verbose:bool= False, waitingtime:float= .01, ins:Literal['2018', '2019', '2025', 2018, 2019, 2025] = 2018):
         """ Import Excel files for all municipalities from the IRM website
 
-            :param store_path: Where to store the downloaded data. Directory
-                will be created if it doesn't exists.
-            :param verbose: If `True`, will print some progress information. If
-                `False`, will do nothing. If a callable, then will call it with a
-                float in [0, 1]. 0 means nothing downloaded, 1 means everything
-                downloaded.
-            :param waitingtime: How long to wait (in seconds) betwenn the download
-                of each station (will make sure we don't overwhelm IRM's website).
+        :param store_path: Where to store the downloaded data. Directory will be created if it doesn't exists.
+        :param verbose: If `True`, will print some progress information.
+                        If `False`, will do nothing.
+                        If a callable, then will call it with a float in [0, 1].
+                        0 means nothing downloaded, 1 means everything downloaded.
+
+        :param waitingtime: How long to wait (in seconds) betwenn the download
+                            of each station (will make sure we don't overwhelm IRM's website).
+
+        :param ins: The year of the INS codes to use.
         """
         import requests
 
+        myloc = Localities(ins)
+
         if not path.exists(store_path):
             mkdir(store_path)
 
-        for key,myins in enumerate(self.myloc.inscode2name):
+        for key,myins in enumerate(myloc.inscode2name):
             #chaîne URL du fichier Excel
             url="https://www.meteo.be//resources//climatology//climateCity//xls//IDF_table_INS"+str(myins)+".xlsx"
             #Obtention du fichiers depuis le site web de l'IRM
@@ -275,7 +359,7 @@ class Qdf_IRM():
                 file.close()
                 if verbose:
                     if callable(verbose):
-                        verbose(key/len(self.myloc.inscode2name))
+                        verbose(key/len(myloc.inscode2name))
                     else:
                         print(myins)
 
@@ -293,22 +377,73 @@ class Qdf_IRM():
                 return _('Bad name ! - Retry')
             loccode=str(self.myloc.insname2code[name.lower()])
 
-        self.code = loccode
-        self.name = name
+        self._code = loccode
+        self._name = name
 
-        pathname = path.join(self.store,loccode+".xlsx")
+        store = Path(self.store)
+
+        pathname_xls = store / (loccode+".xlsx")
+        pathname_csv = store / 'csv' / loccode
+
+        if pathname_csv.exists():
+            self.qdf = pd.read_csv(pathname_csv / 'qdf.csv', index_col=0)
+            self.standarddev = pd.read_csv(pathname_csv / 'standarddev.csv', index_col=0)
+            self.confintlow = pd.read_csv(pathname_csv / 'confintlow.csv', index_col=0)
+            self.confintup = pd.read_csv(pathname_csv / 'confintup.csv', index_col=0)
+            self.montanacoeff = pd.read_csv(pathname_csv / 'montanacoeff.csv', index_col=0)
+            self.montana = MontanaIRM(self.montanacoeff)
+            return True
+        else:
+            with warnings.catch_warnings(record=True):
+                warnings.simplefilter("always")
+                if path.exists(pathname_xls):
+                    self.qdf=pd.read_excel(pathname_xls,"Return level",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl', engine_kwargs={'read_only': True})
+                    self.standarddev=pd.read_excel(pathname_xls,"Standard deviation",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl', engine_kwargs={'read_only': True})
+                    self.confintlow=pd.read_excel(pathname_xls,"Conf. interval, lower bound",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl', engine_kwargs={'read_only': True})
+                    self.confintup=pd.read_excel(pathname_xls,"Conf. interval, upper bound",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl', engine_kwargs={'read_only': True})
+                    self.montanacoeff=pd.read_excel(pathname_xls,"Montana coefficients",index_col=0,skiprows=range(11),nrows=12,usecols="A:G",engine='openpyxl', engine_kwargs={'read_only': True})
+                    self.montana = MontanaIRM(self.montanacoeff)
+                    return True
+                else:
+                    self.qdf=None
+                    self.standarddev=None
+                    self.confintlow=None
+                    self.confintup=None
+                    self.montanacoeff=None
+                    self.montana=None
+                    return False
+
+    @classmethod
+    def convert_xls2csv(cls, store_path= 'irm', ins:Literal['2018', '2019', '2025', 2018, 2019, 2025] = 2018):
+        """ Convert all Excel files to JSON files
+
+        :param store_path: Where to store the downloaded data. Directory will be created if it doesn't exists.
+        :param ins: The year of the INS codes to use.
+        """
+
+        myloc = Localities(ins)
+
+        store_path = Path(store_path)
+
+        for key,myins in enumerate(myloc.get_allcodes()):
+            pathname = store_path / (str(myins)+".xlsx")
+            if pathname.exists():
+
+                qdf=pd.read_excel(pathname,"Return level",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl', engine_kwargs={'read_only': True})
+                standarddev=pd.read_excel(pathname,"Standard deviation",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl', engine_kwargs={'read_only': True})
+                confintlow=pd.read_excel(pathname,"Conf. interval, lower bound",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl', engine_kwargs={'read_only': True})
+                confintup=pd.read_excel(pathname,"Conf. interval, upper bound",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl', engine_kwargs={'read_only': True})
+                montanacoeff=pd.read_excel(pathname,"Montana coefficients",index_col=0,skiprows=range(11),nrows=12,usecols="A:G",engine='openpyxl', engine_kwargs={'read_only': True})
+
+                store_csv = store_path / 'csv' / str(myins)
+                store_csv.mkdir(exist_ok=True)
+
+                qdf.to_csv(store_csv / 'qdf.csv')
+                standarddev.to_csv(store_csv / 'standarddev.csv')
+                confintlow.to_csv(store_csv / 'confintlow.csv')
+                confintup.to_csv(store_csv / 'confintup.csv')
+                montanacoeff.to_csv(store_csv / 'montanacoeff.csv')
 
-        with warnings.catch_warnings(record=True):
-            warnings.simplefilter("always")
-            if path.exists(pathname):
-                self.qdf=pd.read_excel(pathname,"Return level",index_col=0,skiprows=range(7),nrows=19,usecols="A:M")
-                self.standarddev=pd.read_excel(pathname,"Standard deviation",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl')
-                self.confintlow=pd.read_excel(pathname,"Conf. interval, lower bound",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl')
-                self.confintup=pd.read_excel(pathname,"Conf. interval, upper bound",index_col=0,skiprows=range(7),nrows=19,usecols="A:M",engine='openpyxl')
-                self.montanacoeff=pd.read_excel(pathname,"Montana coefficients",index_col=0,skiprows=range(11),nrows=12,usecols="A:G",engine='openpyxl')
-                self.montana = MontanaIRM(self.montanacoeff)
-            else:
-                return 'File not present'
 
     def plot_idf(self,T=None,which='All',color=[27./255.,136./255.,245./255.]):
         """
@@ -347,7 +482,7 @@ class Qdf_IRM():
         ax.set_ylabel(_('Intensity [mm/h]'))
         ax.set_xticks(durations)
         ax.set_xticklabels(durationstext,rotation=45)
-        ax.set_title(self.name + ' - code : ' + str(self.code))
+        ax.set_title(self._name + ' - code : ' + str(self._code))
 
         return fig,ax
 
@@ -386,11 +521,12 @@ class Qdf_IRM():
         ax.set_ylabel(_('Quantity [mm]'))
         ax.set_xticks(durations)
         ax.set_xticklabels(durationstext,rotation=45)
-        ax.set_title(self.name + ' - code : ' + str(self.code))
+        ax.set_title(self._name + ' - code : ' + str(self._code))
 
         return fig,ax
 
     def plot_cdf(self,dur=None):
+        """ Plot the cdf of the QDF data for a given duration """
 
         fig,ax = plt.subplots(1,1,figsize=(10,10))
         if dur is None:
@@ -404,37 +540,85 @@ class Qdf_IRM():
         ax.legend().set_draggable(True)
         ax.set_ylabel(_('Cumulative distribution function (cdf)'))
         ax.set_xlabel(_('Quantity [mm]'))
-        ax.set_title(self.name + ' - code : ' + str(self.code))
+        ax.set_title(self._name + ' - code : ' + str(self._code))
 
         return fig,ax
 
     def fit_all(self):
+        """ Fit all durations with a Generalized Extreme Value distribution """
+
+        self.load_fits_json()
+
+        if self.popt_all == {}:
+            for curdur in durationstext:
+                fig,ax,popt,pcov = self.fit_cdf(curdur)
+                self.popt_all[curdur]=popt
+                # self.pcov_all[curdur]=pcov
+
+            self.save_fits_json()
+
+    def save_fits_json(self):
+        """ Save the fits in a csv file """
 
-        self.popt_all={}
-        self.pcov_all={}
+        with open(path.join(self.store, str(self._code) + '_fits.json'), 'w') as f:
+            df = pd.DataFrame(self.popt_all)
+            df.to_json(f)
 
-        for curdur in durationstext:
-            fig,ax,popt,pcov = self.fit_cdf(curdur)
-            self.popt_all[curdur]=popt
-            self.pcov_all[curdur]=pcov
+        # with open(path.join(self.store, str(self.code) + '_fits_cov.json'), 'w') as f:
+        #     df = pd.DataFrame(self.pcov_all)
+        #     df.to_json(f)
 
-    def fit_cdf(self,dur=None,plot=False):
+    def load_fits_json(self):
+        """ Load the fits from a json file """
+
+        import json
+
+        filename = path.join(self.store, str(self._code) + '_fits.json')
+
+        if path.exists(filename):
+            with open(filename, 'r') as f:
+                self.popt_all = json.load(f)
+
+            for key in self.popt_all.keys():
+                self.popt_all[key] = np.array([val for val in self.popt_all[key].values()])
+        else:
+            self.popt_all = {}
 
-        x=np.asarray(self.qdf.loc[dur],dtype=np.float64)
+        # filename = path.join(self.store, str(self.code) + '_fits_cov.json')
+
+        # if path.exists(filename):
+        #     with open(filename, 'r') as f:
+        #         self.pcov_all = json.load(f)
+        # else:
+        #     self.pcov_all = {}
+
+    def fit_cdf(self, dur=None, plot=False):
+        """ Fit the cdf of the QDF data with a Generalized Extreme Value distribution
+
+        :param dur: the duration to fit
+        :param plot: if True, will plot the cdf with the fit
+        """
+
+        if dur is None:
+            return _('Bad duration - Retry !')
+        if dur not in durationstext:
+            return _('Bad duration - Retry !')
+
+        x=np.asarray(self.qdf.loc[dur], dtype=np.float64)
 
         def locextreme(x,a,b,c):
-            return genextreme.cdf(x,a,loc=b,scale=c)
+            return genextreme.cdf(x, a, loc=b, scale=c)
 
         def locextreme2(a):
             LL = -np.sum(genextreme.logpdf(x,a[0],loc=a[1],scale=a[2]))
             return LL
 
         popt = genextreme.fit(x)
-        popt,pcov=curve_fit(locextreme,x,freqndep,p0=popt)
+        popt, pcov = curve_fit(locextreme, x, freqndep, p0=popt)
 
         #ptest = minimize(locextreme2,popt,bounds=[[-10.,0.],[0.,100.],[0.,100.]])
 
-        perr = np.sqrt(np.diag(pcov))
+        # perr = np.sqrt(np.diag(pcov))
 
         fig=ax=None
         if plot:
@@ -447,27 +631,92 @@ class Qdf_IRM():
 
         return fig,ax,popt,pcov
 
-    def get_Tfromrain(self,Q,dur=dur1h):
-        return 1./self.stat.sf(Q,self.popt_all[dur][0],loc=self.popt_all[dur][1],scale=self.popt_all[dur][2])
+    def get_Tfromrain(self, Q, dur=dur1h):
+        """ Get the return period for a given quantity of rain
+
+        :param Q: the quantity of rain
+        :param dur: the duration
+        """
+
+        return 1./self.stat.sf(Q, self.popt_all[dur][0], loc=self.popt_all[dur][1], scale=self.popt_all[dur][2])
+
+    def get_rainfromT(self, T, dur= dur1h):
+        """ Get the quantity of rain for a given return period and duration
+
+        :param T: the return period
+        :param dur: the duration
+        """
 
-    def get_rainfromT(self,T,dur=dur1h):
         return self.stat.isf(1./T,self.popt_all[dur][0],loc=self.popt_all[dur][1],scale=self.popt_all[dur][2])
 
-    def get_MontanacoeffforT(self,return_period):
+    def get_MontanacoeffforT(self, return_period):
+        """ Get the Montana coefficients for a given return period
+
+        :param return_period: the return period
+        """
 
         if return_period in RT:
             return self.montanacoeff.loc[float(return_period)]
         else:
             return _('Bad RT - Retry !')
 
-    def plot_hyeto(self,durmax,T,r=.5):
+    def plot_hyeto(self, durmax, T, r=.5):
+        """ Plot the hyetogram for a given return period
+
+        :param durmax: the maximum duration of the hyetogram
+        :param T: the return period
+        :param r: the decentration coefficient
+        """
 
         fig,ax = self.montana.plot_hyeto(durmax,T,r)
-        ax.set_title(self.name + ' - code : ' + str(self.code))
+        ax.set_title(self._name + ' - code : ' + str(self._code))
 
         return fig
 
-    def plot_hyetos(self,durmax,r=.5):
+    def plot_hyetos(self, durmax, r=.5):
+        """ Plot the hyetograms for all return periods
+
+        :param durmax: the maximum duration of the hyetograms
+        :param r: the decentration coefficient
+        """
 
         fig,ax = self.montana.plot_hyetos(durmax,r)
-        ax.set_title(self.name + ' - code : ' + str(self.code))
+        ax.set_title(self._name + ' - code : ' + str(self._code))
+
+    def __str__(self) -> str:
+        """ Return the QDF data as a string """
+        return self.qdf.__str__()
+
+class QDF_Belgium():
+
+    def __init__(self, store_path= 'irm', ins:Literal['2018', '2019', '2025', 2018, 2019, 2025] = 2018) -> None:
+
+        self.localities = Localities(ins)
+        self.store_path = Path(store_path)
+
+        self.all = {}
+        for loc_ins in tqdm(self.localities.get_allcodes()):
+            loc = Qdf_IRM(store_path=str(self.store_path), code=loc_ins, localities=self.localities)
+            if loc.qdf is not None:
+                self.all[loc_ins] = loc
+
+    def __getitem__(self, key) -> Qdf_IRM:
+
+        if isinstance(key, int):
+            if key in self.all:
+                return self.all[key]
+            else:
+                logging.error(f"INS code {key} not found in the data")
+                return None
+
+        elif isinstance(key, str):
+            key = self.localities.get_INSfromname(key)
+            if key is not None:
+                if key in self.all:
+                    return self.all[key]
+                else:
+                    logging.error(f"INS code {key} not found in the data")
+                    return None
+            else:
+                logging.error(f"Name {key} not found in the data")
+                return None