cptec-monan 0.1.0__tar.gz

cptec_monan-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 framework-CPTEC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cptec_monan-0.1.0/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.4
+Name: cptec-monan
+Version: 0.1.0
+Summary: Módulo para distribuição de Modelos Numéricos.
+Home-page: https://monan-model.readthedocs.io/en/latest/
+Author: Framework
+Author-email: frameworkcptec@gmail.com
+License: MIT
+Project-URL: Código fonte, https://github.com/framework-CPTEC/CPTEC-user
+Project-URL: Download, https://github.com/framework-CPTEC/CPTEC-user
+Keywords: recuperação de dados modelos numericos CPTEC INPE
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: Portuguese (Brazilian)
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Software Development :: Internationalization
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: matplotlib
+Requires-Dist: netCDF4
+Requires-Dist: xarray
+Requires-Dist: dask
+Requires-Dist: tqdm
+Requires-Dist: gdal
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+Framework
+
+Descrição
+É um pacote Python para a distribuição de dados brutos dos Modelos Numéricos de forma segmentada/particionada. Com esse pacote o usuário não necessita fazer o Download de todo o volume bruto o pacote auxilia a manipular somente a sua necessidade.
+
+support Python >= 3.10.
+
+Documentação completa disponível em: [Read the Docs](https://monan-model.readthedocs.io/en/latest/)
+
+    

cptec_monan-0.1.0/README

@@ -0,0 +1,14 @@
+# Para utilização do framework CPTEC é necessário criar um ambiente no anaconda
+# e instalar todos pacotes listados abaixo:
+
+# Comando para criar ambiente
+conda create -n cptec python=3.10
+
+# Comando para ativar ambiente (toda vez que for utilizar a ferramenta é necessário ativar este ambiente)
+conda activate cptec
+
+# Instalação dos pacotes necessários obrigatorios
+conda install -c conda-forge xarray matplotlib pycurl cfgrib netcdf4 pynio xarray matplotlib pycurl cfgrib netcdf4 pynio dask ipywidgets
+
+# Caso vá executar em notebook é necessário o pacote ipykernel
+conda install -c anaconda ipykernel

cptec_monan-0.1.0/cptec_monan.egg-info/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.4
+Name: cptec-monan
+Version: 0.1.0
+Summary: Módulo para distribuição de Modelos Numéricos.
+Home-page: https://monan-model.readthedocs.io/en/latest/
+Author: Framework
+Author-email: frameworkcptec@gmail.com
+License: MIT
+Project-URL: Código fonte, https://github.com/framework-CPTEC/CPTEC-user
+Project-URL: Download, https://github.com/framework-CPTEC/CPTEC-user
+Keywords: recuperação de dados modelos numericos CPTEC INPE
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: Portuguese (Brazilian)
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Software Development :: Internationalization
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: matplotlib
+Requires-Dist: netCDF4
+Requires-Dist: xarray
+Requires-Dist: dask
+Requires-Dist: tqdm
+Requires-Dist: gdal
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+Framework
+
+Descrição
+É um pacote Python para a distribuição de dados brutos dos Modelos Numéricos de forma segmentada/particionada. Com esse pacote o usuário não necessita fazer o Download de todo o volume bruto o pacote auxilia a manipular somente a sua necessidade.
+
+support Python >= 3.10.
+
+Documentação completa disponível em: [Read the Docs](https://monan-model.readthedocs.io/en/latest/)
+
+    

cptec_monan-0.1.0/cptec_monan.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+LICENSE
+README
+setup.py
+cptec_monan.egg-info/PKG-INFO
+cptec_monan.egg-info/SOURCES.txt
+cptec_monan.egg-info/dependency_links.txt
+cptec_monan.egg-info/entry_points.txt
+cptec_monan.egg-info/requires.txt
+cptec_monan.egg-info/top_level.txt
+monanmodel/CPTEC_MONAN.py
+monanmodel/_dict_monan.py
+monanmodel/_version.py

cptec_monan-0.1.0/cptec_monan.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

cptec_monan-0.1.0/cptec_monan.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+monan_load = monanmodel.CPTEC_MONAN:main

cptec_monan-0.1.0/cptec_monan.egg-info/requires.txt

@@ -0,0 +1,8 @@
+numpy
+pandas
+matplotlib
+netCDF4
+xarray
+dask
+tqdm
+gdal

cptec_monan-0.1.0/cptec_monan.egg-info/top_level.txt

@@ -0,0 +1 @@
+monanmodel

cptec_monan-0.1.0/monanmodel/CPTEC_MONAN.py

@@ -0,0 +1,773 @@
+# import matplotlib.pyplot as plt
+from datetime  import datetime, timedelta
+import numpy as np
+import pandas as pd
+import argparse
+import io
+import xarray as xr
+import time, random, glob, shutil, os
+#import urllib
+import urllib.error
+import urllib.request
+from ._dict_monan import dict_monan as __dict_monan__ 
+from ._version import _version as __version__
+import warnings
+import json
+from osgeo import gdal, osr, ogr
+import time
+from tqdm import tqdm
+
+warnings.filterwarnings('ignore')
+
+class model(object):
+
+    def __init__(self, verbose=True):
+
+        """ 
+
+        Function to initialize the WRF model configurator, returns an object with the load function enabled for use.
+
+        Parameters
+        --------------------------------------------------------------------------------------------------------------------------------------
+
+        * Model: To configure WRF in new resolutions, change the parameter field
+        * Variables: To enable new variables, add the variable name and the corresponding name within the .idx or .inv
+        * Levels: Defines variables with a single level or multiple levels
+        * Area: During initialization, the Reduce field is added. When True, the parameters defined here are applied to zoom in on the desired area
+        * Transform: Performs transformation in the units of the variables to a common unit between the models.
+        * File: Name of the file available on the ftp
+        * Server: FTP server consumed by the application
+        --------------------------------------------------------------------------------------------------------------------------------------
+
+        Returns a model object
+
+        """
+        # Diretório atual
+        current_directory = os.path.dirname(os.path.abspath("__file__"))
+
+        self.dir_shp = os.path.join(current_directory, 'monanmodel/shp')
+
+        with open (os.path.join(self.dir_shp, 'shapefiles.json'),"r") as f: 
+            self.areas = json.load(f)
+
+        self.dict = __dict_monan__
+
+        self.levels = self.dict['model']['levels']
+        self.variables = list(self.dict['variables'])
+        self.stepModel = self.dict['model']['step']
+        self.area = {
+                       "northlat" :    90,
+                       "southlat" :    -90,
+                       "westlon" :    0,
+                       "eastlon" :    360,
+                       "invertedlat" : True
+        }
+        
+        
+        self.dict['area'].update({'reduce': False})
+        self.dict.update({'save_netcdf': False})
+        self.dict.update({'path_to_save': os.getcwd()})
+
+        self.local_path = f"INPE/{self.dict['model']['name']}/{self.dict['model']['parameter']}/brutos"
+        self.ftppath = f"{self.dict['model']['parameter']}/brutos"
+        
+        start = time.strftime("%Y%m%d", time.gmtime(time.time()))
+        end = (datetime.strptime(f'{start}',  '%Y%m%d') - timedelta(days=10)).strftime("%Y%m%d")
+        self.daterange =  pd.date_range(end, start)
+        self.frequency = [0,168]
+        self.var_level = [sigla  for sigla, level in self.dict['levels'].items() if level =='LVL']
+        self.var_surface = [sigla  for sigla, level in self.dict['levels'].items() if level =='SFC']
+
+        if verbose: 
+            print(f"\n#### {self.dict['model']['long_name']} ({self.dict['model']['parameter']}) #####\n")
+            print(f"Forecast data available for reading between {end} and {start}.\n")
+            print(f"Surface variables: {self.var_surface}.\n")
+            print(f"Level variables:   {self.var_level}.\n")
+            print(f"levels (hPa): {self.levels}.\n")
+            print(f"Frequency: every 3 hours  [0,3,6,...,264].\n")
+
+        self.session = random.random()
+        for folder in glob.glob(".temporary_files/*"):
+            if os.path.isdir(folder):
+                shutil.rmtree(folder)
+
+
+
+    def load(self, date=None, steps=[0], var=['t2m'], level=[1000]):
+        """
+        
+        The load function prepares the list of variables, levels and dates that will be loaded into memory.
+
+        During execution, a temporary directory is created to handle the files and is deleted as soon as the request is completed.
+
+        self.date is defined by the frequency at which the model makes its forecasts available, for WRF every 1 hour.
+
+        Parameters
+        ------------------------------------------------------------------------------------------------------------
+        date : Date of the initial condition date=YYYYMMDDHH, use HH for IC 00 and 12.
+        steps : Integer/Array of integers with the desired steps. where 0 is the initialization of the model [0,1, ... ,168], maximum value 168.
+        var: Array of strings with the names of the variables available for reading ['t2m', 'precip']
+        level: Array of integers with the levels available for each model [1000, 850]
+        ------------------------------------------------------------------------------------------------------------
+
+        load(date='2022082300', steps=[0,1,5,9], var=['t', 'precip'], level=[1000, 850])
+        load(date='2022082300', steps= 4, var=['t', 'precip'], level=[1000, 850])
+
+        --------------------------------------------------------------------------------------------
+
+        Returns an Xarray containing all the requested variables with the transformations contained in self.dict 
+
+        ------------------------------------------------------------------------------------------------------------     
+
+        """
+        if (isinstance(steps,int)) : steps = [h for h in range(0, steps+1, self.stepModel)]
+        #if (isinstance(steps,int)) : steps = [h for h in range(0, steps+1, 1)]
+        if (len(steps)<2) :
+            if len(var) == 1 and self.dict['types'][var[0]] == "FCT" and steps[0] == 0: 
+                steps = [h for h in range(0, 2, 1)]
+
+
+        if type(date) == int: date = str(date)
+        if date == None: date = datetime.today().strftime("%Y%m%d")
+
+        if type(level) == int: level = [level]
+        if type(var) == str: var = [var]
+
+        self.start_date = date
+        self.start_date = self.start_date.replace('/', '')
+        self.start_date = self.start_date.replace('-', '')
+        if len(self.start_date) == 8: self.start_date = f"{self.start_date}00"
+
+        #self.query_level = level
+        if level is not None:
+            # Converte para array numérico (se possível)
+            try:
+                level_array = np.array(level, dtype=int)
+                self.query_level = level_array * 100
+            except ValueError:
+                # Se não for numérico, mantém como está
+                self.query_level = level
+        else:
+            self.query_level = None
+
+        #print(f"Level for query: {self.query_level}")
+        print(f"Building request")
+        self.date = [(datetime.strptime(f'{self.start_date}',  '%Y%m%d%H') + timedelta(hours=int(h))).strftime("%Y%m%d%H") for h in steps]
+        self.year       = self.start_date[0:4]
+        self.mon        = self.start_date[4:6]
+        self.day        = self.start_date[6:8]
+        self.hour       = self.start_date[8:10]
+
+        self.query_variables = var
+
+        self.__getrange__()
+        if os.path.exists(f".temporary_files/{self.session}"): shutil.rmtree(f".temporary_files/{self.session}")
+        
+        return self.file
+
+
+    def __repr__(self):
+
+        """
+
+            Function to display definitions contained in the object, accessible through self.dict
+
+        """
+
+        print(f"Reduce area: {self.dict['area']['reduce']}")
+        print(f"Save netcdf: {self.dict['save_netcdf']}")
+        print(f"Path to save: {self.dict['path_to_save']}")
+        print(f"\n#### {self.dict['model']['long_name']} ({self.dict['model']['parameter']}) #####\n")
+        start = time.strftime("%Y%m%d", time.gmtime(time.time()))
+        end = (datetime.strptime(f'{start}',  '%Y%m%d') - timedelta(days=10)).strftime("%Y%m%d")
+        print(f"Forecast data available for reading between {end} and {start}.\n")
+        print(f"Surface variables: {self.var_surface}.\n")
+        print(f"Level variables:   {self.var_level}.\n")
+        print(f"levels (hPa): {self.levels}.\n")
+        print(f"Frequency: hourly frequency [0,1,2,...,22,23].\n")
+        print(f"To see more info use monan.help()")
+
+        return str('')    
+
+    
+    def help(self):
+
+        """
+
+            Function to display model information and their parameterizations.  
+
+        """
+        help(model)
+       
+
+    def read_variable(self, var, name, dt, pbar, outfile):
+        total_bytes = var.nbytes
+        start = time.time()
+        var_loaded = var.load()
+        pbar.update(total_bytes)
+        elapsed = time.time() - start
+        speed = total_bytes / elapsed
+        tqdm.write(f"{dt} - {name}: {total_bytes/1024/1024:.2f} MB, "
+               f"Tempo: {elapsed:.2f}s, Velocidade: {speed/1024/1024:.2f} MB/s")
+        var_loaded.to_netcdf(outfile)
+        return outfile
+
+
+    def __getrange__(self):
+        ds_no_level = []
+        ds_with_level = []
+        pathout = f".temporary_files/{self.session}"
+        os.makedirs(pathout, exist_ok=True)
+
+        try:
+
+            # calcula total de bytes de todos os steps e variáveis
+            total_bytes = 0
+            for dt in self.date:
+                netCDFfile = self.dict['file']['name'].format(self.start_date, dt)
+                netCDFfile = f"{self.ftppath}/{self.year}/{self.mon}/{self.day}/{self.hour}/{netCDFfile}"
+                url = f"{self.dict['server']['ftp']}/{netCDFfile}#mode=bytes"
+                ds = xr.open_dataset(url, chunks={})
+                for var in self.query_variables:
+                    if var in self.var_surface:
+                        total_bytes += ds[self.dict['variables'][var]].nbytes
+                    else:
+                        total_bytes += ds[self.dict['variables'][var]].sel(level=self.query_level).nbytes
+
+            # cria uma barra única para todo o download
+            with tqdm(total=total_bytes, unit="B", unit_scale=True, desc="Download total") as pbar:
+                for dt in self.date:
+                    netCDFfile = self.dict['file']['name'].format(self.start_date, dt)
+                    netCDFfile = f"{self.ftppath}/{self.year}/{self.mon}/{self.day}/{self.hour}/{netCDFfile}"
+                    url = f"{self.dict['server']['ftp']}/{netCDFfile}#mode=bytes"
+                    ds = xr.open_dataset(url, chunks={})
+                    for var in self.query_variables:
+                        if var in self.var_surface:
+                            outfile = f"{pathout}/recorte_{self.start_date}_{dt}_{var}_surface.nc"
+                            self.read_variable(ds[self.dict['variables'][var]], var, dt, pbar, outfile)
+                        else:
+                            outfile = f"{pathout}/recorte_{self.start_date}_{dt}_{var}_level.nc"
+                            self.read_variable(ds[self.dict['variables'][var]].sel(level=self.query_level), var, dt, pbar, outfile)
+
+
+
+            # junta todas as variáveis carregadas
+            ds_final = None
+            for var in self.query_variables:
+                if var in self.var_surface:
+                    files_surface = sorted(glob.glob(f"{pathout}/recorte_*_{var}_surface.nc"))
+                    if files_surface:
+                        ds_surface = xr.open_mfdataset(files_surface, combine="nested", concat_dim="Time", chunks={"Time": 1})
+                        ds_final = xr.merge([ds_final, ds_surface], join="outer", combine_attrs="override") if ds_final else ds_surface
+                else:
+                    files_levels = sorted(glob.glob(f"{pathout}/recorte_*_{var}_level.nc"))
+                    if files_levels:
+                        ds_levels = xr.open_mfdataset(files_levels, combine="nested", concat_dim="Time", chunks={"Time": 1})
+                        ds_final = xr.merge([ds_final, ds_levels], join="outer", combine_attrs="override") if ds_final else ds_levels
+
+
+            self.file = ds_final
+            return ds_final
+
+        except urllib.error.HTTPError as err:
+            print('File not available on server!')
+            self.file = None
+            return
+        except Exception as err:
+            print(err)
+            print(f"Unexpected {err=}, {type(err)=}")
+            self.file = None
+            return
+
+    def save_by_day(self, ds, prefix="MONAN", path="."):
+        # garante que o diretório existe
+        os.makedirs(path, exist_ok=True)
+
+        # extrai apenas a parte da data (YYYY-MM-DD)
+        dias_unicos = np.unique(pd.to_datetime(ds.Time.values).date)
+
+        for dia in dias_unicos:
+            ds_dia = ds.sel(Time=slice(str(dia), str(dia)))
+            outfile = os.path.join(path, f"{prefix}_{dia.strftime('%Y%m%d')}.nc")
+            ds_dia.to_netcdf(outfile, engine="netcdf4")
+            print(f"Saved: {outfile}")
+
+
+    def __curl__(self):
+
+        """
+        
+        The __curl__ function downloads the variables described for each record in self.setup, applies 
+        the transformations defined in self.dict['transform'] and returns in self.file an Xarray in memory with all the times of the requested variables.
+
+        When self.dict['save_netcdf'] == True a netcdf4 file will be saved with a copy of the request automatically.
+
+        
+        """
+
+        pathout = f".temporary_files/{self.session}"
+
+        os.makedirs(pathout, exist_ok=True)
+
+        fidx = glob.glob(f"{pathout}/*.idx")
+        if len(fidx) > 0:
+            [os.remove(f) for f in fidx]
+        
+        for _,row in self.setup.iterrows():
+           
+            grbfile = self.dict['file']['name'].format(row['start_date'], row['forecast_date'])
+            grbfile = f"{self.ftppath}/{self.year}/{self.mon}/{self.day}/{self.hour}/{grbfile}"
+            c = pycurl.Curl()
+            c.setopt(pycurl.URL,f"{self.dict['server']['ftp']}/{grbfile}")
+
+            outfile = self.dict['file']['name'].format(row['start_date'], row['forecast_date'])
+            lvl = row['level'].replace(" ", "")
+            outfile = f"{pathout}/{row['varname']}_{lvl}_{outfile}"
+
+            with open(outfile, "wb") as fout:
+                c.setopt(pycurl.WRITEDATA, fout)
+                c.setopt(c.RANGE, f"{row['lower']}-{row['upper']}") 
+                c.setopt(pycurl.VERBOSE, 0)
+                c.setopt(pycurl.FOLLOWLOCATION, 0)
+                c.perform()
+                c.close()
+            
+            fout.close()
+            
+            f = xr.open_dataset(outfile, engine='cfgrib')
+            f['time'] = datetime.strptime(row['forecast_date'],  '%Y%m%d%H')
+
+            v = list(f.keys())
+            var = outfile.split('/')[-1]
+            var = var.split('_')[0]
+            f = f.rename({v[0] : var})
+
+            if 'step': f = f.drop_vars('step')
+
+            if 'valid_time' in f: f = f.drop_vars('valid_time')
+            
+            if 'surface' in f:
+                
+                f = f.drop_vars('surface')
+
+            if 'isobaricInhPa' in f:
+                f = f.rename({'isobaricInhPa' : 'level'})
+                f = f.expand_dims(['level'])
+
+            if 'heightAboveGround' in f: f = f.drop_vars('heightAboveGround')
+
+            if 'heightAboveGroundLayer' in f: f = f.drop_vars('heightAboveGroundLayer')      
+            
+            if 'pressureFromGroundLayer' in f: f = f.drop_vars('pressureFromGroundLayer')      
+
+            if 'atmosphereSingleLayer' in f: f = f.drop_vars('atmosphereSingleLayer')
+
+            if 'meanSea' in  f: f = f.drop_vars('meanSea')
+
+            if 'depthBelowLandLayer' in  f: f = f.drop_vars('depthBelowLandLayer')  
+                
+            if 'middleCloudLayer' in  f: f = f.drop_vars('middleCloudLayer') 
+
+            if 'highCloudLayer' in  f: f = f.drop_vars('highCloudLayer') 
+
+            if 'lowCloudLayer' in  f: f = f.drop_vars('lowCloudLayer') 
+
+            f = f.expand_dims(['time'])
+            outnc = outfile.split('/')[-1]
+            outnc = outnc.split('.')[:-1][0]
+
+            # Transforma unidade
+            if var in self.dict['transform']:
+                tr = float(self.dict['transform'][var][1:])
+                op = self.dict['transform'][var][0]
+                f = eval(f'f {op} tr')
+                f[var].attrs = {
+                            "long_name" : self.dict['desc'][var]['name'],
+                            "units"  :       self.dict['desc'][var]['unit'],
+                            "standard_name": self.dict['desc'][var]['name']
+                    }
+
+
+            # self.xarray['lev'].attrs['units'] = 'hPa'
+            
+            if self.dict['area']['reduce'] ==  True:
+
+                lat1 = self.dict['area']['minlat']
+                lat2 = self.dict['area']['maxlat'] 
+                lon1 = self.dict['area']['minlon']
+                lon2 = self.dict['area']['maxlon']
+
+            
+                f2 = f.sel(latitude=slice(lat1, lat2), 
+                          longitude=slice(lon1, lon2)).copy()
+            else:
+                f2 = f
+
+            f2.to_netcdf(f'{pathout}/{outnc}.nc4', encoding={'time': {'dtype': 'i4'}})
+            f2.close()
+            f.close()
+  
+                 
+        gc.collect()
+        files = glob.glob(f"{pathout}/*.nc4")
+
+        f = xr.open_mfdataset(files,  combine='nested', parallel=False,  chunks={'latitude': 150, 'longitude': 150})
+        
+        # Transform accumulated precipitation
+        # TimeStamp 0 and 1 without modification
+        if 'precip' in f:
+
+            arr = []
+            for dt in np.arange(len(f.time)):
+                
+                if dt <= 1:
+                    arr.append(f.isel(time=dt)[['precip']])
+                else:
+                    fout = f.isel(time=dt)[['precip']] - f.isel(time=dt-1)[['precip']]
+                    fout = fout.assign_coords({'time': f.time[dt]})
+                    fout = fout.expand_dims('time')
+                    arr.append(fout)
+            
+            f['precip'] = xr.concat(arr, dim="time")['precip']
+
+        f.attrs = {
+                            "center" :  "National Institute for Space Research - INPE",
+                            "model"  :  f"The Weather Research and Forecasting ({self.dict['model']['parameter']})"
+        }
+        f.time.encoding['units'] = "Seconds since 1970-01-01 00:00:00"
+
+        field = f.load()
+
+        if self.dict['save_netcdf'] == True:
+
+            pathout = f"{self.dict['path_to_save']}/{self.local_path}/{self.year}/{self.mon}/{self.day}/{self.hour}"
+            os.makedirs(pathout, exist_ok=True)
+            ncout = self.dict['file']['name'].format(row['start_date'], row['forecast_date'])
+            ncout = ncout.replace(f"{self.dict['file']['format']}","nc4")
+
+            if os.path.exists(f"{pathout}/{ncout}"): os.remove(f"{pathout}/{ncout}")
+            field.to_netcdf(f"{pathout}/{ncout}", encoding={'time': {'dtype': 'i4'}})
+
+        f.close()
+
+        gc.collect()
+        self.file = field
+
+    def get_var_description(self, var=None):
+        """
+        Retrieve the description of meteorological variables as a pandas DataFrame.
+    
+        If a specific variable name is provided, returns a DataFrame with a single row
+        containing its name and unit. If no variable is specified, returns a DataFrame
+        with all available variables and their respective metadata.
+
+        Parameters
+        ----------
+        var : str, optional
+            The key of the variable to retrieve its description. If None, all variable
+            descriptions are returned.
+
+        Returns
+        -------
+        pandas.DataFrame
+            A DataFrame with columns ['Variable', 'Name', 'Unit'].
+            - If `var` is provided: the DataFrame contains one row.
+            - If `var` is None: the DataFrame contains all available variables.
+
+        Examples
+        --------
+        >>> gfs.get_var_description('t')
+        Variable        Name       Unit
+        0        t  Temperature         C
+
+        >>> gfs.get_var_description()
+        Variable                   Name  Unit
+        0        t            Temperature    C
+        1        u  u-component of wind  m/s
+        2        v  v-component of wind  m/s
+        """
+        if var:
+            data = [
+                {'Variable': var, 
+                'Name': self.dict['desc'][var]['name'], 
+                'Unit': self.dict['desc'][var]['unit']
+                }
+
+            ]
+        else:
+            data = [
+                {'Variable': k, 'Name': v['name'], 'Unit': v['unit']}
+                for k, v in self.dict['desc'].items()
+            ]
+        return pd.DataFrame(data)
+
+    def list_levels(self):
+        print("Available levels:", self.levels)
+
+    def list_variables(self):
+        print("Variables at the surface:", self.var_surface)
+        print("Variables at different levels:", self.var_level)
+
+
+
+    def list_areas(self, tipo=None):
+
+        """
+        Function to display and return the abbreviations used in the shapefile definitions to clip the area.
+
+        Parameters
+        ----------
+        tipo      - Required  : Type to filter areas 
+                                (e.g. 'paises', 'regioes',
+                                   'estados', 'bacias', 'biomas') (Str)
+
+
+        Returns
+        -------
+        json : Json
+            The newly Json.
+
+        """
+        if tipo==None:
+            print("Usage: list_areas( 'continentes'/'paises'/'regioes'/'estados'/'bacias'/'biomas') e.g. monan.list_areas('paises') ")
+            return None
+
+        lista = self.areas['shp'][0][tipo][0]
+        print("Usage: type_abbreviation e.g. shp='estados_sp'")
+        print("Area    -   type_abbreviation")
+        for item in lista: 
+            print(f"{lista[item][1]} - {tipo}_{item.lower()}")
+
+        #return self.areas['shp'][0][tipo][0]
+        
+    def makeMask(self,lon,lat,res,shapefile):
+        source_ds = ogr.Open(shapefile)
+        source_layer = source_ds.GetLayer()
+        # Cruar Raster na memoria
+        mem_ds = gdal.GetDriverByName('MEM').Create('', lon.size, lat.size, gdal.GDT_Byte)
+        mem_ds.SetGeoTransform((lon.min(), res, 0, lat.max(), 0, -res))
+        band = mem_ds.GetRasterBand(1)
+        # Rasterizar shapefile para grid
+        gdal.RasterizeLayer(mem_ds, [1], source_layer, burn_values=[1])
+        # Obter rasterizado shapefile como numpy array
+        array = band.ReadAsArray()
+        # Liberar Memoria
+        mem_ds = None
+        band = None
+        return array
+
+    def load_shape(self, date=None, steps=[0], var=['t2m'], level=[1000], shp="estados_sp"):
+        """
+        The load_shape function loads the requested data from the CPTEC server 
+        into memory and cuts the defined shape.
+
+
+            Parameters
+            ----------
+            product      - Optional  : 'climatology', 'hourly', 'hourly_now', 'daily' (Str)
+            accumulated  - Optional  : 'monthly_average', 'monthly_accumulated_yearly',
+                                   'year_accumulated', 'monthly_accumulated', 'daily_average', 
+                                   'monthly_average_yearly'
+
+                                   None for 'hourly', 'hourly_now', 'daily' (Str)
+
+            year         - Optional  : year (Str)
+            month        - Optional  : month (Str)
+            day          - Optional  : day (Str)
+            hour         - Optional  : hour (Str)
+            shp          - Optional  : shape (Str) -  see function list_areas()
+     
+        ------------------------------------------------------------------------------------------------------------       
+
+        load(product='daily', accumulated=None, year="2024", month="01", day="01", hour="00")
+
+        ------------------------------------------------------------------------------------------------------------       
+        
+            Returns
+            -------
+            dataset : Dataset
+                The newly created dataset.
+
+        """
+        if type(var) == str: var = [var]
+        tp_area, sigla = shp.split('_')
+        if (isinstance(steps,int)) : steps = [h for h in range(0, steps+1, self.stepModel)]
+        #if (isinstance(steps,int)) : steps = [h for h in range(0, steps+1, 1)]
+        if (len(steps)<2) :
+            if len(var) == 1 and self.dict['types'][var[0]] == "FCT" and steps[0] == 0: 
+                steps = [h for h in range(0, 2, 1)]
+
+        #produto = (f"{product}_{accumulated}") if accumulated else (f"{product}")
+
+        #variavel= self.dict['products'][produto]['variable']
+
+
+        tipo_lon=360
+        tipo=tp_area
+        try:
+            idx = self.areas['shp'][0][tp_area][0][sigla.upper()][0]
+        except Exception:
+            raise ValueError("Area shape not found - usage: mon.list_areas()")
+
+
+        ds = self.load(  date=date, steps=steps, var=var, level=level)
+        #print(ds)
+        #ds = ds.assign_coords(longitude=((ds.longitude + 180) % 360) - 180)
+        ds = ds.assign_coords(longitude=np.round(((ds.longitude + 180) % 360) - 180, 6))
+        # Remove duplicatas
+        _, index = np.unique(ds.longitude, return_index=True)
+        ds = ds.isel(longitude=index)
+        # Ordena
+        ds = ds.sortby("longitude")
+
+        lons = ds.longitude.values
+        #lons = np.round(lons, 1)  # arredonda para 1 casa decimal
+        #lons_180 = np.round(((lons + 180) % 360) - 180, 1)
+        #lons = lons -180
+        lats = ds.latitude.values  
+        #lats = lats[::-1]
+        cellsize = lons[:][1] - lons[:][0]
+
+        #print(f"Lats: {lats}")
+        #print(f"Lons: {lons}")
+        #print(f"Cellsize: {cellsize}")
+
+        # Pegar shapefile da mascara
+        maskShapefile = f'{self.dir_shp}/{tipo}/{idx}.shp'
+        print(maskShapefile)
+        # Gerar Mascara
+        mask = self.makeMask(lons,lats,cellsize,maskShapefile)
+        mask = mask[::-1]
+        # Pegar valores tmax
+        for vvar in var:
+            variavel = self.dict['levels'][vvar]
+            if variavel == 'LVL':
+                for idxs, stp in enumerate(steps):
+                    for idxl, lev in enumerate(level):
+                        if lev != "surface":
+                            var = ds[self.dict['variables'][vvar]][idxs][idxl].values
+                            var2=np.ma.masked_array(var,np.logical_not(mask))
+                            ds[self.dict['variables'][vvar]][idxs][idxl] = var2[:,:] 
+                            #ds = ds.assign_coords(longitude=((ds.longitude - 180)))               
+            else:
+                for idx, stp in enumerate(steps):
+                    var = ds[self.dict['variables'][vvar]][idx].values
+                    var2=np.ma.masked_array(var,np.logical_not(mask))
+                    ds[self.dict['variables'][vvar]][idx] = var2[:,:] 
+                    #ds = ds.assign_coords(longitude=((ds.longitude - 180)))
+
+
+
+        self.file = ds
+        return self.file
+
+
+
+def __clean__():
+
+    """
+
+    When the request process is interrupted the tool will not remove temporary files,
+    this function removes all temporary directories older than 2 days on disk.
+
+
+    """
+       
+    if os.path.exists(f".temporary_files"):
+
+        today = datetime.today()
+           
+        files = glob.glob(".temporary_files/0.*")
+        for f in files:
+            duration = today - datetime.fromtimestamp(os.path.getmtime(f))
+            if duration.days >= 2:
+                shutil.rmtree(f) 
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='monan_load - load MONAN data from CPTEC dataserver.',
+        formatter_class=argparse.RawTextHelpFormatter
+    )
+
+    parser.add_argument('--date', '-d', type=str, dest='date',
+                        help='Initial condition date YYYYMMDDHH', required=False, default=None)
+
+    parser.add_argument('--steps', '-s', type=int, nargs='+',
+                    dest='steps', help='Lista de steps (ex.: 2 ou 0 2 4)')
+
+    parser.add_argument('--range', '-r', type=int,
+                    dest='range', help='Step máximo para baixar de 0 até N')
+
+    # var pode ser uma ou várias variáveis
+    parser.add_argument('--var', '-v', type=str, nargs='+',
+                        dest='var', default=['t2m'],
+                        help='Variable(s) to load (e.g. t2m ou t2m t)')
+
+    # level pode ser um ou vários níveis
+    parser.add_argument('--level', '-l', type=int, nargs='+',
+                        dest='level', default=[1000],
+                        help='Level(s) to load (e.g. 1000 ou 1000 850)')
+
+    parser.add_argument('--shape', '-shp', type=str, dest='shape',
+                        help='Shape abbreviation (e.g. estados_sp)', default=None)
+
+    parser.add_argument('--prefix', '-p', type=str, dest='prefix',
+        default='MONAN',
+        help='File name prefix for saved NetCDF files (default: MONAN)')
+
+    parser.add_argument('--path', '-o', type=str, dest='path',
+        default='.',
+        help='Output directory where NetCDF files will be saved (default: current directory)')
+
+
+#    parser.add_argument('--areas', action='store_true',
+#                        help='List available areas (continentes, paises, regioes, estados, bacias, biomas)')
+    parser.add_argument('--areas', type=str, choices=['continentes','paises','regioes','estados','bacias','biomas'],
+                    help='List available areas by type')
+
+    parser.add_argument('--list_levels', action='store_true', help='List available levels') 
+                       
+    parser.add_argument('--list_vars', action='store_true', help='List available variables')
+
+    args = parser.parse_args()
+    mon = model(verbose=False)
+
+    if args.range is not None:
+        steps = list(range(0, args.range+1, mon.stepModel))
+        print(f"Requested steps: {steps}")
+    elif args.steps is not None:
+        steps = args.steps
+    else:
+        steps = [0]
+
+    if args.areas:
+        mon.list_areas(tipo=args.areas)   # ou outro tipo padrão
+        return
+
+    if args.list_levels:
+        mon.list_levels()  
+        return
+
+    if args.list_vars:
+        mon.list_variables()   
+        return
+
+
+
+
+    if args.shape:
+        ds = mon.load_shape(date=args.date, steps=steps, var=args.var, level=args.level, shp=args.shape)
+        ds.to_netcdf("tst.nc")
+    else:
+        ds = mon.load(date=args.date, steps=steps, var=args.var, level=args.level)
+
+    if ds is not None:
+        mon.save_by_day(ds, prefix=args.prefix, path=args.path)
+
+
+
+if __name__ == '__main__':
+    main()
+
+    

cptec_monan-0.1.0/monanmodel/_dict_monan.py

@@ -0,0 +1,177 @@
+
+dict_monan  = {   
+            
+            "model"    : {
+                    "name" : "MONAN",
+                    "parameter" : "10km",
+                    "long_name" : "Model for Ocean-laNd-Atmosphere PredictioN - (MONAN)",
+                    "step" : 3,
+                    "levels" : ["1000","925","850","775","700","500","400","300", "250", "200", "150", "100", "70", "50", "30", "20", "10", "3"]
+            },
+            "variables" :  {
+                    "t" : "temperature",
+                    "u" : "uzonal",
+                    "v" : "umeridional",
+                    "rh" : "relhum",
+                    "g" : "zgeo",
+                    "omega" : "omega",
+                    "u10m" : "u10",
+                    "v10m" : "v10",
+                    "t2m" : "t2m",
+                    "slp" : "mslp",
+                    "psfc" : "surface_pressure",
+                    "landmask" : "landmask",
+                    "sbcape" : "cape",
+                    "sbcin" : "cin",
+                    "pw" : "precipw",
+                    "precip" : "rainc",
+                    "spechum" : "spechum",
+                    "rainnc" : "rainnc",
+                    "acswdnb" : "acswdnb",
+                    "aclwupb":"aclwupb",
+                    "aclwupt":"aclwupt",
+                    "q2":"q2",
+                    "hfx":"hfx",
+                    "lh":"lh",
+                    "cldfrac_tot_UPP":"cldfrac_tot_UPP",
+                    "terrain":"ter"
+                        },
+            "levels" :   {  
+                        "t" : "LVL",
+                        "u" : "LVL",
+                        "v" : "LVL",
+                        "rh" : "LVL",
+                        "g" : "LVL",
+                        "omega" : "LVL",
+                        "u10m" : "SFC",
+                        "v10m" : "SFC",
+                        "t2m" : "SFC",
+                        "slp" : "SFC",
+                        "psfc" : "SFC",
+                        "landmask" : "SFC",
+                        "sbcape" : "SFC",
+                        "sbcin" : "SFC",
+                        "pw" : "SFC",
+                        "precip" : "SFC",
+                        "spechum" : "LVL",
+                        "rainnc" : "SFC",
+                        "acswdnb" : "SFC",
+                        "aclwupb":"SFC",
+                        "aclwupt":"SFC",
+                        "q2":"SFC",
+                        "hfx":"SFC",
+                        "lh":"SFC",
+                        "cldfrac_tot_UPP":"SFC",
+                        "terrain":"SFC"
+            },
+            "types" :   {  
+                        "t" : "ALL",
+                        "u" : "ALL",
+                        "v" : "ALL",
+                        "rh" : "ALL",
+                        "g" : "ALL",
+                        "omega" : "ALL",
+                        "fcor"  : "ALL",
+                        "u10m" : "ALL",
+                        "v10m" : "ALL",
+                        "t2m" : "ALL",
+                        "slp" : "ALL",
+                        "psfc" : "ALL",
+                        "landmask" : "ALL",
+                        "sbcape" : "ALL",
+                        "sbcin" : "ALL",
+                        "pw" : "ALL",
+                        "precip" : "FCT",
+                        "cssf" : "FCT",
+                        "clsf" : "FCT",
+                        "gust" : "FCT",
+                        "refc":"ALL",
+                        "vis":"ALL",
+                        "pot":"ALL",
+                        "dpt":"ALL",
+                        "spfh":"ALL",
+                        "mslet":"ALL",
+                        "refd":"ALL",
+                        "tsoil":"ALL",
+                        "soilw":"ALL",
+                        "soill":"ALL",
+                        "weasd":"ALL",
+                        "snowc":"ALL",
+                        "acpcp":"ALL",
+                        "ncpcp":"ALL",
+                        "ssrun":"ALL",
+                        "bgrun":"ALL",
+                        "gflux":"ALL",
+                        "lftx":"ALL",
+                        "lcdc":"ALL",
+                        "mcdc":"ALL",
+                        "hcdc":"ALL",
+                        "tcdc":"ALL",
+                        "dswrf":"ALL",
+                        "dlwrf":"ALL",
+                        "uswrf":"ALL",
+                        "ulwrf":"ALL",
+                        "hlcy":"ALL",
+                        "ustm":"ALL",
+                        "vstm":"ALL",
+                        "4lftx":"ALL",
+                        "hpbl":"ALL",
+                        "plpl":"ALL",
+                        "land":"ALL"
+            },  
+            "desc" :  {
+                    "t" : { "name" : "Temperature interpolated to isobaric surfaces defined in t_iso_levels", "unit": "C"},
+                    "u" : { "name" : "Reconstructed zonal wind at cell centers, interpolated to isobaric surfaces defined in u_iso_levels", "unit": "m/s"},
+                    "v" : { "name" : "Reconstructed meridional wind at cell centers, interpolated to isobaric surfaces defined in u_iso_levels", "unit": "m/s"},
+                    "rh" : { "name" : "Relative humidity interpolated to isobaric surfaces defined in t_iso_levels", "unit": "%"},
+                    "g" : { "name" : "Geopotential height interpolated to isobaric surfaces defined in t_iso_levels", "unit": "gpm"},
+                    "omega" : { "name" : "Omega interpolated to isobaric surfaces defined in t_iso_levels", "unit": ""},
+                    "u10m" : { "name" : "10-meter zonal wind", "unit": "m/s"},
+                    "v10m" : { "name" : "10-meter meridional wind", "unit": "m/s"},
+                    "t2m" : { "name" : "2-meter temperature", "unit": "C"},
+                    "slp" : { "name" : "Mean sea-level pressure", "unit": "Pa"},
+                    "psfc" : { "name" : "Diagnosed surface pressure", "unit": "Pa"},
+                    "landmask" : { "name" : "land-ocean mask", "unit": "gpm"},
+                    "sbcape" : { "name" : "Convective available potential energy", "unit": "J/kg"},
+                    "sbcin" : { "name" : "Convective inhibition", "unit": "J/kg"},
+                    "pw" : { "name" : "Precipitable Water", "unit": "kg/m^2"},
+                    "precip" : { "name" : "Accumulated Convective Precipitation", "unit": "kg/m^2"},
+                    "spechum" : "Specific humidity interpolated to isobaric surfaces defined in t_iso_levels",
+                    "rainnc" : "Accumulated Total grid-scale Precipitation",
+                    "acswdnb" : "Accumulated all-sky downward surface shortwave radiation flux",
+                    "aclwupb":"Accumulated all-sky upward surface longwave radiation flux",
+                    "aclwupt":"accumulated all-sky upward top-of-the-atmosphere longwave radiation flux",
+                    "q2":"2-meter specific humidity",
+                    "hfx":"Upward heat flux at the surface",
+                    "lh":"Latent heat flux at the surface",
+                    "cldfrac_tot_UPP":"Total cloud fraction using UPP column max method",
+                    "terrain":"Terrain height"
+            },
+            "area"    : {
+                        "minlat" :    -90,
+                        "maxlat" :     90,
+                        "minlon" :    0,
+                        "maxlon" :    360
+            },
+            "transform" : {
+                        "t"         : "-273.15",
+                        "omega"     :     "*10",
+                        "t2m"       : "-273.15",
+                        "slp"       :    "/100",
+                        "psfc"      :    "/100",
+                        "pot"       : "-273.15",
+                        "dpt"       : "-273.15",
+                        "tsoil"     : "-273.15",
+                        "lftx"      : "-273.15",
+                        "4lftx"     : "-273.15"                                                                                                
+            },
+            "file"    : {
+                        "name"   :     "MONAN_DIAG_G_POS_GFS_{}_{}.00.00.x5898242L55.nc",
+                        "format" :     "nc"
+            },
+            "server":   {
+                        "ftp"    :     "https://dataserver.cptec.inpe.br/dataserver_modelos/monan"
+            }
+    
+} 
+

cptec_monan-0.1.0/monanmodel/_version.py

@@ -0,0 +1 @@
+_version = '0.0.1'

cptec_monan-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

cptec_monan-0.1.0/setup.py

@@ -0,0 +1,55 @@
+
+from setuptools import setup
+import os
+
+
+# User-friendly description from README.md
+current_directory = os.path.dirname(os.path.abspath(__file__))
+try:
+    with open(os.path.join(current_directory, 'README.md'), encoding='utf-8') as f:
+        long_description = f.read()
+except Exception:
+    long_description = ''
+
+setup(
+    name = 'cptec-monan',
+    version = '0.1.0',
+    author = 'Framework',
+    author_email = 'frameworkcptec@gmail.com',
+    packages = ['monanmodel'],
+    install_requires = ['numpy','pandas','matplotlib', 'netCDF4',  'xarray', 'dask', 'tqdm', 'gdal'],
+    description = 'Módulo para distribuição de Modelos Numéricos.',
+    long_description="""Framework
+
+Descrição
+É um pacote Python para a distribuição de dados brutos dos Modelos Numéricos de forma segmentada/particionada. Com esse pacote o usuário não necessita fazer o Download de todo o volume bruto o pacote auxilia a manipular somente a sua necessidade.
+
+support Python >= 3.10.
+
+Documentação completa disponível em: [Read the Docs](https://monan-model.readthedocs.io/en/latest/)
+
+    \n""",
+    entry_points={
+        'console_scripts': [
+            'monan_load = monanmodel.CPTEC_MONAN:main'
+        ]
+    },
+    long_description_content_type='text/markdown',
+    url = 'https://monan-model.readthedocs.io/en/latest/',
+    project_urls = {
+        'Código fonte': 'https://github.com/framework-CPTEC/CPTEC-user',
+        'Download': 'https://github.com/framework-CPTEC/CPTEC-user'
+    },
+    license = 'MIT',
+    keywords = 'recuperação de dados modelos numericos CPTEC INPE',
+    classifiers = [
+        'Development Status :: 5 - Production/Stable',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Natural Language :: Portuguese (Brazilian)',
+        'Operating System :: OS Independent',
+        'Topic :: Software Development :: Internationalization',
+        'Topic :: Scientific/Engineering :: Physics'
+    ],
+    python_requires='>=3.10',
+)