py-ewr 2.1.9__py3-none-any.whl → 2.2.1__py3-none-any.whl

py_ewr/data_inputs.py

@@ -138,6 +138,19 @@ def get_NSW_codes() -> pd.DataFrame:
     
     return metadata
 
+def get_iqqm_codes() -> dict:
+    '''
+    Load metadata file for Macquarie containing model nodes
+    and gauges they correspond to
+
+    Returns:
+        dict: dict for linking model nodes to gauges
+    '''
+
+    metadf = pd.read_csv( BASE_PATH / 'model_metadata/iqqm_stations.csv', dtype=str)
+    metadata = metadf.set_index(metadf.columns[0]).to_dict()[metadf.columns[1]]
+    return metadata
+
 def get_level_gauges() -> tuple:
     '''Returning level gauges with EWRs
 

py_ewr/io.py

@@ -0,0 +1,24 @@
+import xarray as xr
+from pandas import DataFrame as Dataframe
+
+
+def read_netcdf_as_dataframe(netcdf_path: str) -> Dataframe:
+    dataset = xr.open_dataset(netcdf_path, engine='netcdf4')
+    df = dataset.to_dataframe()
+    dataset.close()
+
+    return df
+
+
+def save_dataframe_as_netcdf(df, output_path: str) -> None:
+    # Convert DataFrame to Xarray Dataset
+    ds = xr.Dataset.from_dataframe(df)
+    
+    # Modify variable names to ensure they are valid for NetCDF
+    for var_name in ds.variables:
+        new_var_name = var_name.replace(" ", "_")  # Replace spaces with underscores
+        new_var_name = ''.join(c for c in new_var_name if c.isalnum() or c == "_")  # Remove non-alphanumeric characters
+        ds = ds.rename({var_name: new_var_name})
+
+    # Save the modified Xarray Dataset as a NetCDF file
+    ds.to_netcdf(output_path)

py_ewr/model_metadata/iqqm_stations.csv

@@ -0,0 +1,11 @@
+IQQM,gauge
+229,421023
+42,421001
+464,421011
+240,421019
+266,421146
+951,421090
+487,421022
+130,421012
+171,421004
+

py_ewr/parameter_metadata/parameter_sheet.csv

@@ -3306,9 +3306,9 @@ PU_0000999,Border Rivers and Moonie Long-term watering plan,Border Rivers,Kanown
 PU_0000999,Border Rivers and Moonie Long-term watering plan,Border Rivers,Goondiwindi,416201A,NE,7,6,,,,1,1,1,5000,10000,4,,,F,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-NF2-NF3-NF4-NF5-NF6-NF8-NF9-NV1-WB1-EF1-EF2-EF3-EF5 ,NF_NV_WB_EF,QLD,Border Rivers,QLD Border Rivers
 PU_0000999,Border Rivers and Moonie Long-term watering plan,Border Rivers,Goondiwindi,416201A,PP,7,6,,,,1,1,1,5000,10000,4,,,F,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-NF2-NF3-NF4-NF5-NF6-NF8-NF9-NV1-WB1-EF1-EF2-EF3-EF5 ,NF_NV_WB_EF,QLD,Border Rivers,QLD Border Rivers
 PU_0000999,Border Rivers and Moonie Long-term watering plan,Border Rivers,Goondiwindi,416201A,rANA,10,2,,,,2,1,1,5000,10000,2,,,F,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-NF2-NF3-NF4-NF5-NF6-NF8-NF9-NV1-WB1-EF1-EF2-EF3-EF5 ,NF_NV_WB_EF,QLD,Border Rivers,QLD Border Rivers
-PU_0000991,Condamine and Balonne Long-term watering plan,Condamine,Wilby Wilby,422016,BBR1_a,7,6,,,,1,90,90,100,1000000,4,,422034,V,,1.08,154000,90,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-NF2-NF3-NF4-NF5-NF6-NF8-NF9-NV1-NV2-NV4-WB1-WB2-WB5-EF1-EF2-EF3-EF5-EF6-EF7 ,NF_NV_WB_EF,NSW,Condamine-Balonne,Intersecting Streams
-PU_0000991,Condamine and Balonne Long-term watering plan,Condamine,Wilby Wilby,422016,BBR1_b,7,6,,,,1,10,10,100,1000000,4,,422034,V,,1.08,20000,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-NF2-NF3-NF4-NF5-NF6-NF8-NF9-NV1-NV2-NV4-WB1-WB2-WB5-EF1-EF2-EF3-EF5-EF6-EF7 ,NF_NV_WB_EF,NSW,Condamine-Balonne,Intersecting Streams
-PU_0000991,Condamine and Balonne Long-term watering plan,Condamine,Wilby Wilby,422016,BBR2,7,6,76,,,1,1,1,0,1000000,,,,V,,,25000,60,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-NF2-NF3-NF4-NF5-NF6-NF8-NF9-NV1-NV2-NV4-WB1-WB2-WB5-EF1-EF2-EF3-EF5-EF6-EF7 ,NF_NV_WB_EF,NSW,Condamine-Balonne,Intersecting Streams
+PU_0000991,Condamine and Balonne Long-term watering plan,Condamine,Wilby Wilby,422016,BBR1_a,7,6,,,,1,90,90,100,1000000,4,,422034,V,,1.08,154000,,90,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-NF2-NF3-NF4-NF5-NF6-NF8-NF9-NV1-NV2-NV4-WB1-WB2-WB5-EF1-EF2-EF3-EF5-EF6-EF7 ,NF_NV_WB_EF,NSW,Condamine-Balonne,Intersecting Streams
+PU_0000991,Condamine and Balonne Long-term watering plan,Condamine,Wilby Wilby,422016,BBR1_b,7,6,,,,1,10,10,100,1000000,4,,422034,V,,1.08,20000,,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-NF2-NF3-NF4-NF5-NF6-NF8-NF9-NV1-NV2-NV4-WB1-WB2-WB5-EF1-EF2-EF3-EF5-EF6-EF7 ,NF_NV_WB_EF,NSW,Condamine-Balonne,Intersecting Streams
+PU_0000991,Condamine and Balonne Long-term watering plan,Condamine,Wilby Wilby,422016,BBR2,7,6,76,,,1,1,1,0,1000000,,,,V,,,25000,,60,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-NF2-NF3-NF4-NF5-NF6-NF8-NF9-NV1-NV2-NV4-WB1-WB2-WB5-EF1-EF2-EF3-EF5-EF6-EF7 ,NF_NV_WB_EF,NSW,Condamine-Balonne,Intersecting Streams
 PU_0000992,Warrego Paroo Bulloo and Nebine Long-term watering plan,Condamine,Nebine Creek at Roseleigh Crossing,422502A,DR1,7,6,100,,,1,1,1,2,,1,,,F,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-EF1-EF2 ,NF_EF,QLD,Condamine-Balonne,Nebine
 PU_0000992,Warrego Paroo Bulloo and Nebine Long-term watering plan,Condamine,Nebine Creek at Roseleigh Crossing,422502A,DR2,7,6,,,,1,1,1,2540,,,,,F,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,F,NF1-NF2-NF3-NF4-NF5-NF6-NF8-NF9-NV1-WB1-EF1-EF2-EF3-EF5 ,NF_NV_WB_EF,QLD,Condamine-Balonne,Nebine
 PU_0000992,Warrego Paroo Bulloo and Nebine Long-term watering plan,Condamine,Nebine Creek at Roseleigh Crossing,422502A,FD1,7,6,,,,1,8,8,8,,,,,F,,,,,,,,,,,,,,,,,,,,,,,548,,,,,,,,,,,,F,NF1-EF1-EF2 ,NF_EF,QLD,Condamine-Balonne,Nebine

py_ewr/scenario_handling.py

@@ -8,6 +8,8 @@ import logging
 
 import pandas as pd
 from tqdm import tqdm
+import xarray as xr
+import netCDF4
  
 log = logging.getLogger(__name__)
 log.addHandler(logging.NullHandler())
@@ -15,6 +17,61 @@ log.addHandler(logging.NullHandler())
 
 from . import data_inputs, evaluate_EWRs, summarise_results
 #----------------------------------- Scenario testing handling functions--------------------------#
+def is_valid_netcdf_file(file_path: str) -> bool:
+    try:
+        with netCDF4.Dataset(file_path, 'r'):
+            # If the file opens successfully, it's a valid NetCDF file
+            return True
+    except Exception as e:
+        # If an exception is raised, it's not a valid NetCDF file
+        return False
+    
+
+def unpack_netcdf_as_dataframe(netcdf_file: str) -> pd.DataFrame:
+    '''Ingesting netCDF files and outputting as dataframes in memory.
+    # Example usage:
+    # df = unpack_netcdf_as_dataframe('your_file.nc')
+    
+    Args:  
+        netcdf_file (str): location of netCDF file
+
+    Results:
+        pd.Dataframe: netCDF file converted to dataframe
+    '''
+    try:
+        # Check if the file is a valid NetCDF file
+        if not is_valid_netcdf_file(netcdf_file):
+            raise ValueError("Not a valid NetCDF file.")
+        
+        # Open the NetCDF file
+        dataset = xr.open_dataset(netcdf_file, engine='netcdf4')
+        
+        # Check if the dataset is empty
+        if dataset is None:
+            raise ValueError("NetCDF dataset is empty.")
+        
+        # extract the bits we actually can use
+            # Some of this needs to move/get cleaned up
+        iqqm_dict = data_inputs.get_iqqm_codes()
+        # the nodes are ints, but the above is str
+        ints_list = list(map(int, list(iqqm_dict)))
+        
+        # Is there any reason to do these in one step?
+        dataset = dataset.sel(node=dataset['node'].isin(ints_list))
+        dataset = dataset[['Simulated flow']]
+
+        # Convert to DataFrame
+        df = dataset.to_dataframe()
+        
+        # Close the dataset
+        dataset.close()
+        
+        return df
+    except Exception as e:
+        # Handle any exceptions that may occur
+        print(f"Error: {str(e)}")
+        return None
+
 
 def unpack_model_file(csv_file: str, main_key: str, header_key: str) -> tuple:
     '''Ingesting scenario file locations of model files with all formats (excluding standard timeseries format), seperates the flow data and header data
@@ -280,6 +337,52 @@ def cleaner_standard_timeseries(input_df: pd.DataFrame, ewr_table_path: str = No
             log.info('Could not identify gauge in column name:', gauge, ', skipping analysis of data in this column.')
     return df_flow, df_level
 
+def cleaner_netcdf_werp(input_df: pd.DataFrame, stations: dict) -> pd.DataFrame:
+
+    '''Ingests dataframe, cleans up into a format matching IQQM csv
+    
+    Args:
+        input_df (pd.DataFrame): raw xarray dataframe read-in
+
+        statios(dict):  dict mapping IQQM stations to gauge numbers 
+
+    Results:
+        tuple[pd.DataFrame, pd.DataFrame]: Cleaned flow dataframe; cleaned water level dataframe
+
+    '''
+
+    # organise like the rest of the dataframes- make this look just like we've read it in from an IQQM csv
+    cleaned_df = input_df.reset_index(level = 'node')
+    cleaned_df['node'] = cleaned_df['node'].astype(str)
+
+    cleaned_df['gauge'] = cleaned_df['node'].map(stations)
+    cleaned_df = cleaned_df.drop('node', axis = 1) 
+
+    # drop the values that don't map to a gauge (lots of nodes in iqqm don't)
+        # This should be deprecated with the new way of choosing nodes on read-in, but being careful
+    cleaned_df = cleaned_df.query('gauge.notna()')
+
+    # give each gauge its own column- that's what the tool expects
+    cleaned_df = cleaned_df.pivot(columns = 'gauge', values = 'Simulated flow')
+    cleaned_df.columns.name = None
+
+    # the csvs return an 'object' type, not a datetime in the index
+    # but it gets converted to datetime in cleaner_***, so leave it.
+    cleaned_df.index.names = ['Date']
+
+    # Split gauges into flow and level, allocate to respective dataframe
+    flow_gauges = data_inputs.get_gauges('flow gauges')
+    level_gauges = data_inputs.get_gauges('level gauges')
+    df_flow = pd.DataFrame(index = cleaned_df.index)
+    df_level = pd.DataFrame(index = cleaned_df.index)
+    for gauge in cleaned_df.columns:
+        if gauge in flow_gauges:
+            df_flow[gauge] = cleaned_df[gauge].copy(deep=True)
+        if gauge in level_gauges:
+            df_level[gauge] = cleaned_df[gauge].copy(deep=True)
+
+    return df_flow, df_level
+
 
 def cleaner_ten_thousand_year(input_df: pd.DataFrame, ewr_table_path: str = None) -> pd.DataFrame:
     '''Ingests dataframe, removes junk columns, fixes date, allocates gauges to either flow/level
@@ -455,6 +558,10 @@ class ScenarioHandler:
                 df_clean = cleaner_NSW(data)
                 df_F, df_L = match_NSW_nodes(df_clean, data_inputs.get_NSW_codes())
 
+            elif self.model_format == 'IQQM - netcdf':
+                df_unpacked = unpack_netcdf_as_dataframe(scenarios[scenario])
+                df_F, df_L = cleaner_netcdf_werp(df_unpacked, data_inputs.get_iqqm_codes())
+
             elif self.model_format == 'ten thousand year':
                 df = pd.read_csv(scenarios[scenario], index_col = 'Date')
                 df_F, df_L = cleaner_ten_thousand_year(df, self.parameter_sheet)

{py_ewr-2.1.9.dist-info → py_ewr-2.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py_ewr
-Version: 2.1.9
+Version: 2.2.1
 Summary: Environmental Water Requirement calculator
 Home-page: https://github.com/MDBAuth/EWR_tool
 Author: Martin Job
@@ -23,9 +23,11 @@ Requires-Dist: ipython ==8.8.0
 Requires-Dist: ipywidgets ==7.7.0
 Requires-Dist: pandas ==2.0.3
 Requires-Dist: requests ==2.25.1
-Requires-Dist: tqdm ==4.64.0
+Requires-Dist: tqdm >=4.66.1
 Requires-Dist: mdba-gauge-getter ==0.5.1
 Requires-Dist: cachetools ==5.2.0
+Requires-Dist: xarray ==2023.01.0
+Requires-Dist: netCDF4 ==1.6.4
 Requires-Dist: numpy <2
 
 [![CI](https://github.com/MDBAuth/EWR_tool/actions/workflows/test-release.yml/badge.svg)]()
@@ -33,9 +35,10 @@ Requires-Dist: numpy <2
 [![PyPI](https://img.shields.io/pypi/v/py-ewr)](https://pypi.org/project/py-ewr/)
 [![DOI](https://zenodo.org/badge/342122359.svg)](https://zenodo.org/badge/latestdoi/342122359)
 
-### **EWR tool version 2.1.9 README**
+### **EWR tool version 2.2.1 README**
 
 ### **Notes on recent version update**
+- Adding new model format handling - 'IQQM - netcdf'
 - Standard time-series handling added - each column needs a gauge, followed by and underscore, followed by either flow or level (e.g. 409025_flow). This handling also has missing date filling - so any missing dates will be filled with NaN values in all columns.
 - ten thousand year handling - This has been briefly taken offline for this version.
 - bug fixes: spells of length equal to the minimum required spell length were getting filtered out of the successful events table and successful interevents table, fixed misclassification of some gauges to flow, level, and lake level categories
@@ -222,3 +225,11 @@ NSW:
 
 Consult the user manual for instructions on how to run the tool. Please email the above email addresses for a copy of the user manual.
 
+To disable progress bars, as for example when running remote scripted runs, use 
+
+``` python
+import os
+os.environ["TQDM_DISABLE"] = "1"
+```
+*before* importing py-ewr in your script.
+

py_ewr-2.2.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+py_ewr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+py_ewr/data_inputs.py,sha256=SmY92Xq7oAzkHsJgxncVzC1C9YqnLpDAbVadBF0nS7k,18440
+py_ewr/evaluate_EWRs.py,sha256=1JYWNtw5MYgf92jXhDNlgGkDx2mcfBP34OO-0jgal1w,229279
+py_ewr/io.py,sha256=Is0xPAzLx6-ylpTFyYJxMimkNVxxoTxUcknTk6bQbgs,840
+py_ewr/observed_handling.py,sha256=mpmLZGq9EEy7x_6YFhtZARhKJvFvcmLThmjan4pfMEo,17893
+py_ewr/scenario_handling.py,sha256=yrx0y_ZIXihpfb8hPhZG3ug1xN-5kKmYJzAASWVIH18,32442
+py_ewr/summarise_results.py,sha256=7w2Tbriwob21UXG0N2rSKffneg3M49hWouJPTHVjDAU,29747
+py_ewr/model_metadata/SiteID_MDBA.csv,sha256=DcwFmBBoLmv1lGik40IwTMSjSBPaDsTt8Nluh2s7wjM,183665
+py_ewr/model_metadata/SiteID_NSW.csv,sha256=UVBxN43Z5KWCvWhQ5Rh6TNEn35q4_sjPxKyHg8wPFws,6805
+py_ewr/model_metadata/iqqm_stations.csv,sha256=vl4CPtPslG5VplSzf_yLZulTrmab-mEBHOfzFtS1kf4,110
+py_ewr/parameter_metadata/ewr_calc_config.json,sha256=l1AgIRlf7UUmk3BNQ4r3kutU48pYHHVKmLELjoB-8rQ,17664
+py_ewr/parameter_metadata/parameter_sheet.csv,sha256=IiYAvf0hG9fchuwqtfDZhI8WSPB5jgaaaJ0MLwfPYAw,899556
+py_ewr-2.2.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
+py_ewr-2.2.1.dist-info/METADATA,sha256=coWcS6PVhx_mqwxT15xFL-gbHugA4KqWrPk-uu8DZNE,10049
+py_ewr-2.2.1.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+py_ewr-2.2.1.dist-info/top_level.txt,sha256=n3725d-64Cjyb-YMUMV64UAuIflzUh2_UZSxiIbrur4,7
+py_ewr-2.2.1.dist-info/RECORD,,

{py_ewr-2.1.9.dist-info → py_ewr-2.2.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.2.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

py_ewr-2.1.9.dist-info/RECORD

@@ -1,15 +0,0 @@
-py_ewr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-py_ewr/data_inputs.py,sha256=BLJrrasZYpSUkC0yVBzVqnz6WGT2Y3SJYJJhguDi-Vo,18046
-py_ewr/evaluate_EWRs.py,sha256=1JYWNtw5MYgf92jXhDNlgGkDx2mcfBP34OO-0jgal1w,229279
-py_ewr/observed_handling.py,sha256=mpmLZGq9EEy7x_6YFhtZARhKJvFvcmLThmjan4pfMEo,17893
-py_ewr/scenario_handling.py,sha256=yMAfRNIZtuGntdey2-FML5nfUaGXMIApAUvIzQbyXUs,28476
-py_ewr/summarise_results.py,sha256=7w2Tbriwob21UXG0N2rSKffneg3M49hWouJPTHVjDAU,29747
-py_ewr/model_metadata/SiteID_MDBA.csv,sha256=DcwFmBBoLmv1lGik40IwTMSjSBPaDsTt8Nluh2s7wjM,183665
-py_ewr/model_metadata/SiteID_NSW.csv,sha256=UVBxN43Z5KWCvWhQ5Rh6TNEn35q4_sjPxKyHg8wPFws,6805
-py_ewr/parameter_metadata/ewr_calc_config.json,sha256=l1AgIRlf7UUmk3BNQ4r3kutU48pYHHVKmLELjoB-8rQ,17664
-py_ewr/parameter_metadata/parameter_sheet.csv,sha256=p92qiaYiBfAVGDrMeJ-vb0qQnepeTgvDBKSNYj2TG5c,899556
-py_ewr-2.1.9.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
-py_ewr-2.1.9.dist-info/METADATA,sha256=7mICFTABXvrkiEsAXUW-Gquev0Z8c9bctPhj520I-jE,9748
-py_ewr-2.1.9.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
-py_ewr-2.1.9.dist-info/top_level.txt,sha256=n3725d-64Cjyb-YMUMV64UAuIflzUh2_UZSxiIbrur4,7
-py_ewr-2.1.9.dist-info/RECORD,,