py-ewr 2.2.1__py3-none-any.whl → 2.2.4__py3-none-any.whl

py_ewr/data_inputs.py

@@ -416,3 +416,27 @@ def get_scenario_gauges(gauge_results: dict) -> list:
         for gauge in scenario.keys():
             scenario_gauges.append(gauge)
     return list(set(scenario_gauges))
+
+
+def gauge_groups(parameter_sheet: pd.DataFrame) -> dict:
+    '''
+    Returns a dictionary of flow, level, and lake level gauges based on the parameter sheet and some hard coding of other EWRs
+
+    Args:
+        parameter_sheet (pd.DataFrame): input parameter sheet
+    
+    Returns:
+        dict: keys as flow, level, and lake level gauges, values as the list of gauges
+    '''
+    
+    # Hard coded gauges for the CLLMM EWRs
+    hard_code_levels = ['A4260527', 'A4260524', 'A4260633', 'A4261209', 'A4261165']
+    hard_code_lake_levels = ['A4261133', 'A4260574', 'A4260575']
+
+    flow_gauges = set(parameter_sheet[parameter_sheet['GaugeType'] == 'F']['Gauge']) + set(parameter_sheet['Multigauge'])
+    level_gauges = set(parameter_sheet[parameter_sheet['GaugeType'] == 'L']['Gauge']) + set(parameter_sheet['WeirpoolGauge']) + set(hard_code_levels)
+    lake_level_gauges = set(parameter_sheet[parameter_sheet['GaugeType'] == 'LL']['Gauge'])+set(hard_code_lake_levels)
+
+    return flow_gauges, level_gauges, lake_level_gauges
+
+# def gauges_to_measurand()

py_ewr/evaluate_EWRs.py

@@ -10,7 +10,6 @@ import logging
 
 import pandas as pd
 import numpy as np
-from tqdm import tqdm
 
 from . import data_inputs
 
@@ -446,8 +445,8 @@ def get_index_date(date_index:Any)-> datetime.date:
     """
     if type(date_index) == pd._libs.tslibs.timestamps.Timestamp:
         return date_index.date()
-    if type(date_index) == pd._libs.tslibs.period.Period:
-        return date_index.to_timestamp().date()
+    # if type(date_index) == pd._libs.tslibs.period.Period:
+    #     return date_index.date()#.to_timestamp()
     else:
         return date_index
 
@@ -1959,7 +1958,7 @@ def water_stability_check(EWR_info:Dict, iteration:int, flows:List, all_events:D
     if levels_are_stable:
         # record event opportunity for the next n days for the total period of (EggDaysSpell)+ larvae (LarvaeDaysSpell)
         # if the last day of the event is not over the last day of the event window
-        iteration_date = flow_date.to_timestamp().date()
+        iteration_date = flow_date.date()#flow_date.to_timestamp().date()
         last_day_window = get_last_day_of_window(iteration_date, EWR_info['end_month'])
         event_size = EWR_info['eggs_days_spell'] + EWR_info['larvae_days_spell']
         if is_date_in_window(iteration_date, last_day_window, event_size):
@@ -1996,7 +1995,7 @@ def water_stability_level_check(EWR_info:Dict, iteration:int, all_events:Dict, w
     if levels_are_stable:
         # record event opportunity for the next n days for the total period of (EggDaysSpell)+ larvae (LarvaeDaysSpell)
         # if the last day of the event is not over the last day of the event window
-        iteration_date = flow_date.to_timestamp().date()
+        iteration_date = flow_date.date()#flow_date.to_timestamp().date()
         last_day_window = get_last_day_of_window(iteration_date, EWR_info['end_month'])
         event_size = EWR_info['eggs_days_spell'] + EWR_info['larvae_days_spell']
         if is_date_in_window(iteration_date, last_day_window, event_size):
@@ -2605,7 +2604,7 @@ def lower_lakes_level_check(EWR_info: dict, levels: pd.Series, event: list, all_
 #------------------------------------ Calculation functions --------------------------------------#
 
 
-def create_water_stability_event(flow_date: pd.Period, flows:List, iteration: int, EWR_info:dict)->List:
+def create_water_stability_event(flow_date: pd.Timestamp, flows:List, iteration: int, EWR_info:dict)->List:#pd.Period
     """create overlapping event that meets an achievement for fish recruitment water stability
 
     Args:
@@ -2618,7 +2617,7 @@ def create_water_stability_event(flow_date: pd.Period, flows:List, iteration: in
     """
     event_size = EWR_info['eggs_days_spell'] + EWR_info['larvae_days_spell']
     event_flows = flows[iteration: iteration + event_size]
-    start_event_date = flow_date.to_timestamp().date()
+    start_event_date = flow_date.date()#flow_date.to_timestamp().date()
     event_dates = [ start_event_date + timedelta(i) for i in range(event_size)]
     
     return [(d, flow)  for d, flow in zip(event_dates, event_flows)]
@@ -3811,16 +3810,16 @@ def nest_calc_percent_trigger(EWR_info:Dict, flows:List, water_years:List, dates
             flow_percent_change = calc_flow_percent_change(i, flows)
             trigger_day = date(dates[i].year,EWR_info["trigger_month"], EWR_info["trigger_day"])
             cut_date = calc_nest_cut_date(EWR_info, i, dates)
-            is_in_trigger_window = dates[i].to_timestamp().date() >= trigger_day \
-                                   and dates[i].to_timestamp().date() <= trigger_day + timedelta(days=14)
+            is_in_trigger_window = dates[i].date() >= trigger_day \
+                                   and dates[i].date() <= trigger_day + timedelta(days=14) #.to_timestamp() .to_timestamp()
             iteration_no_event = 0
             
             ## if there IS an ongoing event check if we are on the trigger season window 
             # if yes then check the current flow
             if total_event > 0:
-                if (dates[i].to_timestamp().date() >= trigger_day) and (dates[i].to_timestamp().date() <= cut_date):
+                if (dates[i].date() >= trigger_day) and (dates[i].date() <= cut_date):
                     event, all_events, gap_track, total_event, iteration_no_event = nest_flow_check(EWR_info, i, flow, event, all_events, 
-                                                         gap_track, water_years, total_event, flow_date, flow_percent_change, iteration_no_event)
+                                                         gap_track, water_years, total_event, flow_date, flow_percent_change, iteration_no_event) #.to_timestamp() .to_timestamp()
 
                 # this path will only be executed if an event extends beyond the cut date    
                 else:
@@ -3841,12 +3840,12 @@ def nest_calc_percent_trigger(EWR_info:Dict, flows:List, water_years:List, dates
     
     # Check final iteration in the flow timeseries, saving any ongoing events/event gaps to their spots in the dictionaries:
     # reset all variable to last flow
-    flow_date = dates[-1].to_timestamp().date()
+    flow_date = dates[-1].date()#.to_timestamp()
     flow_percent_change = calc_flow_percent_change(-1, flows)
     trigger_day = date(dates[-1].year,EWR_info["trigger_month"], EWR_info["trigger_day"])
     cut_date = calc_nest_cut_date(EWR_info, -1, dates)
-    is_in_trigger_window = dates[-1].to_timestamp().date() >= trigger_day - timedelta(days=7) \
-    and dates[-1].to_timestamp().date() <= trigger_day + timedelta(days=7)
+    is_in_trigger_window = dates[-1].date() >= trigger_day - timedelta(days=7) \
+    and dates[-1].date() <= trigger_day + timedelta(days=7) #.to_timestamp() .to_timestamp()
     iteration_no_event = 0
 
     if total_event > 0:
@@ -5086,9 +5085,7 @@ def calc_sorter(df_F:pd.DataFrame, df_L:pd.DataFrame, gauge:str, EWR_table:pd.Da
         EWR_codes = PU_table['Code']
         PU_df = pd.DataFrame()
         PU_events = {}
-        for i, EWR in enumerate(tqdm(EWR_codes, position = 0, leave = False,
-                                     bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}',
-                                     desc= str('Evaluating ewrs for '+ gauge))):
+        for i, EWR in enumerate(EWR_codes):
             events = {}
 
             MULTIGAUGE = is_multigauge(EWR_table, gauge, EWR, PU)

py_ewr/observed_handling.py

@@ -3,7 +3,6 @@ from typing import Dict, List
 import logging
 
 import pandas as pd
-from tqdm import tqdm
 import numpy as np
 
 from . import data_inputs, evaluate_EWRs, summarise_results, scenario_handling
@@ -131,7 +130,7 @@ def observed_cleaner(input_df: pd.DataFrame, dates: dict) -> pd.DataFrame:
     start_date = datetime(dates['start_date'].year, dates['start_date'].month, dates['start_date'].day)
     end_date = datetime(dates['end_date'].year, dates['end_date'].month, dates['end_date'].day)
     
-    df_index = pd.date_range(start=start_date,end=end_date - timedelta(days=1)).to_period()
+    df_index = pd.date_range(start=start_date,end=end_date - timedelta(days=1))#.to_period()
     gauge_data_df = pd.DataFrame()
     gauge_data_df['Date'] = df_index
     gauge_data_df = gauge_data_df.set_index('Date')
@@ -140,7 +139,7 @@ def observed_cleaner(input_df: pd.DataFrame, dates: dict) -> pd.DataFrame:
     
     
     input_df['Date'] = pd.to_datetime(input_df['DATETIME'], format = '%Y-%m-%d')
-    input_df['Date'] = input_df['Date'].apply(lambda x: x.to_period(freq='D'))
+    # input_df['Date'] = input_df['Date'].apply(lambda x: x.to_period(freq='D'))
 
     # Check with states for more codes:
     bad_data_codes = data_inputs.get_bad_QA_codes()

py_ewr/scenario_handling.py

@@ -7,7 +7,6 @@ from datetime import datetime, date
 import logging
 
 import pandas as pd
-from tqdm import tqdm
 import xarray as xr
 import netCDF4
  
@@ -259,7 +258,7 @@ def cleaner_MDBA(input_df: pd.DataFrame) -> pd.DataFrame:
     
     cleaned_df = input_df.rename(columns={'Mn': 'Month', 'Dy': 'Day'})
     cleaned_df['Date'] = pd.to_datetime(cleaned_df[['Year', 'Month', 'Day']], format = '%Y-%m-%d')
-    cleaned_df['Date'] = cleaned_df['Date'].apply(lambda x: x.to_period(freq='D'))
+    # cleaned_df['Date'] = cleaned_df['Date'].apply(lambda x: x.to_period(freq='D'))
     cleaned_df = cleaned_df.drop(['Day', 'Month', 'Year'], axis = 1)
     cleaned_df = cleaned_df.set_index('Date')
     
@@ -437,6 +436,42 @@ def extract_gauge_from_string(input_string: str) -> str:
     gauge = input_string.split('_')[0]
     return gauge
 
+# def match_MDBA_nodes_dev(input_df: pd.DataFrame, model_metadata: pd.DataFrame, ewr_table_path: str) -> tuple:
+#     '''
+#     Iterate over the gauges in the parameter sheet, 
+#     find all the occurences of that gauge in the ARWC column in the model metadata file,
+#     for each match, search for the matching siteID in the model file,
+#     append the column to the flow dataframe.
+
+#     Args:
+#         input_df (pd.DataFrame): flow/water level dataframe
+#         model_metadata (pd.DataFrame): dataframe linking model nodes to gauges
+
+#     Returns:
+#         tuple[pd.DataFrame, pd.DataFrame]: flow dataframe, water level dataframe
+
+#     '''
+#     df_flow = pd.DataFrame(index = input_df.index)
+#     df_level = pd.DataFrame(index = input_df.index)
+
+#     unique_gauges = #Get unique gauges from the parameter sheet
+#     #TODO: include logic to have the measurand included
+#     for i in unique_gauges:
+#         # Subset of the SiteID file with the gauges
+#         subset_df = model_metadata[model_metadata['AWRC'] == i]
+#         # Iterate over the occurences of the gauge and check if the matching SiteID file is in the model file
+#         for j in subset_df.iterrows:
+#             site_mm = j['SITEID']
+#             if site_mm in input_df.columns:
+#                 df_flow[i] = input_df[site_mm+INPUT_MEASURAND+ANY_QUALITY_CODE]
+#                 or
+#                 df_level[i] = input_df[site_mm+INPUT_MEASURAND+ANY_QUALITY_CODE]
+
+#     if df_flow.empty and df_level.empty:
+#         raise ValueError('No relevant gauges and or measurands found in dataset, the EWR tool cannot evaluate this model output file')      
+#     return df_flow, df_level
+
+
 def match_MDBA_nodes(input_df: pd.DataFrame, model_metadata: pd.DataFrame, ewr_table_path: str) -> tuple:
     '''Checks if the source file columns have EWRs available, returns a flow and level dataframe with only 
     the columns with EWRs available. Renames columns to gauges
@@ -461,11 +496,19 @@ def match_MDBA_nodes(input_df: pd.DataFrame, model_metadata: pd.DataFrame, ewr_t
         measure = col_clean.split('-')[1]
         if ((measure in measurands) and (model_metadata['SITEID'] == site).any()):
             subset = model_metadata.query("SITEID==@site")
-            gauge = subset["AWRC"].iloc[0]
-            if gauge in flow_gauges and measure == '1':
-                df_flow[gauge] = input_df[col]
-            if gauge in level_gauges and measure == '35':
-                df_level[gauge] = input_df[col]
+            for iset in range(len(subset)):
+                gauge = subset["AWRC"].iloc[iset]
+                if gauge in flow_gauges and measure == '1':
+                    df_flow[gauge] = input_df[col]
+                if gauge in level_gauges and measure == '35':
+                    aa=input_df[[col]]
+                    if (len(aa.columns)>1):
+                        print('More than one site has been identified, the first site is used')
+                        print('Site info: ', col)
+                        df_level[gauge] = aa.iloc[:,0]
+                    else:
+                        df_level[gauge] = input_df[col]
+
     if df_flow.empty:
         raise ValueError('No relevant gauges and or measurands found in dataset, the EWR tool cannot evaluate this model output file')      
     return df_flow, df_level
@@ -538,9 +581,7 @@ class ScenarioHandler:
         # Analyse all scenarios for EWRs
         detailed_results = {}
         detailed_events = {}
-        for scenario in tqdm(scenarios, position = 0, leave = True, 
-                            bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}',
-                            desc= 'Evaluating scenarios'):
+        for scenario in scenarios:
             if self.model_format == 'Bigmod - MDBA':
                 
                 data, header = unpack_model_file(scenarios[scenario], 'Dy', 'Field')

{py_ewr-2.2.1.dist-info → py_ewr-2.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py_ewr
-Version: 2.2.1
+Version: 2.2.4
 Summary: Environmental Water Requirement calculator
 Home-page: https://github.com/MDBAuth/EWR_tool
 Author: Martin Job
@@ -23,7 +23,6 @@ Requires-Dist: ipython ==8.8.0
 Requires-Dist: ipywidgets ==7.7.0
 Requires-Dist: pandas ==2.0.3
 Requires-Dist: requests ==2.25.1
-Requires-Dist: tqdm >=4.66.1
 Requires-Dist: mdba-gauge-getter ==0.5.1
 Requires-Dist: cachetools ==5.2.0
 Requires-Dist: xarray ==2023.01.0
@@ -35,9 +34,11 @@ Requires-Dist: numpy <2
 [![PyPI](https://img.shields.io/pypi/v/py-ewr)](https://pypi.org/project/py-ewr/)
 [![DOI](https://zenodo.org/badge/342122359.svg)](https://zenodo.org/badge/latestdoi/342122359)
 
-### **EWR tool version 2.2.1 README**
+### **EWR tool version 2.2.4 README**
 
 ### **Notes on recent version update**
+- Remove TQDM loading bars
+- Handle duplicate sites in MDBA siteID file - where a duplicate exists, the first match is used and the rest are skipped over
 - Adding new model format handling - 'IQQM - netcdf'
 - Standard time-series handling added - each column needs a gauge, followed by and underscore, followed by either flow or level (e.g. 409025_flow). This handling also has missing date filling - so any missing dates will be filled with NaN values in all columns.
 - ten thousand year handling - This has been briefly taken offline for this version.

{py_ewr-2.2.1.dist-info → py_ewr-2.2.4.dist-info}/RECORD

@@ -1,17 +1,17 @@
 py_ewr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-py_ewr/data_inputs.py,sha256=SmY92Xq7oAzkHsJgxncVzC1C9YqnLpDAbVadBF0nS7k,18440
-py_ewr/evaluate_EWRs.py,sha256=1JYWNtw5MYgf92jXhDNlgGkDx2mcfBP34OO-0jgal1w,229279
+py_ewr/data_inputs.py,sha256=PtpFyikT_IoMj_Ont6UykSdyiZXapny_Yk1J9VxParQ,19494
+py_ewr/evaluate_EWRs.py,sha256=09sDcSXW-mH3E8U4fxLp5pR55ISFOs2dZhc40eqoL4k,229138
 py_ewr/io.py,sha256=Is0xPAzLx6-ylpTFyYJxMimkNVxxoTxUcknTk6bQbgs,840
-py_ewr/observed_handling.py,sha256=mpmLZGq9EEy7x_6YFhtZARhKJvFvcmLThmjan4pfMEo,17893
-py_ewr/scenario_handling.py,sha256=yrx0y_ZIXihpfb8hPhZG3ug1xN-5kKmYJzAASWVIH18,32442
+py_ewr/observed_handling.py,sha256=mzdJoAUqhFsu3CIkJup_cXiAGLkxEDICnAzemBgvMG4,17874
+py_ewr/scenario_handling.py,sha256=s-Es9n4YXdLhFFsT8ypYKfZL7_puRxLJRuFip1EC95g,34310
 py_ewr/summarise_results.py,sha256=7w2Tbriwob21UXG0N2rSKffneg3M49hWouJPTHVjDAU,29747
 py_ewr/model_metadata/SiteID_MDBA.csv,sha256=DcwFmBBoLmv1lGik40IwTMSjSBPaDsTt8Nluh2s7wjM,183665
 py_ewr/model_metadata/SiteID_NSW.csv,sha256=UVBxN43Z5KWCvWhQ5Rh6TNEn35q4_sjPxKyHg8wPFws,6805
 py_ewr/model_metadata/iqqm_stations.csv,sha256=vl4CPtPslG5VplSzf_yLZulTrmab-mEBHOfzFtS1kf4,110
 py_ewr/parameter_metadata/ewr_calc_config.json,sha256=l1AgIRlf7UUmk3BNQ4r3kutU48pYHHVKmLELjoB-8rQ,17664
 py_ewr/parameter_metadata/parameter_sheet.csv,sha256=IiYAvf0hG9fchuwqtfDZhI8WSPB5jgaaaJ0MLwfPYAw,899556
-py_ewr-2.2.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
-py_ewr-2.2.1.dist-info/METADATA,sha256=coWcS6PVhx_mqwxT15xFL-gbHugA4KqWrPk-uu8DZNE,10049
-py_ewr-2.2.1.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-py_ewr-2.2.1.dist-info/top_level.txt,sha256=n3725d-64Cjyb-YMUMV64UAuIflzUh2_UZSxiIbrur4,7
-py_ewr-2.2.1.dist-info/RECORD,,
+py_ewr-2.2.4.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
+py_ewr-2.2.4.dist-info/METADATA,sha256=npsbWV01a3qKUUM1eJLRpy8YcsEPpchqI5_hVPTDMa8,10174
+py_ewr-2.2.4.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+py_ewr-2.2.4.dist-info/top_level.txt,sha256=n3725d-64Cjyb-YMUMV64UAuIflzUh2_UZSxiIbrur4,7
+py_ewr-2.2.4.dist-info/RECORD,,