mpcaHydro 2.2.8__tar.gz → 2.2.9__tar.gz

{mpcahydro-2.2.8 → mpcahydro-2.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpcaHydro
-Version: 2.2.8
+Version: 2.2.9
 Summary: Python package for downloading MPCA hydrology data
 Project-URL: Homepage, https://github.com/mfratkin1/mpcaHydro
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>

{mpcahydro-2.2.8 → mpcahydro-2.2.9}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "mpcaHydro"
 urls = { "Homepage" = "https://github.com/mfratkin1/mpcaHydro" }  # ? Add this!
-version = "2.2.8"
+version = "2.2.9"
 dependencies = [
   "pandas",
   "requests",

{mpcahydro-2.2.8 → mpcahydro-2.2.9}/src/mpcaHydro/equis.py

@@ -265,6 +265,13 @@ def replace_nondetects(df):
     df.loc[df['value'].isna(), 'value'] = 0
     return df
 
+def filter_years(df, start_year=1996, end_year=None):
+    '''Filter Equis data to include only samples within a certain year range.'''
+    df = df[df['datetime'].dt.year >= start_year]
+    if end_year is not None:
+        df = df[df['datetime'].dt.year <= end_year]
+    return df
+
 def normalize(df):
     '''Normalize Equis data: select relevant columns.'''
     df = map_constituents(df)
@@ -278,6 +285,7 @@ def transform(df):
     
     df = normalize(df)
     df = replace_nondetects(df)
+    df = filter_years(df)
     if not df.empty:
         df = average_results(df)
     return df

{mpcahydro-2.2.8 → mpcahydro-2.2.9}/src/mpcaHydro/outlets.py

@@ -31,7 +31,15 @@ DB_PATH = str(Path(__file__).resolve().parent/'data\\outlet.duckdb')
 MODL_DB = pd.concat([stations_wiski,stations_equis])
 MODL_DB['opnids'] = MODL_DB['opnids'].str.strip().replace('',pd.NA)
 MODL_DB = MODL_DB.dropna(subset='opnids')
+MODL_DB = MODL_DB.dropna(subset = 'repo_name')
 MODL_DB = MODL_DB.drop_duplicates(['station_id','source']).reset_index(drop=True)
+# Add outlet_id column to MODL_DB based on enumerate grouping
+outlet_id_map = {}
+for outlet_id, (_, group) in enumerate(MODL_DB.drop_duplicates(['station_id','source']).groupby(by=['opnids','repo_name'])):
+    for idx in group.index:
+        outlet_id_map[idx] = int(outlet_id)
+MODL_DB['outlet_id'] = MODL_DB.index.map(outlet_id_map)
+
 
 def _reload():
     global _stations_wiski, stations_wiski, _stations_equis, stations_equis, MODL_DB
@@ -47,7 +55,14 @@ def _reload():
     MODL_DB = pd.concat([stations_wiski,stations_equis])
     MODL_DB['opnids'] = MODL_DB['opnids'].str.strip().replace('',pd.NA)
     MODL_DB = MODL_DB.dropna(subset='opnids')
+    MODL_DB = MODL_DB.dropna(subset = 'repo_name')
     MODL_DB = MODL_DB.drop_duplicates(['station_id','source']).reset_index(drop=True)
+    # Add outlet_id column to MODL_DB based on enumerate grouping
+    outlet_id_map = {}
+    for outlet_id, (_, group) in enumerate(MODL_DB.drop_duplicates(['station_id','source']).groupby(by=['opnids','repo_name'])):
+        for idx in group.index:
+            outlet_id_map[idx] = int(outlet_id)
+    MODL_DB['outlet_id'] = MODL_DB.index.map(outlet_id_map)
 
 
 def split_opnids(opnids: list):
@@ -174,6 +189,34 @@ def get_station_opnids(station_id: str, station_origin: str):
         [station_id, station_origin]).fetchdf()
     return df['reach_id'].tolist()
 
+def get_outlet_opnids(outlet_id: int):
+    """
+    Return all model reach IDs (opnids) associated with the given outlet ID.
+    """
+    with connect(DB_PATH) as con:
+        df = con.execute(
+        """
+        SELECT r.reach_id
+        FROM outlets.station_reach_pairs r
+        WHERE r.outlet_id = ?
+        """,
+        [outlet_id]).fetchdf()
+    return list(set(df['reach_id'].tolist()))
+
+def get_outlet_stations(outlet_id: int):
+    """
+    Return all station IDs and origins associated with the given outlet ID.
+    """
+    with connect(DB_PATH) as con:
+        df = con.execute(
+        """
+        SELECT r.station_id, r.station_origin
+        FROM outlets.station_reach_pairs r
+        WHERE r.outlet_id = ?
+        """,
+        [outlet_id]).fetchdf()
+    return df[['station_id', 'station_origin']].drop_duplicates().to_dict(orient='records')
+
 
 class OutletGateway:
     def __init__(self, model_name: str):
@@ -192,7 +235,7 @@ class OutletGateway:
         return equis_station_opnids(self.model_name)
 
     def station_opnids(self):
-        return station_opnids(self.model_name)
+        return mapped_station_opnids(self.model_name)
 
     def equis_stations(self):
         return equis_stations(self.model_name)
@@ -220,6 +263,12 @@ class OutletGateway:
         assert(station_id in self.wiski_stations() + self.equis_stations()), f"Station ID {station_id} not found in model {self.model_name}"
         return get_outlets_by_station(station_id, station_origin)
 
+    def get_outlet_opnids(self, outlet_id: int):
+        return get_outlet_opnids(outlet_id)
+    
+    def get_outlet_stations(self, outlet_id: int):
+        return get_outlet_stations(outlet_id)
+        
 # constructors:
 def build_outlet_db(db_path: str = None):
     if db_path is None:
@@ -235,31 +284,15 @@ def build_outlets(con, model_name: str = None):
     else:
         modl_db = MODL_DB
 
-    for index, (_, group) in enumerate(modl_db.drop_duplicates(['station_id','source']).groupby(by = ['opnids','repo_name'])):
-        repo_name = group['repo_name'].iloc[0]    
-        add_outlet(con, outlet_id = index, outlet_name = None, repository_name = repo_name, notes = None)
-        
+    for outlet_id in modl_db['outlet_id'].unique():
+        group = modl_db.query('outlet_id == @outlet_id')
+        repo_name = group['repo_name'].iloc[0]
+        add_outlet(con, outlet_id = int(outlet_id), outlet_name = None, repository_name = repo_name, notes = None)
         opnids = set(split_opnids(group['opnids'].str.split(',').to_list()))
-
         for opnid in opnids:
-            add_reach(con, outlet_id = index, reach_id = int(opnid), repository_name = repo_name)
-
-        for _, row in group.drop_duplicates(subset=['station_id', 'source']).iterrows():
-            add_station(con, outlet_id = index, station_id = row['station_id'], station_origin = row['source'], true_opnid = row['true_opnid'], repository_name= repo_name, comments = row['comments'])
-
- 
-def create_outlet_schema(con, model_name : str):
-    for index, (_, group) in enumerate(outlets(model_name)):
-        repo_name = group['repo_name'].iloc[0]    
-        add_outlet(con, outlet_id = index, outlet_name = None, repository_name = repo_name, notes = None)
-        
-        opnids = set(split_opnids(group['opnids'].str.split(',').to_list()))
-
-        for opnid in opnids:
-            add_reach(con, outlet_id = index, reach_id = int(opnid), repository_name = repo_name)
-
+            add_reach(con, outlet_id = int(outlet_id), reach_id = int(opnid), repository_name = repo_name)
         for _, row in group.drop_duplicates(subset=['station_id', 'source']).iterrows():
-            add_station(con, outlet_id = index, station_id = row['station_id'], station_origin = row['source'], true_opnid = row['true_opnid'], repository_name= repo_name, comments = row['comments'])
+            add_station(con, outlet_id = int(outlet_id), station_id = row['station_id'], station_origin = row['source'], true_opnid = row['true_opnid'], repository_name= repo_name, comments = row['comments'])
 
 
 def add_outlet(con,

{mpcahydro-2.2.8 → mpcahydro-2.2.9}/src/mpcaHydro/warehouse.py

@@ -151,12 +151,11 @@ def attach_outlets_db(con: duckdb.DuckDBPyConnection, outlets_db_path: str):
     con.execute("DETACH 'outlets_db'")
 
 
-def create_outlets_tables(con: duckdb.DuckDBPyConnection):
+def create_outlets_tables(con: duckdb.DuckDBPyConnection, model_name: str = None):
     """Create tables in the outlets schema to define outlet-station-reach relationships."""
     con.execute(sql_loader.get_outlets_schema_sql())
     con.execute(sql_loader.get_views_outlets_sql())
-    outlets.build_outlets(con)
-
+    outlets.build_outlets(con, model_name=model_name)
 
 def create_filtered_wiski_view(con: duckdb.DuckDBPyConnection, data_codes: list):
     """Create a view filtering WISKI data based on specified data codes."""

{mpcahydro-2.2.8 → mpcahydro-2.2.9}/src/mpcaHydro/warehouse_functions.py

@@ -101,7 +101,8 @@ def download_wiski_data(
         if overwrite:
             warehouse.drop_station_data(con, station_ids, 'wiski')
         warehouse.add_df_to_table(con, df, 'staging', 'wiski')
-        warehouse.add_df_to_table(con, df_transformed, 'analytics', 'wiski')
+        if not df_transformed.empty:
+            warehouse.add_df_to_table(con, df_transformed, 'analytics', 'wiski')
         warehouse.update_views(con)
     else:
         print('No data necessary for HSPF calibration from wiski for:', station_ids)
@@ -351,7 +352,7 @@ def station_reach_pairs(con: duckdb.DuckDBPyConnection):
     query = '''
     SELECT *,
     FROM 
-        reports.station_reach_pairs
+        outlets.station_reach_pairs
     ORDER BY
         outlet_id,
         station_id

{mpcahydro-2.2.8 → mpcahydro-2.2.9}/src/mpcaHydro/wiski.py

@@ -336,6 +336,14 @@ def filter_quality_codes(df, data_codes):
     '''
     return df.loc[df['quality_code'].isin(data_codes)]
 
+def filter_years(df, start_year=1996, end_year=None):
+    '''Filter Equis data to include only samples within a certain year range.'''
+    df = df[df['datetime'].dt.year >= start_year]
+    if end_year is not None:
+        df = df[df['datetime'].dt.year <= end_year]
+    return df
+
+
 def average_results(df):
     #df['datetime'] = pd.to_datetime(df.loc[:,'datetime'])
     df.loc[:,'datetime'] = df.loc[:,'datetime'].dt.round('h')
@@ -392,6 +400,7 @@ def transform(df, filter_qc_codes = True, data_codes = None, baseflow_method = '
             data_codes = DATA_CODES
         df = filter_quality_codes(df, data_codes)
     df = average_results(df)
+    df = filter_years(df, start_year=1996)
     df = calculate_baseflow(df, method = baseflow_method)
     df['station_origin'] = 'wiski'
     #df.set_index('datetime',inplace=True)