mpcaHydro 2.2.4__tar.gz → 2.2.5__tar.gz

{mpcahydro-2.2.4 → mpcahydro-2.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpcaHydro
-Version: 2.2.4
+Version: 2.2.5
 Summary: Python package for downloading MPCA hydrology data
 Project-URL: Homepage, https://github.com/mfratkin1/mpcaHydro
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>

{mpcahydro-2.2.4 → mpcahydro-2.2.5}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "mpcaHydro"
 urls = { "Homepage" = "https://github.com/mfratkin1/mpcaHydro" }  # ? Add this!
-version = "2.2.4"
+version = "2.2.5"
 dependencies = [
   "pandas",
   "requests",

mpcahydro-2.2.5/tests/integration/test_data_manager.py

@@ -0,0 +1,20 @@
+
+from mpcaHydro import data_manager_functions as dmf
+from mpcaHydro import wiski
+
+station_ids = ['E67013001']
+
+
+dmf.download_wiski_data(con,station_ids,start_year = 2010,)
+
+
+
+station_ids = ['H67009001',
+ 'H67012001',
+ 'H67013001',
+ 'H67014001',
+ 'H67016001',
+ 'H67014002',
+ 'E67013001',
+ 'W67036001',
+ 'W67004001']

{mpcahydro-2.2.4 → mpcahydro-2.2.5}/tests/integration/test_wiski.py

@@ -1,9 +1,7 @@
-#%%
 from mpcaHydro import wiski
 from mpcaHydro import pywisk
 from mpcaHydro import outlets
 
-#%%
 station_ids = ['H67014001']
 
 

{mpcahydro-2.2.4 → mpcahydro-2.2.5}/tests/integration/test_wiski_integration.py

@@ -10,10 +10,6 @@ from unittest.mock import patch, MagicMock
 import pytest
 import pandas as pd
 
-# Test ids
-
-TS_ID_FLOW = 169928010 # Example flow ts_id for station H67014001
-
 
 # Check if warehouse module can be imported (requires data files)
 try:
@@ -98,11 +94,18 @@ class TestWiskiDownload:
         """Test that get_ts returns timeseries data (requires network)."""
         from mpcaHydro import pywisk
         
-        df = pywisk.get_ts(TS_ID_FLOW, start_date='2020-01-01', end_date='2020-01-31')
+        # First get a valid ts_id
+        station_ids = ['H67014001']
+        df_ts = pywisk.get_ts_ids(station_nos=station_ids, parametertype_id='11500')  # Flow
         
-        assert isinstance(df, pd.DataFrame)
-        assert(len(df) > 0)
-        assert all(col in df.columns for col in ['Timestamp', 'Value', 'Quality Code', 'Quality Code Name', 'ts_unitsymbol', 'ts_name'])
+        if not df_ts.empty:
+            ts_id = df_ts['ts_id'].iloc[0]
+            df = pywisk.get_ts(ts_id, start_date='2020-01-01', end_date='2020-01-31')
+            
+            assert isinstance(df, pd.DataFrame)
+            if not df.empty:
+                assert 'Timestamp' in df.columns
+                assert 'Value' in df.columns
 
 
 class TestWiskiDataWithMock:
@@ -111,40 +114,27 @@ class TestWiskiDataWithMock:
     @requires_warehouse
     def test_download_with_mock_data(self):
         """Test WISKI download flow with mocked service."""
-        from mpcaHydro import warehouse_functions as wf
+        from mpcaHydro import data_manager_functions as dmf
         from mpcaHydro import warehouse
         
         # Create mock data that mimics WISKI download format
-        ['Timestamp', 'Value', 'Quality Code', 'Quality Code Name',
-       'ts_unitsymbol', 'ts_name', 'ts_id', 'station_no', 'station_name',
-       'station_latitude', 'station_longitude', 'parametertype_id',
-       'parametertype_name', 'stationparameter_no', 'stationparameter_name',
-       'wplmn_flag']
         mock_wiski_data = pd.DataFrame({
+            'station_no': ['H67014001', 'H67014001'],
             'Timestamp': ['2020-01-01 00:00', '2020-01-01 01:00'],
             'Value': [100.0, 105.0],
             'Quality Code': [1, 1],
-            'Quality Code Name': ['Good', 'Good'],
             'ts_unitsymbol': ['cfs', 'cfs'],
-            'ts_name': ['Discharge', 'Discharge'],
-            'ts_id': [TS_ID_FLOW, TS_ID_FLOW],
-            'station_no': ['H67014001', 'H67014001'],
-            'station_name': ['Station H67014001', 'Station H67014001'],
-            'station_latitude': [46.0, 46.0],
-            'station_longitude': [-94.0, -94.0],
             'parametertype_id': ['11500', '11500'],
             'parametertype_name': ['Discharge', 'Discharge'],
-            'stationparameter_no': ['SP11500', 'SP11500'],
-            'stationparameter_name': ['Discharge', 'Discharge'],
-            'wplmn_flag': [0, 0]})
-
+        })
+        
         with tempfile.TemporaryDirectory() as tmpdir:
             db_path = Path(tmpdir) / 'test.duckdb'
             warehouse.init_db(db_path.as_posix(), reset=True)
             
             with warehouse.connect(db_path.as_posix(), read_only=False) as con:
                 # Load mock data directly
-                warehouse.add_df_to_table(con, mock_wiski_data, 'staging','wiski')
+                warehouse.load_df_to_table(con, mock_wiski_data, 'staging.wiski')
                 
                 # Verify data loaded
                 result = con.execute("SELECT COUNT(*) FROM staging.wiski").fetchone()[0]

{mpcahydro-2.2.4 → mpcahydro-2.2.5}/tests/test_data_manager_functions.py

@@ -13,14 +13,14 @@ class TestConstants:
 
     def test_agg_defaults(self):
         """Test aggregation defaults are properly defined."""
-        from mpcaHydro.warehouse_functions import AGG_DEFAULTS
+        from mpcaHydro.data_manager_functions import AGG_DEFAULTS
         assert 'cfs' in AGG_DEFAULTS
         assert AGG_DEFAULTS['cfs'] == 'mean'
         assert AGG_DEFAULTS['lb'] == 'sum'
 
     def test_unit_defaults(self):
         """Test unit defaults are properly defined."""
-        from mpcaHydro.warehouse_functions import UNIT_DEFAULTS
+        from mpcaHydro.data_manager_functions import UNIT_DEFAULTS
         assert 'Q' in UNIT_DEFAULTS
         assert UNIT_DEFAULTS['Q'] == 'cfs'
         assert UNIT_DEFAULTS['TSS'] == 'mg/l'
@@ -32,13 +32,13 @@ class TestGetDbPath:
 
     def test_get_db_path_string(self):
         """Test get_db_path with string input."""
-        from mpcaHydro.warehouse_functions import get_db_path
+        from mpcaHydro.data_manager_functions import get_db_path
         result = get_db_path('/some/folder')
         assert result == Path('/some/folder/observations.duckdb')
 
     def test_get_db_path_path(self):
         """Test get_db_path with Path input."""
-        from mpcaHydro.warehouse_functions import get_db_path
+        from mpcaHydro.data_manager_functions import get_db_path
         result = get_db_path(Path('/another/folder'))
         assert result == Path('/another/folder/observations.duckdb')
 
@@ -48,14 +48,14 @@ class TestValidation:
 
     def test_valid_constituents_in_unit_defaults(self):
         """Test that key constituents have unit defaults defined."""
-        from mpcaHydro.warehouse_functions import UNIT_DEFAULTS
+        from mpcaHydro.data_manager_functions import UNIT_DEFAULTS
         expected_constituents = ['Q', 'TSS', 'TP', 'OP', 'TKN', 'N', 'WT']
         for const in expected_constituents:
             assert const in UNIT_DEFAULTS, f"Constituent {const} missing from UNIT_DEFAULTS"
 
     def test_valid_units_in_agg_defaults(self):
         """Test that common units have aggregation defaults defined."""
-        from mpcaHydro.warehouse_functions import AGG_DEFAULTS
+        from mpcaHydro.data_manager_functions import AGG_DEFAULTS
         expected_units = ['mg/l', 'lb', 'cfs', 'degf']
         for unit in expected_units:
             assert unit in AGG_DEFAULTS, f"Unit {unit} missing from AGG_DEFAULTS"
@@ -66,14 +66,14 @@ class TestDataManagerFunctionsInterface:
 
     def test_get_db_path_returns_path(self):
         """Test get_db_path returns a Path object."""
-        from mpcaHydro.warehouse_functions import get_db_path
+        from mpcaHydro.data_manager_functions import get_db_path
         result = get_db_path('/test/path')
         assert isinstance(result, Path)
         assert result.name == 'observations.duckdb'
 
     def test_constants_are_dicts(self):
         """Test constants are dictionaries."""
-        from mpcaHydro.warehouse_functions import AGG_DEFAULTS, UNIT_DEFAULTS
+        from mpcaHydro.data_manager_functions import AGG_DEFAULTS, UNIT_DEFAULTS
         assert isinstance(AGG_DEFAULTS, dict)
         assert isinstance(UNIT_DEFAULTS, dict)
 
@@ -83,12 +83,12 @@ class TestDropStationFunctions:
 
     def test_drop_wiski_station_data_exists(self):
         """Test drop_wiski_station_data function exists and is callable."""
-        from mpcaHydro.warehouse_functions import drop_wiski_station_data
+        from mpcaHydro.data_manager_functions import drop_wiski_station_data
         assert callable(drop_wiski_station_data)
 
     def test_drop_equis_station_data_exists(self):
         """Test drop_equis_station_data function exists and is callable."""
-        from mpcaHydro.warehouse_functions import drop_equis_station_data
+        from mpcaHydro.data_manager_functions import drop_equis_station_data
         assert callable(drop_equis_station_data)
 
 
@@ -98,7 +98,7 @@ class TestDownloadFunctionsReplaceParameter:
     def test_download_wiski_data_has_replace_param(self):
         """Test download_wiski_data accepts replace parameter."""
         import inspect
-        from mpcaHydro.warehouse_functions import download_wiski_data
+        from mpcaHydro.data_manager_functions import download_wiski_data
         sig = inspect.signature(download_wiski_data)
         assert 'replace' in sig.parameters
         assert sig.parameters['replace'].default is False
@@ -106,7 +106,7 @@ class TestDownloadFunctionsReplaceParameter:
     def test_download_equis_data_has_replace_param(self):
         """Test download_equis_data accepts replace parameter."""
         import inspect
-        from mpcaHydro.warehouse_functions import download_equis_data
+        from mpcaHydro.data_manager_functions import download_equis_data
         sig = inspect.signature(download_equis_data)
         assert 'replace' in sig.parameters
         assert sig.parameters['replace'].default is False
@@ -117,12 +117,12 @@ class TestDataManagerWrapper:
 
     def test_wrapper_exists(self):
         """Test DataManagerWrapper class exists."""
-        from mpcaHydro.warehouse_functions import DataManagerWrapper
+        from mpcaHydro.data_manager_functions import DataManagerWrapper
         assert DataManagerWrapper is not None
 
     def test_wrapper_can_be_instantiated(self):
         """Test DataManagerWrapper can be instantiated with a path."""
-        from mpcaHydro.warehouse_functions import DataManagerWrapper
+        from mpcaHydro.data_manager_functions import DataManagerWrapper
         with tempfile.TemporaryDirectory() as tmpdir:
             db_path = Path(tmpdir) / 'test.duckdb'
             wrapper = DataManagerWrapper(db_path)
@@ -130,7 +130,7 @@ class TestDataManagerWrapper:
 
     def test_wrapper_has_expected_methods(self):
         """Test DataManagerWrapper has all expected methods."""
-        from mpcaHydro.warehouse_functions import DataManagerWrapper
+        from mpcaHydro.data_manager_functions import DataManagerWrapper
         with tempfile.TemporaryDirectory() as tmpdir:
             db_path = Path(tmpdir) / 'test.duckdb'
             wrapper = DataManagerWrapper(db_path)
@@ -163,7 +163,7 @@ class TestDataManagerWrapper:
     def test_wrapper_download_methods_have_replace_param(self):
         """Test wrapper download methods have replace parameter."""
         import inspect
-        from mpcaHydro.warehouse_functions import DataManagerWrapper
+        from mpcaHydro.data_manager_functions import DataManagerWrapper
         with tempfile.TemporaryDirectory() as tmpdir:
             db_path = Path(tmpdir) / 'test.duckdb'
             wrapper = DataManagerWrapper(db_path)
@@ -180,13 +180,13 @@ class TestModuleExports:
 
     def test_drop_functions_exported(self):
         """Test drop functions are exported from data_manager_functions."""
-        from mpcaHydro.warehouse_functions import drop_wiski_station_data, drop_equis_station_data
+        from mpcaHydro.data_manager_functions import drop_wiski_station_data, drop_equis_station_data
         assert callable(drop_wiski_station_data)
         assert callable(drop_equis_station_data)
 
     def test_wrapper_exported(self):
         """Test DataManagerWrapper is exported from data_manager_functions."""
-        from mpcaHydro.warehouse_functions import DataManagerWrapper
+        from mpcaHydro.data_manager_functions import DataManagerWrapper
         assert DataManagerWrapper is not None
 
 

mpcahydro-2.2.4/tests/integration/test.json

@@ -1,258 +0,0 @@
-{
-  "repository_name": "Clearwater",
-  "locations": [
-    {
-      "location_id": 1,
-      "location_name": "Clearwater Outlet",
-      "repository_name": "Clearwater",
-      "reach_ids": [
-        650
-      ],
-      "upstream_reach_ids": [
-        649,
-        648
-      ],
-      "flow_station_ids": [
-        "E66050001"
-      ],
-      "stations": [
-        {
-          "station_id": "E66050001",
-          "station_origin": "wiski",
-          "repository_name": "Clearwater",
-          "true_reach_id": 650,
-          "observations": [
-            {
-              "constituent": "Q",
-              "start_year": 2000,
-              "end_year": 2023,
-              "avg_samples_per_year": 365.0,
-              "median_samples_per_year": 365.0,
-              "years_with_data": 24,
-              "total_samples": 8760,
-              "metrics": [
-                {
-                  "name": "NSE",
-                  "target": 0.5,
-                  "weight": 1.0,
-                  "enabled": true
-                },
-                {
-                  "name": "logNSE",
-                  "target": 0.5,
-                  "weight": 1.0,
-                  "enabled": true
-                },
-                {
-                  "name": "Pbias",
-                  "target": 10.0,
-                  "weight": 1.0,
-                  "enabled": true
-                },
-                {
-                  "name": "monthly_average",
-                  "target": 15.0,
-                  "weight": 0.5,
-                  "enabled": true
-                },
-                {
-                  "name": "annual_average",
-                  "target": 15.0,
-                  "weight": 0.5,
-                  "enabled": true
-                },
-                {
-                  "name": "percentile_10",
-                  "target": 20.0,
-                  "weight": 0.5,
-                  "enabled": true
-                },
-                {
-                  "name": "percentile_90",
-                  "target": 20.0,
-                  "weight": 0.5,
-                  "enabled": true
-                },
-                {
-                  "name": "seasonal_average",
-                  "target": 15.0,
-                  "weight": 0.5,
-                  "enabled": true
-                }
-              ],
-              "derived_from": []
-            }
-          ],
-          "comments": "Primary flow monitoring station"
-        },
-        {
-          "station_id": "S002-118",
-          "station_origin": "equis",
-          "repository_name": "Clearwater",
-          "true_reach_id": 650,
-          "observations": [
-            {
-              "constituent": "TSS",
-              "start_year": 2005,
-              "end_year": 2020,
-              "avg_samples_per_year": 12.0,
-              "median_samples_per_year": 10.0,
-              "years_with_data": 15,
-              "total_samples": 180,
-              "metrics": [
-                {
-                  "name": "Pbias",
-                  "target": 25.0,
-                  "weight": 1.0,
-                  "enabled": true
-                },
-                {
-                  "name": "monthly_average",
-                  "target": 25.0,
-                  "weight": 0.5,
-                  "enabled": true
-                },
-                {
-                  "name": "seasonal_average",
-                  "target": 25.0,
-                  "weight": 0.5,
-                  "enabled": true
-                }
-              ],
-              "derived_from": []
-            },
-            {
-              "constituent": "TP",
-              "start_year": 2005,
-              "end_year": 2020,
-              "avg_samples_per_year": 12.0,
-              "median_samples_per_year": 10.0,
-              "years_with_data": 15,
-              "total_samples": 180,
-              "metrics": [
-                {
-                  "name": "Pbias",
-                  "target": 25.0,
-                  "weight": 1.0,
-                  "enabled": true
-                },
-                {
-                  "name": "monthly_average",
-                  "target": 25.0,
-                  "weight": 0.5,
-                  "enabled": true
-                },
-                {
-                  "name": "seasonal_average",
-                  "target": 25.0,
-                  "weight": 0.5,
-                  "enabled": true
-                }
-              ],
-              "derived_from": []
-            },
-            {
-              "constituent": "TP_load",
-              "start_year": null,
-              "end_year": null,
-              "avg_samples_per_year": null,
-              "median_samples_per_year": null,
-              "years_with_data": null,
-              "total_samples": null,
-              "metrics": [
-                {
-                  "name": "Pbias",
-                  "target": 30.0,
-                  "weight": 1.0,
-                  "enabled": true
-                }
-              ],
-              "derived_from": [
-                "TP",
-                "Q"
-              ]
-            }
-          ],
-          "comments": "Water quality monitoring station"
-        }
-      ],
-      "watershed_constraints": [
-        {
-          "constituent": "TP",
-          "target_rate": 0.5,
-          "min_rate": 0.2,
-          "max_rate": 1.0,
-          "landcover_constraints": [
-            {
-              "landcover_type": "forest",
-              "constituent": "TP",
-              "target_rate": 0.1,
-              "min_rate": 0.05,
-              "max_rate": 0.2
-            },
-            {
-              "landcover_type": "agricultural",
-              "constituent": "TP",
-              "target_rate": 0.8,
-              "min_rate": 0.4,
-              "max_rate": 1.5
-            }
-          ]
-        }
-      ],
-      "notes": "Example calibration location combining flow and water quality stations"
-    }
-  ],
-  "default_metrics": [
-    {
-      "name": "NSE",
-      "target": 0.5,
-      "weight": 1.0,
-      "enabled": true
-    },
-    {
-      "name": "logNSE",
-      "target": 0.5,
-      "weight": 1.0,
-      "enabled": true
-    },
-    {
-      "name": "Pbias",
-      "target": 10.0,
-      "weight": 1.0,
-      "enabled": true
-    },
-    {
-      "name": "monthly_average",
-      "target": 15.0,
-      "weight": 0.5,
-      "enabled": true
-    },
-    {
-      "name": "annual_average",
-      "target": 15.0,
-      "weight": 0.5,
-      "enabled": true
-    },
-    {
-      "name": "percentile_10",
-      "target": 20.0,
-      "weight": 0.5,
-      "enabled": true
-    },
-    {
-      "name": "percentile_90",
-      "target": 20.0,
-      "weight": 0.5,
-      "enabled": true
-    },
-    {
-      "name": "seasonal_average",
-      "target": 15.0,
-      "weight": 0.5,
-      "enabled": true
-    }
-  ],
-  "general_constraints": [],
-  "version": "1.0"
-}

mpcahydro-2.2.4/tests/integration/test_data_manager.py

@@ -1,120 +0,0 @@
-
-from mpcaHydro import calibration_config, calibration_dataclasses, outlets, warehouse_functions as dmf
-from mpcaHydro import wiski
-
-
-dm = dmf.DataManagerWrapper('test_db.duckb',True)
-station_ids = ['E67013001']
-
-dm.download_wiski_data(station_ids)
-
-dmf.download_wiski_data(con,station_ids,start_year = 2010,)
-
-station_ids = ['H67012001']
-dm.download_wiski_data(station_ids)
-
-model_name = 'Rum'
-config = calibration_config.create_config(model_name)
-
-parts = {value: subdf.copy() for value, subdf in  outlets.get_outlets_by_model(model_name).groupby('outlet_id') }
-
-
-
-folder_path = Path('C:/Users/mfratki/Documents/Projects/Tests')
-oracle_username = 'MFRATKI'
-oracle_password = 'Deltat#3xg4duw1'
-oracle_connection = equis.connect(oracle_username, oracle_password)
-
-
-for model_name in Repository.valid_models():    
-    wiski_stations = outlets.mapped_wiski_stations(model_name)
-    equis_stations = outlets.mapped_equis_stations(model_name)
-    wiski_info = wiski.info(wiski_stations)
-    equis_info = equis.info(equis_stations,oracle_connection)
-    stations = []
-    
-    for station_id, info in wiski_info.groupby('station_id'):
-        observations = []
-        for constituent in info['constituent'].unique():
-            metrics = calibration_config.get_default_timeseries_metrics()
-            observations.append(calibration_dataclasses.Observation(constituent,
-                                                    1996,
-                                                    2024,
-                                                    metrics = metrics))
-            #TODO: get_Default_timeseries_metrics(constituent)
-        
-        stations.append(calibration_dataclasses.Station(station_id,
-                                        'wiski',
-                                        model_name,
-                                        observations = observations))
-        
-    for station_id, info in equis_info.groupby('station_id'):
-        observations = []
-        for constituent in info['constituent'].unique():
-            metrics = calibration_config.get_default_discrete_metrics()
-            observations.append(calibration_dataclasses.Observation(constituent,
-                                                        1996,
-                                                        2024,
-                                                        metrics = metrics))
-            #TODO: get_Default_timeseries_metrics(constituent)
-
-        stations.append(calibration_dataclasses.Station(station_id,
-                                        'equis',
-                                        model_name,
-                                        observations = observations))
-
-    locations = []
-    locs = outlets.get_outlets_by_model(model_name)
-    for outlet_id, info in locs.groupby('outlet_id'):
-        loc_stations = []
-        for station_id,station_origin in zip(info['station_id'],info['station_origin']):
-            for station in stations:
-                if station.station_id == station_id and station.station_origin == station_origin:
-                    loc_stations.append(station)
-        reach_ids = info['reach_id'].tolist()
-        locations.append(calibration_dataclasses.Location(outlet_id,
-                                                          str(outlet_id),
-                                                          model_name,
-                                                          reach_ids,
-                                                          stations = loc_stations))
-
-config = calibration_dataclasses.CalibrationConfig(model_name,
-                                          locations)
-
-df = wiski.info(wiski_stations)
-
-obs = calibration_dataclasses.Observation('Q',
-                                    1996,
-                                    2024)
-
-
-for location in locations:
-    reach_ids = location.reach_ids
-    upstream_reach_ids = location.upstream_reach_ids
-    flow_station_ids = []
-    station_ids = location.station_ids
-
-    calibration_dataclasses.Observation(constituent,)
-
-    calibration_dataclasses.Station(station_id,
-                                    station_origin,
-                                    model_name
-                                    )
-    
-    
-    calibration_dataclasses.Location(1,
-                                 'test',
-                                 model_name,
-                                 reach_ids,
-                                 upstream_reach_ids,
-                                 flow_station_ids,
-                                 station_ids,[])
-station_ids = ['H67009001',
- 'H67012001',
- 'H67013001',
- 'H67014001',
- 'H67016001',
- 'H67014002',
- 'E67013001',
- 'W67036001',
- 'W67004001']