mpcaHydro 2.2.3__tar.gz → 2.2.4__tar.gz

{mpcahydro-2.2.3 → mpcahydro-2.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpcaHydro
-Version: 2.2.3
+Version: 2.2.4
 Summary: Python package for downloading MPCA hydrology data
 Project-URL: Homepage, https://github.com/mfratkin1/mpcaHydro
 Author-email: Mulu Fratkin <michael.fratkin@state.mn.us>

{mpcahydro-2.2.3 → mpcahydro-2.2.4}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "mpcaHydro"
 urls = { "Homepage" = "https://github.com/mfratkin1/mpcaHydro" }  # ? Add this!
-version = "2.2.3"
+version = "2.2.4"
 dependencies = [
   "pandas",
   "requests",

mpcahydro-2.2.4/src/mpcaHydro/__init__.py

@@ -0,0 +1,4 @@
+# mpcaHydro - Modules for downloading hydrology data from MPCA servers and databases
+
+# Modules are imported lazily to avoid issues with missing data files
+# Use: from mpcaHydro import calibration_config

mpcahydro-2.2.4/src/mpcaHydro/data/wiskiweb01.pca.state.mn.us.crt

@@ -0,0 +1,48 @@
+-----BEGIN CERTIFICATE-----
+MIIIjzCCBvegAwIBAgIQIOC9Vbo5TbuTGYi1z42GyjANBgkqhkiG9w0BAQsFADBg
+MQswCQYDVQQGEwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTcwNQYDVQQD
+Ey5TZWN0aWdvIFB1YmxpYyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gQ0EgRVYgUjM2
+MB4XDTI1MDkyNTAwMDAwMFoXDTI2MDkyNTIzNTk1OVowgbAxGjAYBgNVBAUTEUdv
+dmVybm1lbnQgRW50aXR5MRMwEQYLKwYBBAGCNzwCAQMTAlVTMRowGAYDVQQPExFH
+b3Zlcm5tZW50IEVudGl0eTELMAkGA1UEBhMCVVMxEjAQBgNVBAgTCU1pbm5lc290
+YTEbMBkGA1UEChMSU3RhdGUgb2YgTWlubmVzb3RhMSMwIQYDVQQDExp3aXNraXdl
+YjAxLnBjYS5zdGF0ZS5tbi51czCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC
+ggIBAJ8aX33B4Jq37RF0QcjYrsxKwjiFLDcgrLGzsUCm/WJUuvsX3pbU8rfpYte8
+BuI6XyudiPpwqPezSInc3hFCbZzYmdcy/RgjsQ1Uqsfj+0NB3A44CHGh6NHY59X0
+thAroUgnz1gcnHfHiHvJPHwYOAnhbVYel8hBBfIq2wqnk2B1Z99DM9u/Q9/ZGpIg
+2HvbWg+8YjoMVQl3vH1jycE0i3VVTLWr5OfkV1jhl/rdFSq+Ott8Ipmni0J0GBox
+EHNKaTbBwB5EzV6a67uiiA6x5rkNEIM0oMCGWdAvm5hT2gzwVwj+kvSgQHsls4jy
+Gyjxlcy7N3A43dxTs/203jiGk9M55C9JEf9RbmA0muDxn7OlAmEzCyMY20BdlGBI
+fBkPtByIE2MOOIz6sG0PdhO57+7nIvSMftgJfjlhszwtMrpLLQr2gOqk5/XFOqw4
+SSXRQC24mKQ78q1OIohhfeLJdecAkPGcb0uSYQdpOA8RO0hgBGvsFBu/8zn/J3WM
+Pn2wYvE13xz39XQ6G3ySH/MqC7cY9FlzM+RMk6qQyionZ9O7lC6tFufyJNX1E9Ps
+W9E4enrunXHG5k+5glA/43hmhs8CbaeK4xdv1ZuJj7TvLLPAZ3+ojh6tzyV2iK66
+YRzWOW3IaXIZ1/hoWlFHr++WcWJi72k3C85TdUITvxJKoqgrAgMBAAGjggNyMIID
+bjAfBgNVHSMEGDAWgBSYLV4ej+tU9Ln/VZWtTMd+pJiuezAdBgNVHQ4EFgQUaW8x
+VX6iXFQFTWYYY6ZZZyCBl9owDgYDVR0PAQH/BAQDAgWgMAwGA1UdEwEB/wQCMAAw
+HQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMEkGA1UdIARCMEAwNQYMKwYB
+BAGyMQECAQUBMCUwIwYIKwYBBQUHAgEWF2h0dHBzOi8vc2VjdGlnby5jb20vQ1BT
+MAcGBWeBDAEBMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly9jcmwuc2VjdGlnby5j
+b20vU2VjdGlnb1B1YmxpY1NlcnZlckF1dGhlbnRpY2F0aW9uQ0FFVlIzNi5jcmww
+gYQGCCsGAQUFBwEBBHgwdjBPBggrBgEFBQcwAoZDaHR0cDovL2NydC5zZWN0aWdv
+LmNvbS9TZWN0aWdvUHVibGljU2VydmVyQXV0aGVudGljYXRpb25DQUVWUjM2LmNy
+dDAjBggrBgEFBQcwAYYXaHR0cDovL29jc3Auc2VjdGlnby5jb20wRQYDVR0RBD4w
+PIIad2lza2l3ZWIwMS5wY2Euc3RhdGUubW4udXOCHnd3dy53aXNraXdlYjAxLnBj
+YS5zdGF0ZS5tbi51czCCAX4GCisGAQQB1nkCBAIEggFuBIIBagFoAHcA2AlVO5RP
+ev/IFhlvlE+Fq7D4/F6HVSYPFdEucrtFSxQAAAGZgmRjOAAABAMASDBGAiEAmmM+
+KMiBitsTZegFbO80sXyZSvBejMJ1zoyOKAtKLmwCIQDfbCgMVJoTyhmqTyTHSd/w
+OmlNc/v9YCiBaYgq3mpvwwB2AK9niDtXsE7dj6bZfvYuqOuBCsdxYPAkXlXWDC/n
+hYc6AAABmYJkY6MAAAQDAEcwRQIgQduMFxbWG6okiji3zTQsobhymG4Hj06i3q95
+/+xIJLUCIQC8DYbjvaUudd6TgiJWoKMYf9pYIirfPJmID129oLkAZQB1AKyrMHBs
+6+yEMfQT0vSRXxEeQiRDsfKmjE88KzunHgLDAAABmYJkYswAAAQDAEYwRAIgCyMR
+cyQpwfFfIpp5qTTjuZVK1IvnGoIWvWTYbQkojzYCIAM+XXKTC7a91IwdC51PidF2
+Pr21+PSOptD7pN8C9pD7MA0GCSqGSIb3DQEBCwUAA4IBgQBqfDG3CPDoY1tE56fp
+gGcqVlXknE5ttqTqDzJtQUagf6OAQ52d8McwsWsV7IIjpMXCNikJFw7crE45ICg8
+FCaqUkpQb9Du+OhNUJNHQiC82gPd4xo3VTK3mM2wrhesXrOhenpzhz0z5TRFXX2J
+8dl2k/sC0fz7QBzw1iCmucG+4f3rvQdr5zfHVqblSC0k6fP6N7cVSCB3d+nMZiHP
+2FhHJnWHzLqPynarKj7aBugCwv+/jAkT8xfyzYhf4X7IH1L5uCHKB+phHj1/AqG2
+n2aVDooyfCYqdAwFiUCtg8iEeXvXBc9Kcov81H94oq1yuZrQOy6lrJ8UpnDO5Zlu
+C4jzg/mSgDioLmSrIQYFEp8R77xbjRBMb0mqko/Hvp5bysb5XTWSgmJrR+3/b/0E
+jkIBv5LdEuxNrd28L1rneDEztIVbTc7bq/V9fQlq6j4YJ+BMh442DMrGnRweicD5
+PFFFejW+IfQT8Z9GwApcgGXNne3SzfAd5PtJqxdXC1KZkbk=
+-----END CERTIFICATE-----

{mpcahydro-2.2.3 → mpcahydro-2.2.4}/src/mpcaHydro/equis.py

@@ -2,7 +2,7 @@
 
 from datetime import datetime, timezone, timedelta
 import pandas as pd
-from typing import Union
+from typing import Union, Optional
 import oracledb
 import duckdb
 
@@ -33,10 +33,18 @@ def connect(user: str, password: str, host: str = "DELTAT", port: int = 1521, si
                                  sid=sid) 
     return CONNECTION
 
-def close_connection():
-    '''Close the global Oracle database connection if it exists.'''
+def close_connection(connection: Optional[oracledb.Connection] = None):
+    '''Close the Oracle database connection.
+    
+    Args:
+        connection: Optional connection to close. If not provided, closes global CONNECTION.
+    '''
     global CONNECTION
-    if CONNECTION:
+    if connection is not None:
+        connection.close()
+        if connection is CONNECTION:
+            CONNECTION = None
+    elif CONNECTION is not None:
         CONNECTION.close()
         CONNECTION = None
 
@@ -80,11 +88,32 @@ def to_dataframe(odb_cursor):
     return df
 
 #%% Query for station locations with HSPF related constituents
- 
-def download(station_ids):
+
+def info(station_ids, connection: Optional[oracledb.Connection] = None):
+    '''Get information for given station IDs from Oracle database.'''
+    conn = connection if connection is not None else CONNECTION
+    if conn is None:
+        raise ValueError("No connection provided and global CONNECTION is not set. Call connect() first or pass a connection.")
+    
+    df = normalize(download(station_ids, connection=conn)).drop_duplicates(subset=['station_id','constituent'])
+    return df
+
+    
+
+
+def download(station_ids, connection: Optional[oracledb.Connection] = None):
     '''Download data for given station IDs from Oracle database.
     This grabs data from the Data access Layer (DAL) equis result view for
-    river/stream locations and HSPF related constituents only.'''
+    river/stream locations and HSPF related constituents only.
+    
+    Args:
+        station_ids: List of station IDs to download
+        connection: Optional Oracle connection. If not provided, uses global CONNECTION.
+    '''
+    conn = connection if connection is not None else CONNECTION
+    if conn is None:
+        raise ValueError("No connection provided and global CONNECTION is not set. Call connect() first or pass a connection.")
+    
     placeholders, binds = make_placeholders(station_ids)
     query = f"""
 SELECT
@@ -122,7 +151,7 @@ SELECT
         AND mpca_dal.eq_sample.sample_method IN ('G-EVT', 'G', 'FIELDMSROBS', 'LKSURF1M', 'LKSURF2M', 'LKSURFOTH')
         AND mpca_dal.mv_eq_result.sys_loc_code IN ({placeholders})
     """
-    with CONNECTION.cursor() as cursor:
+    with conn.cursor() as cursor:
         cursor.execute(query,binds)
         return to_dataframe(cursor)
     
@@ -441,8 +470,16 @@ def transform(df):
 #         GROUP BY station_id, DATE_TRUNC('hour', datetime), constituent, unit
 #     """ )
 
-def fetch_station_locations():
-    '''Fetch station location data for stations with HSPF related constituents.'''
+def fetch_station_locations(connection: Optional[oracledb.Connection] = None):
+    '''Fetch station location data for stations with HSPF related constituents.
+    
+    Args:
+        connection: Optional Oracle connection. If not provided, uses global CONNECTION.
+    '''
+    conn = connection if connection is not None else CONNECTION
+    if conn is None:
+        raise ValueError("No connection provided and global CONNECTION is not set. Call connect() first or pass a connection.")
+    
     query ="""SELECT DISTINCT
     m.SYS_LOC_CODE,
     stn.LONGITUDE,
@@ -467,22 +504,7 @@ def fetch_station_locations():
                         'TEMP-W',
                         '7664-41-7')
         """
-    with CONNECTION.cursor() as cursor:
+    with conn.cursor() as cursor:
         cursor.execute(query)
         df = to_dataframe(cursor)
-        
-        # dups = set(df.loc[df['SYS_LOC_CODE'].isin(df.loc[df['SYS_LOC_CODE'].duplicated()]['SYS_LOC_CODE']),'SYS_LOC_CODE'].to_list())
-        # for dup in dups:
-        #     #percent difference between lat/long values
-        #     sub = df.loc[df['SYS_LOC_CODE'] == dup]
-        #     lat_diff = abs(sub['LATITUDE'].max() - sub['LATITUDE'].min()) / ((sub['LATITUDE'].max() + sub['LATITUDE'].min()) / 2) * 100
-        #     long_diff = abs(sub['LONGITUDE'].max() - sub['LONGITUDE'].min()) / ((sub['LONGITUDE'].max() + sub['LONGITUDE'].min()) / 2) * 100
-        #     print(f'Duplicate station {dup} has {lat_diff:.6f}% latitude difference')
-        #     print(f'Duplicate station {dup} has {long_diff:.6f}% longitude difference')
-
-        
-        # geometry = gpd.points_from_xy(df['LONGITUDE'], df['LATITUDE'])
-        # gdf = gpd.GeoDataFrame(df, geometry=geometry, crs="EPSG:4326")
-        # filename = 'EQ_STATION_' + str(date.today()) + '.gpkg'
-        # gdf.to_file(save_path.joinpath(filename), driver = 'GPKG')
-        # gdf.rename(columns={'SYS_LOC_CODE':'station_id'}, inplace=True)
+        return df

{mpcahydro-2.2.3 → mpcahydro-2.2.4}/src/mpcaHydro/outlets.py

@@ -9,6 +9,7 @@ from pathlib import Path
 import geopandas as gpd
 import pandas as pd
 import duckdb
+from mpcaHydro.sql_loader import get_outlets_schema_sql
 #from hspf_tools.calibrator import etlWISKI, etlSWD
 
 
@@ -79,10 +80,14 @@ def equis_station_opnids(model_name):
     opnids = MODL_DB.dropna(subset=['opnids']).query('repo_name == @model_name and source == "equis"')['opnids'].str.split(',').to_list()
     return split_opnids(opnids)
 
-def station_opnids(model_name):
-    opnids = MODL_DB.dropna(subset=['opnids']).query('repo_name == @model_name')['opnids'].str.split(',').to_list()
+def mapped_station_opnids(station_id, station_origin):
+    opnids = MODL_DB.dropna(subset=['opnids']).query('station_id == @station_id and source == @station_origin')['opnids'].str.split(',').to_list()
     return split_opnids(opnids)
 
+def mapped_stations(model_name,station_origin):
+    assert(station_origin in ['wiski', 'equis'])
+    return MODL_DB.dropna(subset=['opnids']).query('repo_name == @model_name and source == @station_origin')['station_id'].tolist()
+    
 def mapped_equis_stations(model_name):
     return MODL_DB.dropna(subset=['opnids']).query('repo_name == @model_name and source == "equis"')['station_id'].tolist()
 
@@ -108,9 +113,8 @@ def init_db(db_path: str,reset: bool = False):
     db_path = Path(db_path)
     if reset and db_path.exists():
         db_path.unlink()
-
     with connect(db_path.as_posix(),False) as con:
-        con.execute(OUTLETS_SCHEMA)
+        con.execute(get_outlets_schema_sql())
 
 
 
@@ -293,60 +297,6 @@ def add_reach(con,
     )
 
 
-OUTLETS_SCHEMA  = """-- schema.sql
--- Simple 3-table design to manage associations between model reaches and observation stations via outlets.
--- Compatible with DuckDB and SQLite.
-
--- Table 1: outlets
--- Represents a logical grouping that ties stations and reaches together.
-CREATE SCHEMA IF NOT EXISTS outlets;
-
-CREATE TABLE IF NOT EXISTS outlets.outlet_groups  (
-  outlet_id INTEGER PRIMARY KEY,
-  repository_name TEXT NOT NULL,
-  outlet_name TEXT,
-  notes TEXT             -- optional: general notes about the outlet grouping
-);
-
--- Table 2: outlet_stations
--- One-to-many: outlet -> stations
-CREATE TABLE IF NOT EXISTS outlets.outlet_stations (
-  outlet_id INTEGER NOT NULL,
-  station_id TEXT NOT NULL,
-  station_origin TEXT NOT NULL,       -- e.g., 'wiski', 'equis'
-  repository_name TEXT NOT NULL,  -- repository model the station is physically located in
-  true_opnid INTEGER NOT NULL,           -- The specific reach the station physically sits on (optional)
-  comments TEXT,             -- Per-station comments, issues, etc.
-  CONSTRAINT uq_station_origin UNIQUE (station_id, station_origin),
-  FOREIGN KEY (outlet_id) REFERENCES outlets.outlet_groups(outlet_id)
-);
-
--- Table 3: outlet_reaches
--- One-to-many: outlet -> reaches
--- A reach can appear in multiple outlets, enabling many-to-many overall.
-CREATE TABLE IF NOT EXISTS outlets.outlet_reaches (
-  outlet_id INTEGER NOT NULL,
-  reach_id INTEGER NOT NULL,    -- model reach identifier (aka opind)
-  repository_name TEXT NOT NULL,  -- optional: where the mapping comes from
-  FOREIGN KEY (outlet_id) REFERENCES outlets.outlet_groups(outlet_id)
-);
-
--- Useful views:
-
--- View: station_reach_pairs
--- Derives the implicit many-to-many station <-> reach relationship via shared outlet_id
-CREATE OR REPLACE VIEW outlets.station_reach_pairs AS
-SELECT
-  s.outlet_id,
-  s.station_id,
-  s.station_origin,
-  r.reach_id,
-  r.repository_name
-FROM outlets.outlet_stations AS s
-JOIN outlets.outlet_reaches AS r
-  ON s.outlet_id = r.outlet_id;
-
-"""  
     
 #row = modl_db.MODL_DB.iloc[0]
 

{mpcahydro-2.2.3 → mpcahydro-2.2.4}/src/mpcaHydro/pywisk.py

@@ -4,14 +4,16 @@ Created on Mon Jul 10 16:18:03 2023
 
 @author: mfratki
 """
+from pathlib import Path
 import requests
 from requests.exceptions import ConnectionError, Timeout, HTTPError, RequestException
 import pandas as pd
 import time
 
+CERT_PATH = str(Path(__file__).resolve().parent/'data\\wiskiweb01.pca.state.mn.us.crt')
 #TODO: Use this url to make sure web service is working https://wiskiweb01.pca.state.mn.us/
 class Service():
-    base_url = 'http://wiskiweb01.pca.state.mn.us/KiWIS/KiWIS?'
+    base_url = 'https://wiskiweb01.pca.state.mn.us/KiWIS/KiWIS?'
     base_dict = {
         'datasource': '0',
         'service': 'kisters',
@@ -28,7 +30,7 @@ class Service():
         try:
             # Using requests.head() to fetch headers is faster than requests.get()
             # as it doesn't download the full content
-            response = requests.head('http://wiskiweb01.pca.state.mn.us', timeout=timeout)
+            response = requests.head('https://wiskiweb01.pca.state.mn.us', timeout=timeout)
             
             # raise_for_status() raises an HTTPError for 4xx or 5xx status codes
             response.raise_for_status() 
@@ -51,7 +53,7 @@ class Service():
 
     def _requestTypes(self):
         url = self.url({'request': 'getrequestinfo'}) 
-        return requests.get(url).json()[0]  
+        return requests.get(url,verify=CERT_PATH).json()[0]  
         
     def getRequests(self):
         return list(self._requestTypes()['Requests'].keys())
@@ -70,7 +72,7 @@ class Service():
     
     def info(self,request_type):
         url = self.url({'request': 'getrequestinfo'})
-        response = requests.get(url)
+        response = requests.get(url, verify=CERT_PATH)
         get_requests = response.json()    
         return get_requests[0]['Requests'].keys()
         
@@ -93,7 +95,7 @@ class Service():
     
     def get_json(self,args_dict):
         # Download request
-        self.response = requests.get(self.url(args_dict))
+        self.response = requests.get(self.url(args_dict), verify=CERT_PATH)
         if self.response.status_code != 200:
             print('Error: ' + self.response.json()['message'])
             self.response.raise_for_status()  # raises exception when not a 2xx response
@@ -192,7 +194,10 @@ def construct_aggregation(interval, aggregation_type):
     return f'aggregate({interval}~{aggregation_type})'
 
 def validate_aggregation_type(aggregation_type):
-    assert(aggregation_type in VALID_AGGREGATION_TYPES or validate_percentile(aggregation_type))
+    if aggregation_type.startswith('perc-'):
+        validate_percentile(aggregation_type)
+    else: 
+        assert(aggregation_type in VALID_AGGREGATION_TYPES)
     return True
    
 def validate_percentile(aggregation_type):
@@ -297,7 +302,8 @@ def get_stations(
     return df
 
 def get_ts_ids(
-                station_nos,
+                station_nos = None,
+                ts_ids = None,
                 parametertype_id = None,
                 stationparameter_no = None,
                 stationgroup_id = None,
@@ -317,6 +323,7 @@ def get_ts_ids(
 
     args ={'request': 'getTimeseriesList',
             'station_no': station_nos,
+            'ts_id': ts_ids,
             'parametertype_id': parametertype_id,
             'stationparameter_no': stationparameter_no,
             'ts_name' : ts_name,

mpcahydro-2.2.4/src/mpcaHydro/sql/analytics_tables.sql

@@ -0,0 +1,20 @@
+-- analytics_tables.sql
+-- Create tables in the analytics schema for processed/transformed data
+
+CREATE TABLE IF NOT EXISTS analytics.equis (
+    datetime TIMESTAMP,
+    value DOUBLE,
+    station_id TEXT,
+    station_origin TEXT,
+    constituent TEXT,
+    unit TEXT
+);
+
+CREATE TABLE IF NOT EXISTS analytics.wiski (
+    datetime TIMESTAMP,
+    value DOUBLE,
+    station_id TEXT,
+    station_origin TEXT,
+    constituent TEXT,
+    unit TEXT
+);

mpcahydro-2.2.4/src/mpcaHydro/sql/outlets_schema.sql

@@ -0,0 +1,53 @@
+-- outlets_schema.sql
+-- Schema for managing associations between model reaches and observation stations via outlets
+-- Compatible with DuckDB and SQLite
+
+CREATE SCHEMA IF NOT EXISTS outlets;
+
+-- Table 1: outlet_groups
+-- Represents a logical grouping that ties stations and reaches together
+CREATE TABLE IF NOT EXISTS outlets.outlet_groups (
+    outlet_id INTEGER PRIMARY KEY,
+    repository_name TEXT NOT NULL,
+    outlet_name TEXT,
+    notes TEXT
+);
+
+-- Table 2: outlet_stations
+-- One-to-many: outlet -> stations
+CREATE TABLE IF NOT EXISTS outlets.outlet_stations (
+    outlet_id INTEGER NOT NULL,
+    station_id TEXT NOT NULL,
+    station_origin TEXT NOT NULL,
+    repository_name TEXT NOT NULL,
+    true_opnid INTEGER NOT NULL,
+    comments TEXT,
+    CONSTRAINT uq_station_origin UNIQUE (station_id, station_origin),
+    FOREIGN KEY (outlet_id) REFERENCES outlets.outlet_groups(outlet_id)
+);
+
+-- Table 3: outlet_reaches
+-- One-to-many: outlet -> reaches
+-- A reach can appear in multiple outlets, enabling many-to-many overall
+CREATE TABLE IF NOT EXISTS outlets.outlet_reaches (
+    outlet_id INTEGER NOT NULL,
+    reach_id INTEGER NOT NULL,
+    repository_name TEXT NOT NULL,
+    FOREIGN KEY (outlet_id) REFERENCES outlets.outlet_groups(outlet_id)
+);
+
+-- Useful views:
+
+-- View: station_reach_pairs
+-- Derives the implicit many-to-many station <-> reach relationship via shared outlet_id
+CREATE OR REPLACE VIEW outlets.station_reach_pairs AS
+SELECT
+  s.outlet_id,
+  s.station_id,
+  s.station_origin,
+  r.reach_id,
+  r.repository_name
+FROM outlets.outlet_stations AS s
+JOIN outlets.outlet_reaches AS r
+  ON s.outlet_id = r.outlet_id;
+

mpcahydro-2.2.4/src/mpcaHydro/sql/schemas.sql

@@ -0,0 +1,8 @@
+-- schemas.sql
+-- Create all database schemas for the data warehouse
+
+CREATE SCHEMA IF NOT EXISTS staging;
+CREATE SCHEMA IF NOT EXISTS analytics;
+CREATE SCHEMA IF NOT EXISTS reports;
+CREATE SCHEMA IF NOT EXISTS outlets;
+CREATE SCHEMA IF NOT EXISTS mappings;

mpcahydro-2.2.4/src/mpcaHydro/sql/staging_tables.sql

@@ -0,0 +1,90 @@
+-- staging_tables.sql
+-- Create tables in the staging schema for raw data from external sources
+
+CREATE TABLE IF NOT EXISTS staging.equis(
+    LATITUDE DOUBLE,
+    LONGITUDE DOUBLE,
+    WID_LIST VARCHAR,
+    SAMPLE_METHOD VARCHAR,
+    SAMPLE_REMARK VARCHAR,
+    FACILITY_ID BIGINT,
+    FACILITY_NAME VARCHAR,
+    FACILITY_TYPE VARCHAR,
+    SYS_LOC_CODE VARCHAR,
+    LOC_NAME VARCHAR,
+    LOC_TYPE VARCHAR,
+    LOC_TYPE_2 VARCHAR,
+    TASK_CODE VARCHAR,
+    SAMPLE_ID BIGINT,
+    SYS_SAMPLE_CODE VARCHAR,
+    TEST_ID BIGINT,
+    ANALYTE_TYPE VARCHAR,
+    ANALYTE_TYPE_DESC VARCHAR,
+    ANALYTIC_METHOD VARCHAR,
+    PREFERRED_NAME VARCHAR,
+    PARAMETER VARCHAR,
+    CAS_RN VARCHAR,
+    CHEMICAL_NAME VARCHAR,
+    GTLT VARCHAR,
+    RESULT_TEXT VARCHAR,
+    RESULT_NUMERIC DOUBLE,
+    RESULT_UNIT VARCHAR,
+    STAT_TYPE INTEGER,
+    VALUE_TYPE VARCHAR,
+    DETECT_FLAG VARCHAR,
+    DETECT_DESC VARCHAR,
+    RESULT_REMARK VARCHAR,
+    RESULT_TYPE_CODE VARCHAR,
+    METHOD_DETECTION_LIMIT VARCHAR,
+    REPORTING_DETECTION_LIMIT VARCHAR,
+    QUANTITATION_LIMIT INTEGER,
+    LAB_QUALIFIERS VARCHAR,
+    INTERPRETED_QUALIFIERS VARCHAR,
+    REPORTABLE_RESULT VARCHAR,
+    APPROVAL_CODE VARCHAR,
+    SENSITIVE_NOTPUBLIC VARCHAR,
+    TEST_TYPE VARCHAR,
+    DILUTION_FACTOR DOUBLE,
+    FRACTION VARCHAR,
+    BASIS VARCHAR,
+    TEMP_BASIS VARCHAR,
+    TEST_REMARK VARCHAR,
+    ANALYSIS_DATE_TIME TIMESTAMP_NS,
+    ANALYSIS_DATE VARCHAR,
+    ANALYSIS_TIME VARCHAR,
+    ANALYSIS_DATE_TIMEZONE VARCHAR,
+    COMPANY_NAME VARCHAR,
+    LAB_NAME_CODE VARCHAR,
+    LAB_SAMPLE_ID VARCHAR,
+    SAMPLE_TYPE_GROUP VARCHAR,
+    SAMPLE_TYPE_CODE VARCHAR,
+    SAMPLE_TYPE_DESC VARCHAR,
+    MEDIUM_CODE VARCHAR,
+    MATRIX_CODE VARCHAR,
+    START_DEPTH DOUBLE,
+    DEPTH_UNIT VARCHAR,
+    SAMPLE_DATE_TIME TIMESTAMP_NS,
+    SAMPLE_DATE VARCHAR,
+    SAMPLE_TIME VARCHAR,
+    SAMPLE_DATE_TIMEZONE VARCHAR,
+    EBATCH DOUBLE
+);
+
+CREATE TABLE IF NOT EXISTS staging.wiski(
+    "Timestamp" VARCHAR,
+    "Value" DOUBLE,
+    "Quality Code" BIGINT,
+    "Quality Code Name" VARCHAR,
+    ts_unitsymbol VARCHAR,
+    ts_name VARCHAR,
+    ts_id VARCHAR,
+    station_no VARCHAR,
+    station_name VARCHAR,
+    station_latitude VARCHAR,
+    station_longitude VARCHAR,
+    parametertype_id VARCHAR,
+    parametertype_name VARCHAR,
+    stationparameter_no VARCHAR,
+    stationparameter_name VARCHAR,
+    wplmn_flag BIGINT
+);

mpcahydro-2.2.4/src/mpcaHydro/sql/views_analytics.sql

@@ -0,0 +1,117 @@
+-- views_analytics.sql
+-- Views for the analytics schema
+
+-- View: wiski_normalized
+-- Normalized WISKI data with unit conversions and column renames
+-- CREATE OR REPLACE VIEW analytics.wiski_normalized AS
+-- SELECT 
+--     -- Convert °C to °F and keep other values unchanged
+--     CASE 
+--         WHEN LOWER(ts_unitsymbol) = '°c' THEN (value * 9.0 / 5.0) + 32
+--         WHEN ts_unitsymbol = 'kg' THEN value * 2.20462
+--         ELSE value
+--     END AS value,
+
+--     -- Normalize units
+--     CASE 
+--         WHEN LOWER(ts_unitsymbol) = '°c' THEN 'degf'
+--         WHEN ts_unitsymbol = 'kg' THEN 'lb'
+--         WHEN ts_unitsymbol = 'ft³/s' THEN 'cfs'
+--         ELSE ts_unitsymbol
+--     END AS unit,
+
+--     -- Normalize column names
+--     station_no AS station_id,
+--     Timestamp AS datetime,
+--     "Quality Code" AS quality_code,
+--     "Quality Code Name" AS quality_code_name,
+--     parametertype_id,
+--     constituent
+-- FROM staging.wiski;
+
+-- View: observations
+-- Combined observations from equis and wiski processed tables
+CREATE OR REPLACE VIEW analytics.observations AS
+SELECT datetime, value, station_id, station_origin, constituent, unit
+FROM analytics.equis
+UNION ALL
+SELECT datetime, value, station_id, station_origin, constituent, unit
+FROM analytics.wiski;
+
+-- View: outlet_observations
+-- Links observations to model reaches via outlets
+CREATE OR REPLACE VIEW analytics.outlet_observations AS 
+SELECT
+    o.datetime,
+    os.outlet_id,
+    o.constituent,
+    AVG(o.value) AS value,
+    COUNT(o.value) AS count
+FROM
+    analytics.observations AS o
+INNER JOIN
+    outlets.outlet_stations AS os 
+    ON o.station_id = os.station_id AND o.station_origin = os.station_origin
+WHERE os.outlet_id IS NOT NULL
+GROUP BY
+    os.outlet_id,
+    o.constituent,
+    o.datetime;
+
+-- View: outlet_observations_with_flow
+-- Outlet observations joined with flow and baseflow data
+CREATE OR REPLACE VIEW analytics.outlet_observations_with_flow AS
+WITH 
+    baseflow_data AS (
+        SELECT
+            outlet_id,
+            datetime,
+            "value" AS baseflow_value
+        FROM
+            analytics.outlet_observations
+        WHERE
+            constituent = 'QB'
+    ),
+
+    flow_data AS (
+        SELECT
+            outlet_id,
+            datetime,
+            "value" AS flow_value
+        FROM
+            analytics.outlet_observations
+        WHERE
+            constituent = 'Q'
+    ),
+
+    constituent_data AS (
+        SELECT
+            outlet_id,
+            datetime,
+            constituent,
+            "value",
+            count
+        FROM
+            analytics.outlet_observations
+        WHERE
+            constituent NOT IN ('Q', 'QB')
+    )
+
+SELECT
+    c.outlet_id,
+    c.constituent,
+    c.datetime,
+    c."value",
+    c.count,
+    f.flow_value,
+    b.baseflow_value
+FROM
+    constituent_data AS c
+LEFT JOIN
+    flow_data AS f
+    ON c.outlet_id = f.outlet_id 
+    AND c.datetime = f.datetime
+LEFT JOIN
+    baseflow_data AS b
+    ON c.outlet_id = b.outlet_id 
+    AND c.datetime = b.datetime;