ecopipeline 0.10.2__py3-none-any.whl → 0.11.1__py3-none-any.whl

ecopipeline/__init__.py

@@ -1,4 +1,5 @@
 from .utils.ConfigManager import ConfigManager
+from .utils.NOAADataDownloader import NOAADataDownloader
 from . import extract
 from . import transform
 from . import event_tracking

ecopipeline/event_tracking/event_tracking.py

@@ -4,15 +4,17 @@ import datetime as dt
 from ecopipeline import ConfigManager
 import re
 import mysql.connector.errors as mysqlerrors
+from datetime import timedelta
 
 def central_alarm_df_creator(df: pd.DataFrame, daily_data : pd.DataFrame, config : ConfigManager, system: str = "", 
                              default_cop_high_bound : float = 4.5, default_cop_low_bound : float = 0,
-                             default_boundary_fault_time : int = 15, site_name : str = None) -> pd.DataFrame:
+                             default_boundary_fault_time : int = 15, site_name : str = None, day_table_name_header : str = "day",
+                             power_ratio_period_days : int = 7) -> pd.DataFrame:
     day_list = daily_data.index.to_list()
     print('Checking for alarms...')
     alarm_df = _convert_silent_alarm_dict_to_df({})
     boundary_alarm_df = flag_boundary_alarms(df, config, full_days=day_list, system=system, default_fault_time= default_boundary_fault_time)
-    pwr_alarm_df = power_ratio_alarm(daily_data, config, system=system)
+    pwr_alarm_df = power_ratio_alarm(daily_data, config, day_table_name = config.get_table_name(day_table_name_header), system=system, ratio_period_days=power_ratio_period_days)
     abnormal_COP_df = flag_abnormal_COP(daily_data, config, system = system, default_high_bound=default_cop_high_bound, default_low_bound=default_cop_low_bound)
 
     if len(boundary_alarm_df) > 0:
@@ -251,7 +253,31 @@ def _check_and_add_alarm(df : pd.DataFrame, mask : pd.Series, alarms_dict, day,
         else:
             alarms_dict[day] = [[var_name, alarm_string]]
 
-def power_ratio_alarm(daily_df: pd.DataFrame, config : ConfigManager, system: str = "", verbose : bool = False) -> pd.DataFrame:
+def power_ratio_alarm(daily_df: pd.DataFrame, config : ConfigManager, day_table_name : str, system: str = "", verbose : bool = False, ratio_period_days : int = 7) -> pd.DataFrame:
+    """
+    Function will take a pandas dataframe of daily data and location of alarm information in a csv,
+    and create an dataframe with applicable alarm events
+
+    Parameters
+    ----------
+    daily_df: pd.DataFrame
+        post-transformed dataframe for daily data. It should be noted that this function expects consecutive, in order days. If days
+        are out of order or have gaps, the function may return erroneous alarms.
+    config : ecopipeline.ConfigManager
+        The ConfigManager object that holds configuration data for the pipeline. Among other things, this object will point to a file 
+        called Varriable_Names.csv in the input folder of the pipeline (e.g. "full/path/to/pipeline/input/Variable_Names.csv").
+        The file must have at least two columns which must be titled "variable_name", "alarm_codes" which should contain the
+        name of each variable in the dataframe that requires the alarming and the ratio alarm code in the form "PR_{Power Ratio Name}:{low percentage}-{high percentage}
+    system: str
+        string of system name if processing a particular system in a Variable_Names.csv file with multiple systems. Leave as an empty string if not aplicable.
+    verbose : bool
+        add print statements in power ratio
+
+    Returns
+    ------- 
+    pd.DataFrame:
+        Pandas dataframe with alarm events, empty if no alarms triggered
+    """
     daily_df_copy = daily_df.copy()
     variable_names_path = config.get_var_names_path()
     try:
@@ -274,8 +300,15 @@ def power_ratio_alarm(daily_df: pd.DataFrame, config : ConfigManager, system: st
     ratios_df = ratios_df.loc[:, ["variable_name", "alarm_codes", "pretty_name"]]
     ratios_df = ratios_df[ratios_df['alarm_codes'].str.contains('PR', na=False)]
     ratios_df.dropna(axis=0, thresh=2, inplace=True)
-    ratios_df.set_index(['variable_name'], inplace=True)
+    if ratio_period_days > 1:
+        if verbose:
+            print(f"adding last {ratio_period_days} to daily_df")
+        daily_df_copy = _append_previous_days_to_df(daily_df_copy, config, ratio_period_days, day_table_name)
+    elif ratio_period_days < 1:
+        print("power ratio alarm period, ratio_period_days, must be more than 1")
+        return pd.DataFrame()
 
+    ratios_df.set_index(['variable_name'], inplace=True)
     ratio_dict = {}
     for ratios_var, ratios in ratios_df.iterrows():
         if not ratios_var in daily_df_copy.columns:
@@ -297,26 +330,111 @@ def power_ratio_alarm(daily_df: pd.DataFrame, config : ConfigManager, system: st
                     ratio_dict[pr_id] = [[ratios_var],[float(low_high[0])],[float(low_high[1])],[ratios['pretty_name']]]
     if verbose:
         print("ratio_dict keys:", ratio_dict.keys())
+    # Create blocks of ratio_period_days
+    blocks_df = _create_period_blocks(daily_df_copy, ratio_period_days, verbose)
+
+    if blocks_df.empty:
+        print("No complete blocks available for analysis")
+        return pd.DataFrame()
+    
     alarms = {}
     for key, value_list in ratio_dict.items():
-        daily_df_copy[key] = daily_df_copy[value_list[0]].sum(axis=1)
+        # Calculate total for each block
+        blocks_df[key] = blocks_df[value_list[0]].sum(axis=1)
         for i in range(len(value_list[0])):
             column_name = value_list[0][i]
-            daily_df_copy[f'{column_name}_{key}'] = (daily_df_copy[column_name]/daily_df_copy[key]) * 100
+            # Calculate ratio for each block
+            blocks_df[f'{column_name}_{key}'] = (blocks_df[column_name]/blocks_df[key]) * 100
             if verbose:
-                print(f"Ratios for {column_name}_{key}",daily_df_copy[f'{column_name}_{key}'])
-            _check_and_add_ratio_alarm(daily_df_copy, key, column_name, value_list[3][i], alarms, value_list[2][i], value_list[1][i])
-    return _convert_silent_alarm_dict_to_df(alarms)      
-
-def _check_and_add_ratio_alarm(daily_df: pd.DataFrame, alarm_key : str, column_name : str, pretty_name : str, alarms_dict : dict, high_bound : float, low_bound : float):
-    alarm_daily_df = daily_df.loc[(daily_df[f"{column_name}_{alarm_key}"] < low_bound) | (daily_df[f"{column_name}_{alarm_key}"] > high_bound)]
-    if not alarm_daily_df.empty:
-        for day, values in alarm_daily_df.iterrows():
-            alarm_str = f"Power ratio alarm: {pretty_name} accounted for {round(values[f'{column_name}_{alarm_key}'], 2)}% of {alarm_key} energy use. {round(low_bound, 2)}-{round(high_bound, 2)}% of {alarm_key} energy use expected."
-            if day in alarms_dict:
-                alarms_dict[day].append([column_name, alarm_str])
+                print(f"Block ratios for {column_name}_{key}:", blocks_df[f'{column_name}_{key}'])
+            _check_and_add_ratio_alarm_blocks(blocks_df, key, column_name, value_list[3][i], alarms, value_list[2][i], value_list[1][i], ratio_period_days)
+    return _convert_silent_alarm_dict_to_df(alarms) 
+    # alarms = {}
+    # for key, value_list in ratio_dict.items():
+    #     daily_df_copy[key] = daily_df_copy[value_list[0]].sum(axis=1)
+    #     for i in range(len(value_list[0])):
+    #         column_name = value_list[0][i]
+    #         daily_df_copy[f'{column_name}_{key}'] = (daily_df_copy[column_name]/daily_df_copy[key]) * 100
+    #         if verbose:
+    #             print(f"Ratios for {column_name}_{key}",daily_df_copy[f'{column_name}_{key}'])
+    #         _check_and_add_ratio_alarm(daily_df_copy, key, column_name, value_list[3][i], alarms, value_list[2][i], value_list[1][i])
+    # return _convert_silent_alarm_dict_to_df(alarms)      
+
+# def _check_and_add_ratio_alarm(daily_df: pd.DataFrame, alarm_key : str, column_name : str, pretty_name : str, alarms_dict : dict, high_bound : float, low_bound : float):
+#     alarm_daily_df = daily_df.loc[(daily_df[f"{column_name}_{alarm_key}"] < low_bound) | (daily_df[f"{column_name}_{alarm_key}"] > high_bound)]
+#     if not alarm_daily_df.empty:
+#         for day, values in alarm_daily_df.iterrows():
+#             alarm_str = f"Power ratio alarm: {pretty_name} accounted for {round(values[f'{column_name}_{alarm_key}'], 2)}% of {alarm_key} energy use. {round(low_bound, 2)}-{round(high_bound, 2)}% of {alarm_key} energy use expected."
+#             if day in alarms_dict:
+#                 alarms_dict[day].append([column_name, alarm_str])
+#             else:
+#                 alarms_dict[day] = [[column_name, alarm_str]]
+def _check_and_add_ratio_alarm_blocks(blocks_df: pd.DataFrame, alarm_key: str, column_name: str, pretty_name: str, alarms_dict: dict, high_bound: float, low_bound: float, ratio_period_days: int):
+    """
+    Check for alarms in block-based ratios and add to alarms dictionary.
+    """
+    alarm_blocks_df = blocks_df.loc[(blocks_df[f"{column_name}_{alarm_key}"] < low_bound) | (blocks_df[f"{column_name}_{alarm_key}"] > high_bound)]
+    if not alarm_blocks_df.empty:
+        for block_end_date, values in alarm_blocks_df.iterrows():
+            alarm_str = f"Power ratio alarm ({ratio_period_days}-day block ending {block_end_date.strftime('%Y-%m-%d')}): {pretty_name} accounted for {round(values[f'{column_name}_{alarm_key}'], 2)}% of {alarm_key} energy use. {round(low_bound, 2)}-{round(high_bound, 2)}% of {alarm_key} energy use expected."
+            if block_end_date in alarms_dict:
+                alarms_dict[block_end_date].append([column_name, alarm_str])
             else:
-                alarms_dict[day] = [[column_name, alarm_str]]
+                alarms_dict[block_end_date] = [[column_name, alarm_str]]
+
+def _create_period_blocks(daily_df: pd.DataFrame, ratio_period_days: int, verbose: bool = False) -> pd.DataFrame:
+    """
+    Create blocks of ratio_period_days by summing values within each block.
+    Each block will be represented by its end date.
+    """
+    if len(daily_df) < ratio_period_days:
+        if verbose:
+            print(f"Not enough data for {ratio_period_days}-day blocks. Need at least {ratio_period_days} days, have {len(daily_df)}")
+        return pd.DataFrame()
+    
+    blocks = []
+    block_dates = []
+    
+    # Create blocks by summing consecutive groups of ratio_period_days
+    for i in range(ratio_period_days - 1, len(daily_df)):
+        start_idx = i - ratio_period_days + 1
+        end_idx = i + 1
+        
+        block_data = daily_df.iloc[start_idx:end_idx].sum()
+        blocks.append(block_data)
+        # Use the end date of the block as the identifier
+        block_dates.append(daily_df.index[i])
+    
+    if not blocks:
+        return pd.DataFrame()
+    
+    blocks_df = pd.DataFrame(blocks, index=block_dates)
+    
+    if verbose:
+        print(f"Created {len(blocks_df)} blocks of {ratio_period_days} days each")
+        print(f"Block date range: {blocks_df.index.min()} to {blocks_df.index.max()}")
+    
+    return blocks_df
+
+def _append_previous_days_to_df(daily_df: pd.DataFrame, config : ConfigManager, ratio_period_days : int, day_table_name : str, primary_key : str = "time_pt") -> pd.DataFrame:
+    db_connection, cursor = config.connect_db()
+    period_start = daily_df.index.min() - timedelta(ratio_period_days)
+    try:
+        # find existing times in database for upsert statement
+        cursor.execute(
+            f"SELECT * FROM {day_table_name} WHERE {primary_key} < '{daily_df.index.min()}' AND {primary_key} >= '{period_start}'")
+        result = cursor.fetchall()
+        column_names = [desc[0] for desc in cursor.description]
+        old_days_df = pd.DataFrame(result, columns=column_names)
+        old_days_df = old_days_df.set_index(primary_key)
+        daily_df = pd.concat([daily_df, old_days_df])
+        daily_df = daily_df.sort_index(ascending=True)
+    except mysqlerrors.Error:
+        print(f"Table {day_table_name} has no data.")
+
+    db_connection.close()
+    cursor.close()
+    return daily_df
 
 # def flag_dhw_outage(df: pd.DataFrame, daily_df : pd.DataFrame, dhw_outlet_column : str, supply_temp : int = 110, consecutive_minutes : int = 15) -> pd.DataFrame:
 #     """

ecopipeline/utils/NOAADataDownloader.py

@@ -0,0 +1,498 @@
+import requests
+import pandas as pd
+# from datetime import datetime, timedelta
+# import os
+# import gzip
+# import urllib.request
+from io import StringIO
+
+class NOAADataDownloader:
+    def __init__(self, station_code, api_token=None):
+        """
+        Initialize downloader for a specific weather station
+        
+        Args:
+            station_code (str): Airport code (e.g., 'KLAX', 'LAX', 'JFK', 'ORD')
+            api_token (str, optional): NOAA API token for daily data access
+        """
+        self.station_code = station_code.upper().strip()
+        self.api_token = api_token
+        self.base_url = "https://www.ncdc.noaa.gov/cdo-web/api/v2/"
+        
+        # Clean airport code - add K if not present for US airports
+        if len(self.station_code) == 3 and not self.station_code.startswith('K'):
+            self.station_code = 'K' + self.station_code
+        
+        # Find station information
+        self.station_info = self._find_station_info()
+        
+        if not self.station_info:
+            raise ValueError(f"Could not find weather station for {station_code}")
+        
+        print(f"Initialized downloader for: {self.station_info['name']}")
+        if self.station_info.get('usaf') and self.station_info.get('wban'):
+            print(f"ISD Station ID: {self.station_info['usaf']}-{self.station_info['wban']}")
+        if self.station_info.get('ghcn_id'):
+            print(f"GHCN-D Station ID: {self.station_info['ghcn_id']}")
+    
+    def _find_station_info(self):
+        """Find station information for the given airport code"""
+        
+        # First try common stations mapping
+        common_stations = self._get_common_stations()
+        if self.station_code in common_stations:
+            return common_stations[self.station_code]
+        
+        # Try searching ISD station history
+        isd_station = self._search_isd_stations()
+        if isd_station:
+            return isd_station
+        
+        # Try API search if token available
+        if self.api_token:
+            api_station = self._search_api_stations()
+            if api_station:
+                return api_station
+        
+        return None
+    
+    def _get_common_stations(self):
+        """Return mapping of common airport codes to station information"""
+        return {
+            'KLAX': {
+                'name': 'LOS ANGELES INTERNATIONAL AIRPORT',
+                'usaf': '722950',
+                'wban': '23174',
+                'ghcn_id': 'GHCND:USW00023174',
+                'latitude': 33.938,
+                'longitude': -118.389,
+                'elevation': 32.0
+            },
+            'KJFK': {
+                'name': 'JOHN F KENNEDY INTERNATIONAL AIRPORT',
+                'usaf': '744860',
+                'wban': '94789',
+                'ghcn_id': 'GHCND:USW00094789',
+                'latitude': 40.640,
+                'longitude': -73.779,
+                'elevation': 3.4
+            },
+            'KORD': {
+                'name': 'CHICAGO OHARE INTERNATIONAL AIRPORT',
+                'usaf': '725300',
+                'wban': '94846',
+                'ghcn_id': 'GHCND:USW00094846',
+                'latitude': 41.995,
+                'longitude': -87.934,
+                'elevation': 201.5
+            },
+            'KDEN': {
+                'name': 'DENVER INTERNATIONAL AIRPORT',
+                'usaf': '725650',
+                'wban': '03017',
+                'ghcn_id': 'GHCND:USW00003017',
+                'latitude': 39.833,
+                'longitude': -104.65,
+                'elevation': 1640.0
+            },
+            'KATL': {
+                'name': 'HARTSFIELD JACKSON ATLANTA INTERNATIONAL AIRPORT',
+                'usaf': '722190',
+                'wban': '13874',
+                'ghcn_id': 'GHCND:USW00013874',
+                'latitude': 33.640,
+                'longitude': -84.427,
+                'elevation': 308.5
+            },
+            'KMIA': {
+                'name': 'MIAMI INTERNATIONAL AIRPORT',
+                'usaf': '722020',
+                'wban': '12839',
+                'ghcn_id': 'GHCND:USW00012839',
+                'latitude': 25.793,
+                'longitude': -80.290,
+                'elevation': 11.0
+            },
+            'KSEA': {
+                'name': 'SEATTLE TACOMA INTERNATIONAL AIRPORT',
+                'usaf': '727930',
+                'wban': '24233',
+                'ghcn_id': 'GHCND:USW00024233',
+                'latitude': 47.449,
+                'longitude': -122.309,
+                'elevation': 131.1
+            },
+            'KBOS': {
+                'name': 'BOSTON LOGAN INTERNATIONAL AIRPORT',
+                'usaf': '725090',
+                'wban': '14739',
+                'ghcn_id': 'GHCND:USW00014739',
+                'latitude': 42.361,
+                'longitude': -71.020,
+                'elevation': 6.1
+            },
+            'KPHX': {
+                'name': 'PHOENIX SKY HARBOR INTERNATIONAL AIRPORT',
+                'usaf': '722780',
+                'wban': '23183',
+                'ghcn_id': 'GHCND:USW00023183',
+                'latitude': 33.434,
+                'longitude': -112.008,
+                'elevation': 337.1
+            },
+            'KLAS': {
+                'name': 'LAS VEGAS MCCARRAN INTERNATIONAL AIRPORT',
+                'usaf': '723860',
+                'wban': '23169',
+                'ghcn_id': 'GHCND:USW00023169',
+                'latitude': 36.080,
+                'longitude': -115.152,
+                'elevation': 664.1
+            }
+        }
+    
+    def _search_isd_stations(self):
+        """Search ISD station history for the airport"""
+        try:
+            url = "https://www.ncei.noaa.gov/data/global-hourly/doc/isd-history.csv"
+            response = requests.get(url, timeout=10)
+            response.raise_for_status()
+            
+            df = pd.read_csv(StringIO(response.text))
+            
+            # Search for airport code in station name
+            search_terms = [
+                self.station_code.replace('K', ''),  # LAX from KLAX
+                self.station_code,                   # KLAX
+                self.station_code + ' ',             # Exact match with space
+            ]
+            
+            for term in search_terms:
+                mask = df['STATION NAME'].str.contains(term, case=False, na=False)
+                matches = df[mask]
+                
+                if not matches.empty:
+                    # Take the first match with recent data
+                    best_match = matches.iloc[0]
+                    
+                    return {
+                        'name': best_match['STATION NAME'],
+                        'usaf': str(best_match['USAF']).zfill(6),
+                        'wban': str(best_match['WBAN']).zfill(5),
+                        'country': best_match['CTRY'],
+                        'state': best_match.get('STATE', ''),
+                        'latitude': best_match['LAT'],
+                        'longitude': best_match['LON'],
+                        'elevation': best_match['ELEV(M)'],
+                        'begin_date': str(best_match['BEGIN']),
+                        'end_date': str(best_match['END'])
+                    }
+            
+            return None
+            
+        except Exception as e:
+            print(f"ISD search failed: {e}")
+            return None
+    
+    def _search_api_stations(self):
+        """Search for stations using NOAA API"""
+        if not self.api_token:
+            return None
+            
+        try:
+            url = f"{self.base_url}stations"
+            params = {'limit': 100, 'format': 'json'}
+            headers = {"token": self.api_token}
+            
+            response = requests.get(url, params=params, headers=headers, timeout=10)
+            response.raise_for_status()
+            
+            data = response.json()
+            if 'results' in data:
+                search_terms = [self.station_code.replace('K', ''), self.station_code]
+                
+                for station in data['results']:
+                    name = station.get('name', '').upper()
+                    for term in search_terms:
+                        if term in name:
+                            return {
+                                'name': station.get('name'),
+                                'ghcn_id': station.get('id'),
+                                'latitude': station.get('latitude'),
+                                'longitude': station.get('longitude'),
+                                'elevation': station.get('elevation'),
+                                'mindate': station.get('mindate'),
+                                'maxdate': station.get('maxdate')
+                            }
+            
+            return None
+            
+        except Exception as e:
+            print(f"API search failed: {e}")
+            return None
+    
+    def get_station_info(self):
+        """Return station information"""
+        return self.station_info.copy()
+    
+    # def download_hourly_data(self, start_date, end_date, data_types=None):
+    #     """
+    #     Download hourly weather data using NOAA's data access API
+        
+    #     Args:
+    #         start_date (str or pd.Timestamp): Start date in YYYY-MM-DD format or pandas Timestamp or datetime
+    #         end_date (str or pd.Timestamp): End date in YYYY-MM-DD format or pandas Timestamp or datetime
+    #         data_types (list, optional): List of data types to download
+            
+    #     Returns:
+    #         pandas.DataFrame: Hourly weather data
+    #     """
+    #     if not (self.station_info.get('usaf') and self.station_info.get('wban')):
+    #         raise ValueError("Station does not have ISD identifiers for hourly data")
+        
+    #     # Convert pd.Timestamp to string format if needed
+    #     if isinstance(start_date, pd.Timestamp):
+    #         start_date = start_date.strftime('%Y-%m-%d')
+    #     elif hasattr(start_date, 'strftime'):  # datetime.datetime or similar
+    #         start_date = start_date.strftime('%Y-%m-%d')
+            
+    #     if isinstance(end_date, pd.Timestamp):
+    #         end_date = end_date.strftime('%Y-%m-%d')
+    #     elif hasattr(end_date, 'strftime'):  # datetime.datetime or similar
+    #         end_date = end_date.strftime('%Y-%m-%d')
+        
+    #     # Create station ID in format expected by the API
+    #     station_id = f"{self.station_info['usaf']}{self.station_info['wban']}"
+    #     # station_id = "USW00023174"#"USC00457180"
+    #     # print("station_id is ",station_id)
+    #     # Default data types for hourly weather data
+    #     if not data_types:
+    #         data_types = [
+    #             'TMP',      # Temperature
+    #             'DEW',      # Dew point
+    #             'SLP',      # Sea level pressure  
+    #             'WND',      # Wind direction and speed
+    #             'VIS',      # Visibility
+    #             'AA1'       # Precipitation (if available)
+    #         ]
+        
+    #     # NOAA's data access API endpoint
+    #     base_url = "https://www.ncei.noaa.gov/access/services/data/v1"
+        
+    #     params = {
+    #         'dataset': 'global-hourly',
+    #         # 'dataTypes': 'TMP',#','.join(data_types),
+    #         'stations': station_id,
+    #         'startDate': start_date,
+    #         'endDate': end_date,
+    #         'format': 'json',
+    #         'includeAttributes': 'true',
+    #         'includeStationName': 'true',
+    #         'includeStationLocation': 'true'
+    #     }
+        
+    #     try:
+    #         print(f"Downloading hourly data from {start_date} to {end_date}...")
+    #         print(f"Station: {station_id} ({self.station_info.get('name', 'Unknown')})")
+    #         full_url = requests.Request('GET', base_url, params=params).prepare().url
+    #         print(f"API Request URL:")
+    #         print(f"{full_url}")
+    #         print()
+    #         # https://www.ncei.noaa.gov/access/services/data/v1?dataset=global-hourly
+    #         # &dataTypes=TMP%2CDEW%2CSLP%2CWND%2CVIS%2CAA1&stations=USW00023174&startDate=2025-08-26&endDate=2025-09-18&format=json
+    #         # &includeAttributes=true&includeStationName=true&includeStationLocation=true
+
+    #         # https://www.ncei.noaa.gov/access/services/data/v1?dataset=global-summary-of-the-year
+    #         # &dataTypes=DP01,DP05,DP10,DSND,DSNW,DT00,DT32,DX32,DX70,DX90,SNOW,PRCP&stations=ASN00084027&startDate=1952-01-01&endDate=1970-12-31&includeAttributes=true&format=pdf
+            
+    #         response = requests.get(base_url, params=params, timeout=60)
+    #         response.raise_for_status()
+            
+    #         # Parse JSON response
+    #         data = response.json()
+            
+    #         if not data:
+    #             print("No data returned from API")
+    #             return pd.DataFrame()
+            
+    #         # Convert to DataFrame
+    #         df = pd.DataFrame(data)
+            
+    #         if df.empty:
+    #             print("No hourly data found for the specified parameters")
+    #             return pd.DataFrame()
+            
+    #         # Process the data
+    #         df = self._process_hourly_data(df)
+            
+    #         print(f"Successfully downloaded {len(df)} hourly records")
+    #         return df
+            
+    #     except requests.exceptions.RequestException as e:
+    #         print(f"API request failed: {e}")
+    #         if hasattr(e, 'response') and e.response is not None:
+    #             print(f"Response status: {e.response.status_code}")
+    #             print(f"Response text: {e.response.text[:500]}...")
+    #         return pd.DataFrame()
+    #     except Exception as e:
+    #         print(f"Failed to download hourly data: {e}")
+    #         return pd.DataFrame()
+    
+    # def _process_hourly_data(self, df):
+    #     """Process and clean hourly data from NOAA API"""
+    #     try:
+    #         # Convert DATE to datetime
+    #         if 'DATE' in df.columns:
+    #             df['datetime'] = pd.to_datetime(df['DATE'], errors='coerce')
+    #             df = df.dropna(subset=['datetime'])
+    #             df = df.sort_values('datetime')
+            
+    #         # Process temperature data (convert tenths of degrees C to C)
+    #         if 'TMP' in df.columns:
+    #             df['temperature_c'] = pd.to_numeric(df['TMP'], errors='coerce') / 10
+    #             df['temperature_f'] = df['temperature_c'] * 9/5 + 32
+            
+    #         # Process dew point data
+    #         if 'DEW' in df.columns:
+    #             df['dewpoint_c'] = pd.to_numeric(df['DEW'], errors='coerce') / 10
+    #             df['dewpoint_f'] = df['dewpoint_c'] * 9/5 + 32
+            
+    #         # Process sea level pressure (convert tenths of hPa to hPa)
+    #         if 'SLP' in df.columns:
+    #             df['pressure_hpa'] = pd.to_numeric(df['SLP'], errors='coerce') / 10
+            
+    #         # Process wind data - format is typically "999,9" (direction,speed)
+    #         if 'WND' in df.columns:
+    #             wind_data = df['WND'].astype(str)
+                
+    #             # Extract wind direction and speed
+    #             wind_direction = []
+    #             wind_speed = []
+                
+    #             for wind_str in wind_data:
+    #                 try:
+    #                     if ',' in wind_str:
+    #                         dir_str, speed_str = wind_str.split(',')[:2]
+                            
+    #                         # Wind direction (degrees)
+    #                         direction = int(dir_str) if dir_str != '999' else None
+    #                         wind_direction.append(direction)
+                            
+    #                         # Wind speed (tenths of m/s to m/s)
+    #                         speed = float(speed_str) / 10 if speed_str != '9999' else None
+    #                         wind_speed.append(speed)
+    #                     else:
+    #                         wind_direction.append(None)
+    #                         wind_speed.append(None)
+    #                 except (ValueError, IndexError):
+    #                     wind_direction.append(None)
+    #                     wind_speed.append(None)
+                
+    #             df['wind_direction'] = wind_direction
+    #             df['wind_speed_mps'] = wind_speed
+    #             df['wind_speed_kmh'] = pd.Series(wind_speed) * 3.6
+    #             df['wind_speed_mph'] = pd.Series(wind_speed) * 2.237
+            
+    #         # Process visibility (meters)
+    #         if 'VIS' in df.columns:
+    #             df['visibility_m'] = pd.to_numeric(df['VIS'], errors='coerce')
+    #             df['visibility_km'] = df['visibility_m'] / 1000
+    #             df['visibility_mi'] = df['visibility_m'] / 1609.34
+            
+    #         # Add station information columns
+    #         if 'STATION' in df.columns:
+    #             df['station_id'] = df['STATION']
+            
+    #         if 'NAME' in df.columns:
+    #             df['station_name'] = df['NAME']
+            
+    #         if 'LATITUDE' in df.columns:
+    #             df['latitude'] = pd.to_numeric(df['LATITUDE'], errors='coerce')
+            
+    #         if 'LONGITUDE' in df.columns:
+    #             df['longitude'] = pd.to_numeric(df['LONGITUDE'], errors='coerce')
+            
+    #         if 'ELEVATION' in df.columns:
+    #             df['elevation_m'] = pd.to_numeric(df['ELEVATION'], errors='coerce')
+            
+    #         return df
+            
+    #     except Exception as e:
+    #         print(f"Error processing hourly data: {e}")
+    #         return df
+        
+
+    def download_daily_TAVG_data(self, start_date, end_date, convert_to_fahrenheit = True):
+        """
+        Download daily Average Temperature data using NOAA API
+        
+        Args:
+            start_date (str or pd.Timestamp): Start date in YYYY-MM-DD format or pandas Timestamp or datetime
+            end_date (str or pd.Timestamp): End date in YYYY-MM-DD format or pandas Timestamp or datetime
+            convert_to_fahrenheit (bool): converts temperature values to fahrenhiet. Otherwise will be celcius*10 
+            
+        Returns:
+            pandas.DataFrame: Daily weather data
+        """
+        if not self.api_token:
+            raise ValueError("API token required for daily data. Get one from https://www.ncdc.noaa.gov/cdo-web/token")
+        
+        if not self.station_info.get('ghcn_id'):
+            raise ValueError("Station does not have GHCN-D identifier for daily data")
+        
+        # Convert pd.Timestamp to string format if needed
+        if isinstance(start_date, pd.Timestamp):
+            start_date = start_date.strftime('%Y-%m-%d')
+        elif hasattr(start_date, 'strftime'):  # datetime.datetime or similar
+            start_date = start_date.strftime('%Y-%m-%d')
+            
+        if isinstance(end_date, pd.Timestamp):
+            end_date = end_date.strftime('%Y-%m-%d')
+        elif hasattr(end_date, 'strftime'):  # datetime.datetime or similar
+            end_date = end_date.strftime('%Y-%m-%d')
+        
+        # if not datatypes:
+        #     datatypes = ['TAVG']
+        
+        url = f"{self.base_url}data"
+        params = {
+            'datasetid': 'GHCND',
+            'stationid': self.station_info['ghcn_id'],
+            'startdate': start_date,
+            'enddate': end_date,
+            'datatypeid': 'TAVG',
+            'limit': 1000,
+            'format': 'json'
+        }
+        
+        try:
+            print(f"Downloading daily data from {start_date} to {end_date}...")
+            
+            headers = {"token": self.api_token}
+            response = requests.get(url, params=params, headers=headers)
+            response.raise_for_status()
+            
+            data = response.json()
+            if 'results' in data:
+                df = pd.DataFrame(data['results'])
+                
+                if not df.empty:
+                    df['date'] = pd.to_datetime(df['date'])
+                    df = df.sort_values('date')
+                    # Convert value from tenths of Celsius to Fahrenheit
+                    df['value'] = (df['value'] / 10) * 9/5 + 32
+                    df = df.set_index('date')
+                    df = df[['value']].rename(columns={'value': 'OAT_NOAA'})
+                    
+                    print(f"Successfully downloaded {len(df)} daily records")
+                    return df
+                else:
+                    print("No daily data found for the specified parameters")
+                    return pd.DataFrame()
+            else:
+                print("No daily data found")
+                return pd.DataFrame()
+                
+        except requests.exceptions.RequestException as e:
+            print(f"Daily data download failed: {e}")
+            return pd.DataFrame()

ecopipeline/utils/__init__.py

@@ -1 +1,2 @@
-from .ConfigManager import *
+from .ConfigManager import *
+from .NOAADataDownloader import *

{ecopipeline-0.10.2.dist-info → ecopipeline-0.11.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecopipeline
-Version: 0.10.2
+Version: 0.11.1
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

{ecopipeline-0.10.2.dist-info → ecopipeline-0.11.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
-ecopipeline/__init__.py,sha256=d48mO5La6OrQDkRe_qqoY6lUx7x-e8krOH388jmWjwU,218
+ecopipeline/__init__.py,sha256=pjC00JWsjVAhS0jUKHD-wyi4UIpTsWbIg9JaxLS1mlc,275
 ecopipeline/event_tracking/__init__.py,sha256=SV2kkvJgptjeyLQlqHWcDRpQO6-JC433_dRZ3H9-ZNU,131
-ecopipeline/event_tracking/event_tracking.py,sha256=LhL2qffLbEqpRE2PmRvZjj67oUxTKC2MKSqawI4XqEA,23264
+ecopipeline/event_tracking/event_tracking.py,sha256=HffWAIAkNJ8INdG3_86RnDgw2bpHwv9hhkZ5oiiugZY,29653
 ecopipeline/extract/__init__.py,sha256=gQ3sak6NJ63Gpo-hZXrtZfeKOTHLRyAVXfTgxxRpqPo,675
 ecopipeline/extract/extract.py,sha256=y32feIIzgABwrwfduNQM1hICmkVOU4PYu6-M07zCLpU,51422
 ecopipeline/load/__init__.py,sha256=NLa_efQJZ8aP-J0Y5xx9DP7mtfRH9jY6Jz1ZMZN_BAA,292
@@ -10,10 +10,11 @@ ecopipeline/transform/bayview.py,sha256=TP24dnTsUD95X-f6732egPZKjepFLJgDm9ImGr-f
 ecopipeline/transform/lbnl.py,sha256=EQ54G4rJXaZ7pwVusKcdK2KBehSdCsNo2ybphtMGs7o,33400
 ecopipeline/transform/transform.py,sha256=wL4B00XBwLWVlf7goOLSHKgLFmIsXprQNepGLLO_wTk,50028
 ecopipeline/utils/ConfigManager.py,sha256=-g1wtExdvhYO5Y6Q3cRbywa__DxRMFruLrB4YanwaPY,12168
-ecopipeline/utils/__init__.py,sha256=ccWUR0m7gD9DfcgsxBCLOfi4lho6RdYuB2Ugy_g6ZdQ,28
+ecopipeline/utils/NOAADataDownloader.py,sha256=iC2nl_O4PS1KFrchcPXRZxshwZwUMSqXy6BQBUwnOUU,20927
+ecopipeline/utils/__init__.py,sha256=7dT3tP6SMK4uBW6NBmQ8i6LaNTTuV6fpAZToBBlJ904,62
 ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
-ecopipeline-0.10.2.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ecopipeline-0.10.2.dist-info/METADATA,sha256=hKZenaQpTLsiAohHLP92cdWypwupi9yJgRTwZjG4GHM,2330
-ecopipeline-0.10.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ecopipeline-0.10.2.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
-ecopipeline-0.10.2.dist-info/RECORD,,
+ecopipeline-0.11.1.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ecopipeline-0.11.1.dist-info/METADATA,sha256=_-HP7vfIrz6JBltdDkX4obF-AUJGrbxZfnFtrUBQ49k,2330
+ecopipeline-0.11.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ecopipeline-0.11.1.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
+ecopipeline-0.11.1.dist-info/RECORD,,