ecopipeline 0.4.18__tar.gz → 0.5.1__tar.gz

{ecopipeline-0.4.18/src/ecopipeline.egg-info → ecopipeline-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ecopipeline
-Version: 0.4.18
+Version: 0.5.1
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

{ecopipeline-0.4.18 → ecopipeline-0.5.1}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = ecopipeline
-version = 0.4.18
+version = 0.5.1
 authors = ["Carlos Bello, <bellocarlos@seattleu.edu>, Emil Fahrig <fahrigemil@seattleu.edu>, Casey Mang <cmang@seattleu.edu>, Julian Harris <harrisjulian@seattleu.edu>, Roger Tram <rtram@seattleu.edu>, Nolan Price <nolan@ecotope.com>"]
 description = Contains functions for use in Ecotope Datapipelines
 long_description = file: README.md

{ecopipeline-0.4.18 → ecopipeline-0.5.1}/src/ecopipeline/extract/extract.py

@@ -940,7 +940,7 @@ def _download_noaa_data(stations: dict, weather_directory : str) -> List[str]:
         print("FTP ERROR")
         return
     # Download files for each station from 2010 till present year
-    for year in range(2010, year_end):
+    for year in range(2010, year_end + 1):
         # Set FTP credentials and connect
         wd = f"/pub/data/noaa/isd-lite/{year}/"
         ftp_server.cwd(wd)

{ecopipeline-0.4.18 → ecopipeline-0.5.1}/src/ecopipeline/load/__init__.py

@@ -1,2 +1,2 @@
-from .load import check_table_exists, create_new_table, load_overwrite_database, load_event_table
-__all__ = ["check_table_exists", "create_new_table", "load_overwrite_database", "load_event_table"]
+from .load import check_table_exists, create_new_table, load_overwrite_database, load_event_table, report_data_loss
+__all__ = ["check_table_exists", "create_new_table", "load_overwrite_database", "load_event_table", "report_data_loss"]

{ecopipeline-0.4.18 → ecopipeline-0.5.1}/src/ecopipeline/load/load.py

@@ -368,6 +368,69 @@ def load_event_table(config : ConfigManager, event_df: pd.DataFrame, site_name :
     cursor.close()
     return True
 
+def report_data_loss(config : ConfigManager, site_name : str = None):
+    """
+    Logs data loss event in event database (assumes one exists)
+
+    Parameters
+    ----------  
+    config : ecopipeline.ConfigManager
+        The ConfigManager object that holds configuration data for the pipeline.
+    site_name : str
+        the name of the site to correspond the events with. If left blank will default to minute table name
+
+    Returns
+    ------- 
+    bool: 
+        A boolean value indicating if the data was successfully written to the database. 
+    """
+    # Drop empty columns
+
+    dbname = config.get_db_name()
+    table_name = "site_events"
+    if site_name is None:
+        site_name = config.get_site_name()
+    error_string = "Error proccessing data. Please check logs to resolve."
+
+    print(f"logging DATA_LOSS into {table_name}")
+
+    # create SQL statement
+    insert_str = "INSERT INTO " + table_name + " (start_time_pt, site_name, event_detail, event_type, last_modified_date, last_modified_by) VALUES "
+    insert_str += f"('{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}','{site_name}','{error_string}','DATA_LOSS','{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}','automatic_upload')"
+
+    existing_rows = pd.DataFrame({
+        'id' : []
+    })
+
+    connection, cursor = config.connect_db() 
+
+    # create db table if it does not exist, otherwise add missing columns to existing table
+    if not check_table_exists(cursor, table_name, dbname):
+        print(f"Cannot log data loss. {table_name} does not exist in database {dbname}")
+        return False
+    else:
+        try:
+            # find existing times in database for upsert statement
+            cursor.execute(
+                f"SELECT id FROM {table_name} WHERE end_time_pt IS NULL AND site_name = '{site_name}' AND event_type = 'DATA_LOSS' and event_detail = '{error_string}'")
+            # Fetch the results into a DataFrame
+            existing_rows = pd.DataFrame(cursor.fetchall(), columns=['id'])
+
+        except mysqlerrors.Error as e:
+            print(f"Retrieving data from {table_name} caused exception: {e}")
+    try:
+        
+        if existing_rows.empty:
+            cursor.execute(insert_str)
+        connection.commit()
+        print("Successfully logged data loss.")
+    except Exception as e:
+        # Print the exception message
+        print(f"Caught an exception when uploading to site_events table: {e}")
+    connection.close()
+    cursor.close()
+    return True
+
 def _generate_mysql_update_event_table(row, id):
     statement = f"UPDATE site_events SET "
     statment_elems = []

{ecopipeline-0.4.18 → ecopipeline-0.5.1}/src/ecopipeline/transform/transform.py

@@ -449,28 +449,32 @@ def add_relative_humidity(df : pd.DataFrame, temp_col : str ='airTemp_F', dew_po
     A = 6.11
     B = 7.5
     C = 237.3
+    try:
+        if degree_f:
+            df[f"{temp_col}_C"] = df[temp_col].apply(temp_f_to_c)
+            df[f"{dew_point_col}_C"] = df[dew_point_col].apply(temp_f_to_c)
+            temp_col_c = f"{temp_col}_C"
+            dew_point_col_c = f"{dew_point_col}_C"
+        else:
+            temp_col_c = temp_col
+            dew_point_col_c = dew_point_col
 
-    if degree_f:
-        df[f"{temp_col}_C"] = df[temp_col].apply(temp_f_to_c)
-        df[f"{dew_point_col}_C"] = df[dew_point_col].apply(temp_f_to_c)
-        temp_col_c = f"{temp_col}_C"
-        dew_point_col_c = f"{dew_point_col}_C"
-    else:
-        temp_col_c = temp_col
-        dew_point_col_c = dew_point_col
-
-    # Calculate saturation vapor pressure (e_s) and actual vapor pressure (e)
-    e_s = A * 10 ** ((B * df[temp_col_c]) / (df[temp_col_c] + C))
-    e = A * 10 ** ((B * df[dew_point_col_c]) / (df[dew_point_col_c] + C))
+        # Calculate saturation vapor pressure (e_s) and actual vapor pressure (e)
+        e_s = A * 10 ** ((B * df[temp_col_c]) / (df[temp_col_c] + C))
+        e = A * 10 ** ((B * df[dew_point_col_c]) / (df[dew_point_col_c] + C))
 
-    # Calculate relative humidity
-    df['relative_humidity'] = (e / e_s) * 100.0
+        # Calculate relative humidity
+        df['relative_humidity'] = (e / e_s) * 100.0
 
-    # Handle cases where relative humidity exceeds 100% due to rounding
-    df['relative_humidity'] = np.clip(df['relative_humidity'], 0.0, 100.0)
+        # Handle cases where relative humidity exceeds 100% due to rounding
+        df['relative_humidity'] = np.clip(df['relative_humidity'], 0.0, 100.0)
 
-    if degree_f:
-        df.drop(columns=[temp_col_c, dew_point_col_c])
+        if degree_f:
+            df.drop(columns=[temp_col_c, dew_point_col_c])
+    except:
+       
+        df['relative_humidity'] = None
+        print("Unable to calculate relative humidity data for timeframe")
 
     return df
 

{ecopipeline-0.4.18 → ecopipeline-0.5.1/src/ecopipeline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ecopipeline
-Version: 0.4.18
+Version: 0.5.1
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)