ecopipeline 0.11.9__tar.gz → 1.0.1__tar.gz

ecopipeline-1.0.1/MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+recursive-include src/ecopipeline/utils/pkls *.pkl

{ecopipeline-0.11.9/src/ecopipeline.egg-info → ecopipeline-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecopipeline
-Version: 0.11.9
+Version: 1.0.1
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

{ecopipeline-0.11.9 → ecopipeline-1.0.1}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = ecopipeline
-version = 0.11.9
+version = 1.0.1
 authors = ["Carlos Bello, <bellocarlos@seattleu.edu>, Emil Fahrig <fahrigemil@seattleu.edu>, Casey Mang <cmang@seattleu.edu>, Julian Harris <harrisjulian@seattleu.edu>, Roger Tram <rtram@seattleu.edu>, Nolan Price <nolan@ecotope.com>"]
 description = Contains functions for use in Ecotope Datapipelines
 long_description = file: README.md

{ecopipeline-0.11.9 → ecopipeline-1.0.1}/src/ecopipeline/extract/extract.py

@@ -776,7 +776,7 @@ def pull_egauge_data(config: ConfigManager, eGauge_ids: list, eGauge_usr : str,
     os.chdir(original_directory)
 
 def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: datetime = None, create_csv : bool = True, query_hours : float = 1,
-                 sensor_keys : list = [], seperate_keys : bool = False):
+                 sensor_keys : list = [], seperate_keys : bool = False, device_id_overwrite : str = None, csv_prefix : str = ""):
     """
     Function connects to the things board manager api to pull data and returns a dataframe.
 
@@ -796,6 +796,11 @@ def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: dat
         create csv files as you process such that API need not be relied upon for reprocessing
     query_hours : float
         number of hours to query at a time from ThingsBoard API
+
+    device_id_overwrite : str
+        Overwrites device ID for API pull
+    csv_prefix : str
+        prefix to add to the csv title
     
     Returns
     ------- 
@@ -804,16 +809,17 @@ def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: dat
         Will return with index in UTC so needs to be converted after to appropriate timezone
     """
     df = pd.DataFrame()
+    api_device_id = device_id_overwrite if not device_id_overwrite is None else config.api_device_id
     if len(sensor_keys) <= 0:
         token = config.get_thingsboard_token()
-        key_list = _get_tb_keys(config, token)
+        key_list = _get_tb_keys(token, api_device_id)
         if len(key_list) <= 0:
-            raise Exception(f"No sensors available at ThingsBoard site with id {config.api_device_id}")
-        return tb_api_to_df(config, startTime, endTime, create_csv, query_hours, key_list, seperate_keys)
+            raise Exception(f"No sensors available at ThingsBoard site with id {api_device_id}")
+        return tb_api_to_df(config, startTime, endTime, create_csv, query_hours, key_list, seperate_keys, device_id_overwrite, csv_prefix)
     if seperate_keys:
         df_list = []
         for sensor_key in sensor_keys:
-            df_list.append(tb_api_to_df(config, startTime, endTime, False, query_hours, [sensor_key], False))
+            df_list.append(tb_api_to_df(config, startTime, endTime, False, query_hours, [sensor_key], False, device_id_overwrite, csv_prefix))
         df = pd.concat(df_list)
     else:    
     # not seperate_keys:
@@ -826,13 +832,13 @@ def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: dat
         if endTime - timedelta(hours=query_hours) > startTime:
             time_diff = endTime - startTime
             midpointTime = startTime + time_diff / 2
-            df_1 = tb_api_to_df(config, startTime, midpointTime, query_hours=query_hours, sensor_keys=sensor_keys, create_csv=False)#True if startTime >= datetime(2025,7,13,9) and startTime <= datetime(2025,7,13,10) else csv_pass_down)
-            df_2 = tb_api_to_df(config, midpointTime, endTime, query_hours=query_hours, sensor_keys=sensor_keys,create_csv=False)#True if endTime >= datetime(2025,7,13,9) and endTime <= datetime(2025,7,13,10) else csv_pass_down)
+            df_1 = tb_api_to_df(config, startTime, midpointTime, query_hours=query_hours, sensor_keys=sensor_keys, create_csv=False, device_id_overwrite = device_id_overwrite)#True if startTime >= datetime(2025,7,13,9) and startTime <= datetime(2025,7,13,10) else csv_pass_down)
+            df_2 = tb_api_to_df(config, midpointTime, endTime, query_hours=query_hours, sensor_keys=sensor_keys,create_csv=False, device_id_overwrite = device_id_overwrite)#True if endTime >= datetime(2025,7,13,9) and endTime <= datetime(2025,7,13,10) else csv_pass_down)
             df = pd.concat([df_1, df_2])
             df = df.sort_index()
             df = df.groupby(df.index).mean()
         else:
-            url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{config.api_device_id}/values/timeseries'
+            url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{api_device_id}/values/timeseries'
             token = config.get_thingsboard_token()
             key_string = ','.join(sensor_keys)
             params = {
@@ -844,7 +850,6 @@ def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: dat
                 'interval' : '0',
                 'agg' : 'NONE'
             }
-
             # Headers
             headers = {
                 'accept': 'application/json',
@@ -855,14 +860,6 @@ def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: dat
                 response = requests.get(url, headers=headers, params=params)
                 if response.status_code == 200:
                     response_json = response.json()
-                    # if create_csv:
-                    #     json_filename = f"{startTime.strftime('%Y%m%d%H%M%S')}.json"
-                    #     print(f"filename: {json_filename}, url: {url}, params: {params}")
-                    #     original_directory = os.getcwd()
-                    #     os.chdir(config.data_directory)
-                    #     with open(json_filename, 'w') as f:
-                    #         json.dump(response_json, f, indent=4)  # indent=4 makes it human-readable
-                    #     os.chdir(original_directory)
                         
                     data = {}
                     for key, records in response_json.items():
@@ -886,7 +883,7 @@ def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: dat
                 df = pd.DataFrame()
     # save to file
     if create_csv:
-        filename = f"{startTime.strftime('%Y%m%d%H%M%S')}.csv"
+        filename = f"{csv_prefix}{startTime.strftime('%Y%m%d%H%M%S')}.csv"
         original_directory = os.getcwd()
         os.chdir(config.data_directory)
         df.to_csv(filename, index_label='time_pt')
@@ -900,8 +897,8 @@ def _get_float_value(value):
     except (ValueError, TypeError):
         return None
     
-def _get_tb_keys(config: ConfigManager, token : str) -> List[str]:
-    url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{config.api_device_id}/keys/timeseries'
+def _get_tb_keys(token : str, api_device_id : str) -> List[str]:
+    url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{api_device_id}/keys/timeseries'
 
     # Headers
     headers = {

{ecopipeline-0.11.9 → ecopipeline-1.0.1}/src/ecopipeline/transform/transform.py

@@ -157,20 +157,29 @@ def _rm_cols(col, bounds_df):  # Helper function for remove_outliers
     """
     Function will take in a pandas series and bounds information
     stored in a dataframe, then check each element of that column and set it to nan
-    if it is outside the given bounds. 
+    if it is outside the given bounds.
 
-    Args: 
-        col: pd.Series 
+    Args:
+        col: pd.Series
             Pandas dataframe column from data being processed
         bounds_df: pd.DataFrame
             Pandas dataframe indexed by the names of the columns from the dataframe that col came from. There should be at least
             two columns in this dataframe, lower_bound and upper_bound, for use in removing outliers
-    Returns: 
-        None 
+    Returns:
+        None
     """
     if (col.name in bounds_df.index):
-        c_lower = float(bounds_df.loc[col.name]["lower_bound"])
-        c_upper = float(bounds_df.loc[col.name]["upper_bound"])
+        c_lower = bounds_df.loc[col.name]["lower_bound"]
+        c_upper = bounds_df.loc[col.name]["upper_bound"]
+
+        # Skip if both bounds are NaN
+        if pd.isna(c_lower) and pd.isna(c_upper):
+            return
+
+        # Convert bounds to float, handling NaN values
+        c_lower = float(c_lower) if not pd.isna(c_lower) else -np.inf
+        c_upper = float(c_upper) if not pd.isna(c_upper) else np.inf
+
         col.mask((col > c_upper) | (col < c_lower), other=np.NaN, inplace=True)
 
 # TODO: remove_outliers STRETCH GOAL: Functionality for alarms being raised based on bounds needs to happen here.
@@ -784,7 +793,7 @@ def convert_on_off_col_to_bool(df: pd.DataFrame, column_names: list) -> pd.DataF
     pd.DataFrame: Dataframe with specified columns converted from Celsius to Farenhiet.
     """
     
-    mapping = {'ON': True, 'OFF': False}
+    mapping = {'ON': True, 'OFF': False, 'On': True, 'Off': False}
     
     for column_name in column_names: 
         df[column_name] = df[column_name].map(mapping).where(df[column_name].notna(), df[column_name])

{ecopipeline-0.11.9 → ecopipeline-1.0.1/src/ecopipeline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecopipeline
-Version: 0.11.9
+Version: 1.0.1
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

{ecopipeline-0.11.9 → ecopipeline-1.0.1}/src/ecopipeline.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 LICENSE
+MANIFEST.in
 README.md
 pyproject.toml
 setup.cfg
@@ -23,4 +24,8 @@ src/ecopipeline/utils/ConfigManager.py
 src/ecopipeline/utils/NOAADataDownloader.py
 src/ecopipeline/utils/__init__.py
 src/ecopipeline/utils/unit_convert.py
-src/ecopipeline/utils/pkls/__init__.py
+src/ecopipeline/utils/pkls/__init__.py
+src/ecopipeline/utils/pkls/tasseron_resistance_to_temp_3.pkl
+src/ecopipeline/utils/pkls/tasseron_temp_to_resistance_2.pkl
+src/ecopipeline/utils/pkls/veris_resistance_to_temp_3.pkl
+src/ecopipeline/utils/pkls/veris_temp_to_resistance_2.pkl