ecopipeline 0.8.10__py3-none-any.whl → 0.8.12__py3-none-any.whl

ecopipeline/extract/extract.py

@@ -767,7 +767,8 @@ def pull_egauge_data(config: ConfigManager, eGauge_ids: list, eGauge_usr : str,
 
     os.chdir(original_directory)
 
-def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: datetime = None, create_csv : bool = True, query_hours : int = 12):
+def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: datetime = None, create_csv : bool = True, query_hours : float = 1,
+                 sensor_keys : list = [], seperate_keys : bool = False):
     """
     Function connects to the things board manager api to pull data and returns a dataframe.
 
@@ -785,7 +786,7 @@ def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: dat
         is local time from the data's index. 
     create_csv : bool
         create csv files as you process such that API need not be relied upon for reprocessing
-    query_hours : int
+    query_hours : float
         number of hours to query at a time from ThingsBoard API
     
     Returns
@@ -794,80 +795,102 @@ def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: dat
         Pandas Dataframe containing data from the API pull with column headers the same as the variable names in the data from the pull.
         Will return with index in UTC so needs to be converted after to appropriate timezone
     """
-    if endTime is None:
-        endTime = datetime.now()
-    if startTime is None:
-        # 28 hours to ensure encapsulation of last day
-        startTime = endTime - timedelta(hours=28)
-
-    if endTime - timedelta(hours=query_hours) > startTime:
-        time_diff = endTime - startTime
-        midpointTime = startTime + time_diff / 2
-        # recursively construct the df
-        df_1 = tb_api_to_df(config, startTime, midpointTime, create_csv=False,query_hours=query_hours)
-        df_2 = tb_api_to_df(config, midpointTime, endTime, create_csv=False,query_hours=query_hours)
-        df = pd.concat([df_1, df_2])
-        df = df.sort_index()
-        df = df.groupby(df.index).mean()
-        if create_csv:
-                filename = f"{startTime.strftime('%Y%m%d%H%M%S')}.csv"
-                original_directory = os.getcwd()
-                os.chdir(config.data_directory)
-                df.to_csv(filename, index_label='time_pt')
-                os.chdir(original_directory)
-
-        return df
-    url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{config.api_device_id}/values/timeseries'
-    token = config.get_thingsboard_token()
-    keys = _get_tb_keys(config, token)
-    if len(keys) <= 0:
-        raise Exception(f"No sensors available at ThingsBoard site with id {config.api_device_id}")
-    key_string = ','.join(keys)
-    params = {
-        'keys': key_string,
-        'startTs': f'{int(startTime.timestamp())*1000}',
-        'endTs': f'{int(endTime.timestamp())*1000}',
-        'orderBy': 'ASC',
-        'useStrictDataTypes': 'false'
-    }
-
-    # Headers
-    headers = {
-        'accept': 'application/json',
-        'X-Authorization': f'Bearer {token}'
-    }
-
-    try:
-        response = requests.get(url, headers=headers, params=params)
-        if response.status_code == 200:
-            response_json = response.json()
-            data = {}
-            for key, records in response_json.items():
-                try:
-                    series = pd.Series(
-                        data={record['ts']: float(record['value']) for record in records}
-                    )
-                    data[key] = series
-                except:
-                    print_statement = f"Could not convert {key} values to floats."
-                    # print(print_statement)
-            df = pd.DataFrame(data)
-            df.index = pd.to_datetime(df.index, unit='ms')
+    df = pd.DataFrame()
+    if len(sensor_keys) <= 0:
+        token = config.get_thingsboard_token()
+        key_list = _get_tb_keys(config, token)
+        if len(key_list) <= 0:
+            raise Exception(f"No sensors available at ThingsBoard site with id {config.api_device_id}")
+        return tb_api_to_df(config, startTime, endTime, create_csv, query_hours, key_list, seperate_keys)
+    if seperate_keys:
+        df_list = []
+        for sensor_key in sensor_keys:
+            df_list.append(tb_api_to_df(config, startTime, endTime, False, query_hours, [sensor_key], False))
+        df = pd.concat(df_list)
+    else:    
+    # not seperate_keys:
+        if endTime is None:
+            endTime = datetime.now()
+        if startTime is None:
+            # 28 hours to ensure encapsulation of last day
+            startTime = endTime - timedelta(hours=28)
+
+        if endTime - timedelta(hours=query_hours) > startTime:
+            time_diff = endTime - startTime
+            midpointTime = startTime + time_diff / 2
+            df_1 = tb_api_to_df(config, startTime, midpointTime, query_hours=query_hours, sensor_keys=sensor_keys, create_csv=False)#True if startTime >= datetime(2025,7,13,9) and startTime <= datetime(2025,7,13,10) else csv_pass_down)
+            df_2 = tb_api_to_df(config, midpointTime, endTime, query_hours=query_hours, sensor_keys=sensor_keys,create_csv=False)#True if endTime >= datetime(2025,7,13,9) and endTime <= datetime(2025,7,13,10) else csv_pass_down)
+            df = pd.concat([df_1, df_2])
             df = df.sort_index()
-            # save to file
-            if create_csv:
-                filename = f"{startTime.strftime('%Y%m%d%H%M%S')}.csv"
-                original_directory = os.getcwd()
-                os.chdir(config.data_directory)
-                df.to_csv(filename, index_label='time_pt')
-                os.chdir(original_directory)
-            return df
-        print(f"Failed to make GET request. Status code: {response.status_code} {response.json()}")
-        return pd.DataFrame()
-    except Exception as e:
-        traceback.print_exc()
-        print(f"An error occurred: {e}")
-        return pd.DataFrame()
+            df = df.groupby(df.index).mean()
+        else:
+            url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{config.api_device_id}/values/timeseries'
+            token = config.get_thingsboard_token()
+            key_string = ','.join(sensor_keys)
+            params = {
+                'keys': key_string,
+                'startTs': f'{int(startTime.timestamp())*1000}',
+                'endTs': f'{int(endTime.timestamp())*1000}',
+                'orderBy': 'ASC',
+                'useStrictDataTypes': 'false',
+                'interval' : '0',
+                'agg' : 'NONE'
+            }
+
+            # Headers
+            headers = {
+                'accept': 'application/json',
+                'X-Authorization': f'Bearer {token}'
+            }
+
+            try:
+                response = requests.get(url, headers=headers, params=params)
+                if response.status_code == 200:
+                    response_json = response.json()
+                    # if create_csv:
+                    #     json_filename = f"{startTime.strftime('%Y%m%d%H%M%S')}.json"
+                    #     print(f"filename: {json_filename}, url: {url}, params: {params}")
+                    #     original_directory = os.getcwd()
+                    #     os.chdir(config.data_directory)
+                    #     with open(json_filename, 'w') as f:
+                    #         json.dump(response_json, f, indent=4)  # indent=4 makes it human-readable
+                    #     os.chdir(original_directory)
+                        
+                    data = {}
+                    for key, records in response_json.items():
+                        try:
+                            series = pd.Series(
+                                data={record['ts']: _get_float_value(record['value'])  for record in records}
+                            )
+                            data[key] = series
+                        except:
+                            print_statement = f"Could not convert {key} values to floats."
+                            print(print_statement)
+                    df = pd.DataFrame(data)
+                    df.index = pd.to_datetime(df.index, unit='ms')
+                    df = df.sort_index()
+                else:
+                    print(f"Failed to make GET request. Status code: {response.status_code} {response.json()}")
+                    df = pd.DataFrame()
+            except Exception as e:
+                traceback.print_exc()
+                print(f"An error occurred: {e}")
+                df = pd.DataFrame()
+    # save to file
+    if create_csv:
+        filename = f"{startTime.strftime('%Y%m%d%H%M%S')}.csv"
+        original_directory = os.getcwd()
+        os.chdir(config.data_directory)
+        df.to_csv(filename, index_label='time_pt')
+        os.chdir(original_directory)
+    return df
+    
+def _get_float_value(value):
+    try:
+        ret_val = float(value)
+        return ret_val
+    except (ValueError, TypeError):
+        return None
     
 def _get_tb_keys(config: ConfigManager, token : str) -> List[str]:
     url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{config.api_device_id}/keys/timeseries'

{ecopipeline-0.8.10.dist-info → ecopipeline-0.8.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecopipeline
-Version: 0.8.10
+Version: 0.8.12
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

{ecopipeline-0.8.10.dist-info → ecopipeline-0.8.12.dist-info}/RECORD

@@ -2,7 +2,7 @@ ecopipeline/__init__.py,sha256=d48mO5La6OrQDkRe_qqoY6lUx7x-e8krOH388jmWjwU,218
 ecopipeline/event_tracking/__init__.py,sha256=q49j46fXMUjNUPzL4FvXEppB93i3lUni-QUZpp61tt0,64
 ecopipeline/event_tracking/event_tracking.py,sha256=LOCLE7ju320O7CrwnWRIqHRa2uAqoq-KvXZ3zWQ2S74,13224
 ecopipeline/extract/__init__.py,sha256=gQ3sak6NJ63Gpo-hZXrtZfeKOTHLRyAVXfTgxxRpqPo,675
-ecopipeline/extract/extract.py,sha256=lWyjD7f95nfmJ-BojAGFA-2mXiyPgXceDci-jAJmCSE,49494
+ecopipeline/extract/extract.py,sha256=i5e-mL4KiJfmWMFnKwvCW_BdPELGJiYKRk6FbvcEi1U,51106
 ecopipeline/load/__init__.py,sha256=NLa_efQJZ8aP-J0Y5xx9DP7mtfRH9jY6Jz1ZMZN_BAA,292
 ecopipeline/load/load.py,sha256=Ptxr0MOjns_HeVSmZsLLApHJGB-z6XOB2m8LNiVaD7E,23860
 ecopipeline/transform/__init__.py,sha256=hYb4F64fXdXtjBSYCqv6gLFBwKZjjnl0z7s291pFE98,2505
@@ -12,8 +12,8 @@ ecopipeline/transform/transform.py,sha256=S8fpAb45XBcYzeGNkxELiHM8-1jlNQqADV7_m-
 ecopipeline/utils/ConfigManager.py,sha256=-g1wtExdvhYO5Y6Q3cRbywa__DxRMFruLrB4YanwaPY,12168
 ecopipeline/utils/__init__.py,sha256=ccWUR0m7gD9DfcgsxBCLOfi4lho6RdYuB2Ugy_g6ZdQ,28
 ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
-ecopipeline-0.8.10.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ecopipeline-0.8.10.dist-info/METADATA,sha256=Jpq72FqjQUHRG0VUeCaLOLMzaVJqWANDzpe_QZouTnI,2330
-ecopipeline-0.8.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ecopipeline-0.8.10.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
-ecopipeline-0.8.10.dist-info/RECORD,,
+ecopipeline-0.8.12.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ecopipeline-0.8.12.dist-info/METADATA,sha256=j4dFIoOVhvTICE2htIZhbUJPZmE6lpVuWDGk33BdBNc,2330
+ecopipeline-0.8.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ecopipeline-0.8.12.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
+ecopipeline-0.8.12.dist-info/RECORD,,