ecopipeline 0.7.5__py3-none-any.whl → 0.7.7__py3-none-any.whl

ecopipeline/extract/__init__.py

@@ -1,3 +1,3 @@
-from .extract import get_noaa_data, json_to_df, extract_files, get_last_full_day_from_db, get_db_row_from_time, extract_new, csv_to_df, get_sub_dirs, msa_to_df, fm_api_to_df, small_planet_control_to_df, dent_csv_to_df, flow_csv_to_df, pull_egauge_data, egauge_csv_to_df, remove_char_sequence_from_csv_header
+from .extract import get_noaa_data, json_to_df, extract_files, get_last_full_day_from_db, get_db_row_from_time, extract_new, csv_to_df, get_sub_dirs, msa_to_df, fm_api_to_df, small_planet_control_to_df, dent_csv_to_df, flow_csv_to_df, pull_egauge_data, egauge_csv_to_df, remove_char_sequence_from_csv_header, tb_api_to_df
 __all__ = ["get_noaa_data", "json_to_df", "extract_files", "get_last_full_day_from_db", "get_db_row_from_time", 'extract_new', "csv_to_df", "get_sub_dirs", "msa_to_df", "fm_api_to_df",
-           "small_planet_control_to_df","dent_csv_to_df","flow_csv_to_df","pull_egauge_data", "egauge_csv_to_df","remove_char_sequence_from_csv_header"]
+           "small_planet_control_to_df","dent_csv_to_df","flow_csv_to_df","pull_egauge_data", "egauge_csv_to_df","remove_char_sequence_from_csv_header", "tb_api_to_df"]

ecopipeline/extract/extract.py

@@ -14,6 +14,7 @@ from pytz import timezone, utc
 import mysql.connector.errors as mysqlerrors
 import requests
 import subprocess
+import traceback
 
 
 def get_last_full_day_from_db(config : ConfigManager, table_identifier : str = "minute") -> datetime:
@@ -151,11 +152,17 @@ def extract_new(startTime: datetime, filenames: List[str], decihex = False, time
 
 
     else: 
+        endTime_int = endTime
         if epochFormat:
             startTime_int = int(startTime.timestamp())
+            if not endTime is None:
+                endTime_int = int(endTime.timestamp())
         else:
             startTime_int = int(startTime.strftime(dateFormat))
-        return_list = list(filter(lambda filename: int(filename[dateStringStartIdx:dateStringEndIdx]) >= startTime_int and (endTime is None or int(filename[dateStringStartIdx:dateStringStartIdx+14]) < int(endTime.strftime("%Y%m%d%H%M%S"))), filenames))
+            if not endTime is None:
+                endTime_int = int(endTime.strftime(dateFormat)
+                                  )
+        return_list = list(filter(lambda filename: int(filename[dateStringStartIdx:dateStringEndIdx]) >= startTime_int and (endTime_int is None or int(filename[dateStringStartIdx:dateStringEndIdx]) < endTime_int), filenames))
     return return_list
 
 def extract_files(extension: str, config: ConfigManager, data_sub_dir : str = "", file_prefix : str = "") -> List[str]:
@@ -745,6 +752,90 @@ def pull_egauge_data(config: ConfigManager, eGauge_ids: list, eGauge_usr : str,
         print(f"Could not download new data from eGauge device: {e}")
 
     os.chdir(original_directory)
+
+def tb_api_to_df(config: ConfigManager, startTime: datetime = None, endTime: datetime = None):
+    if endTime is None:
+        endTime = datetime.now()
+    if startTime is None:
+        # 28 hours to ensure encapsulation of last day
+        startTime = endTime - timedelta(hours=28)
+
+    if endTime - timedelta(hours=4) > startTime:
+        time_diff = endTime - startTime
+        midpointTime = startTime + time_diff / 2
+        # recursively construct the df
+        df_1 = tb_api_to_df(config, startTime, midpointTime)
+        df_2 = tb_api_to_df(config, midpointTime, endTime)
+        df = pd.concat([df_1, df_2])
+        df = df.sort_index()
+        df = df.groupby(df.index).mean()
+        return df
+
+    url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{config.api_device_id}/values/timeseries'
+    token = config.get_thingsboard_token()
+    keys = _get_tb_keys(config, token)
+    if len(keys) <= 0:
+        raise Exception(f"No sensors available at ThingsBoard site with id {config.api_device_id}")
+    key_string = ','.join(keys)
+    params = {
+        'keys': key_string,
+        'startTs': f'{int(startTime.timestamp())*1000}',
+        'endTs': f'{int(endTime.timestamp())*1000}',
+        'orderBy': 'ASC',
+        'useStrictDataTypes': 'false'
+    }
+
+    # Headers
+    headers = {
+        'accept': 'application/json',
+        'X-Authorization': f'Bearer {token}'
+    }
+
+    try:
+        response = requests.get(url, headers=headers, params=params)
+        if response.status_code == 200:
+            response_json = response.json()
+            data = {}
+            for key, records in response_json.items():
+                try:
+                    series = pd.Series(
+                        data={record['ts']: float(record['value']) for record in records}
+                    )
+                    data[key] = series
+                except:
+                    print_statement = f"Could not convert {key} values to floats."
+                    # print(print_statement)
+            df = pd.DataFrame(data)
+            df.index = pd.to_datetime(df.index, unit='ms')
+            df = df.sort_index()
+            return df
+            
+        print(f"Failed to make GET request. Status code: {response.status_code} {response.json()}")
+        return pd.DataFrame()
+    except Exception as e:
+        traceback.print_exc()
+        print(f"An error occurred: {e}")
+        return pd.DataFrame()
+    
+def _get_tb_keys(config: ConfigManager, token : str) -> List[str]:
+    url = f'https://thingsboard.cloud/api/plugins/telemetry/DEVICE/{config.api_device_id}/keys/timeseries'
+
+    # Headers
+    headers = {
+        'accept': 'application/json',
+        'X-Authorization': f'Bearer {token}'
+    }
+
+    try:
+        response = requests.get(url, headers=headers)
+        if response.status_code == 200:
+            return response.json()
+            
+        print(f"Failed to make GET request. Status code: {response.status_code} {response.json()}")
+        return []
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return []
     
 def get_sub_dirs(dir: str) -> List[str]:
     """

ecopipeline/utils/ConfigManager.py

@@ -64,10 +64,16 @@ class ConfigManager:
                     self.data_directory = configure.get('data', 'directory')
                     configured_data_method = True
                 if 'fieldManager_api_usr' in configure['data'] and 'fieldManager_api_pw' in configure['data'] and 'fieldManager_device_id' in configure['data']:
+                    # LEGACY, Remove when you can
                     self.api_usr = configure.get('data', 'fieldManager_api_usr')
                     self.api_pw = configure.get('data', 'fieldManager_api_pw')
                     self.api_device_id = configure.get('data','fieldManager_device_id')
                     configured_data_method = True
+                elif 'api_usr' in configure['data'] and 'api_pw' in configure['data'] and 'device_id' in configure['data']:
+                    self.api_usr = configure.get('data', 'api_usr')
+                    self.api_pw = configure.get('data', 'api_pw')
+                    self.api_device_id = configure.get('data','device_id')
+                    configured_data_method = True
             if not configured_data_method:
                 raise Exception('data configuration section missing or incomplete in configuration file.')
 
@@ -205,6 +211,7 @@ class ConfigManager:
         return connection, connection.cursor()
     
     def get_fm_token(self) -> str:
+        # for getting feild manager api token
         if self.api_usr is None or self.api_pw is None:
             raise Exception("Cannot retrieve Field Manager API token. Credentials were not provided in configuration file.")
         url = f"https://www.fieldpop.io/rest/login?username={self.api_usr}&password={self.api_pw}"
@@ -221,6 +228,36 @@ class ConfigManager:
             print(f"An error occurred: {e}")
             return None
         
+    def get_thingsboard_token(self) -> str:
+        # for getting ThingsBoard api token
+        if self.api_usr is None or self.api_pw is None:
+            raise Exception("Cannot retrieve ThingsBoard API token. Credentials were not provided in configuration file.")
+        url = 'https://thingsboard.cloud/api/auth/login'
+
+        # Request payload (data to send in the POST)
+        payload = {
+            'username': self.api_usr,
+            'password': self.api_pw
+        }
+
+        # Headers
+        headers = {
+            'Content-Type': 'application/json',
+            'Accept': 'application/json'
+        }
+        try:
+            response = requests.post(url, json=payload, headers=headers)
+            # Check if the request was successful (status code 200)
+            if response.status_code == 200:
+                response = response.json()  # Return the response data as JSON
+                return response['token']
+            else:
+                print(f"Failed to make GET request. Status code: {response.status_code}")
+                return None
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+        
     def get_fm_device_id(self) -> str:
         if self.api_device_id is None:
             raise Exception("Field Manager device ID has not been configured.")

{ecopipeline-0.7.5.dist-info → ecopipeline-0.7.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecopipeline
-Version: 0.7.5
+Version: 0.7.7
 Summary: Contains functions for use in Ecotope Datapipelines
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License (GPL)

{ecopipeline-0.7.5.dist-info → ecopipeline-0.7.7.dist-info}/RECORD

@@ -1,17 +1,17 @@
 ecopipeline/__init__.py,sha256=vCRzwd781ciCSXMP1ycM_BXAqxj3KVaNKIjsLOPcbwc,171
-ecopipeline/extract/__init__.py,sha256=3u_CUMdCguVewU3kN8x6xhVNyo1-p-gwTrhjOh7Psqg,645
-ecopipeline/extract/extract.py,sha256=heWcWTeRVTRITh_1sHVnkaKOOi5PwUOEVIi4k5tw2Z8,43384
+ecopipeline/extract/__init__.py,sha256=gQ3sak6NJ63Gpo-hZXrtZfeKOTHLRyAVXfTgxxRpqPo,675
+ecopipeline/extract/extract.py,sha256=ITXr8XuadQ81kYA006hygvIYgmkpQXTL6KN4QTpNOvE,46707
 ecopipeline/load/__init__.py,sha256=NLa_efQJZ8aP-J0Y5xx9DP7mtfRH9jY6Jz1ZMZN_BAA,292
 ecopipeline/load/load.py,sha256=D3KdQZ08hYRso42p59YE-eVQaTHuANCJ0YeJvRnOgG4,23078
 ecopipeline/transform/__init__.py,sha256=hYb4F64fXdXtjBSYCqv6gLFBwKZjjnl0z7s291pFE98,2505
 ecopipeline/transform/bayview.py,sha256=TP24dnTsUD95X-f6732egPZKjepFLJgDm9ImGr-fppY,17899
 ecopipeline/transform/lbnl.py,sha256=EQ54G4rJXaZ7pwVusKcdK2KBehSdCsNo2ybphtMGs7o,33400
 ecopipeline/transform/transform.py,sha256=DBQD4WqKmdXnGQMAj6tg75HtXiSemIc7c6nZxzA2aXc,48958
-ecopipeline/utils/ConfigManager.py,sha256=avhUEpx7WhhmnT-MB8OQ_siVgS1k5zp7KCYOyzzOJkE,10506
+ecopipeline/utils/ConfigManager.py,sha256=-g1wtExdvhYO5Y6Q3cRbywa__DxRMFruLrB4YanwaPY,12168
 ecopipeline/utils/__init__.py,sha256=ccWUR0m7gD9DfcgsxBCLOfi4lho6RdYuB2Ugy_g6ZdQ,28
 ecopipeline/utils/unit_convert.py,sha256=VFh1we2Y8KV3u21BeWb-U3TlZJXo83q5vdxxkpgcuME,3064
-ecopipeline-0.7.5.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ecopipeline-0.7.5.dist-info/METADATA,sha256=rj-QU_0DXM8mf-P6vLFzhxRB5ErzXVkqTMWFhA3ZywU,2329
-ecopipeline-0.7.5.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
-ecopipeline-0.7.5.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
-ecopipeline-0.7.5.dist-info/RECORD,,
+ecopipeline-0.7.7.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ecopipeline-0.7.7.dist-info/METADATA,sha256=ShFLOFd95TQUDVZ0IZU4QIS8iLX_kjyd2wS95fMQCEU,2329
+ecopipeline-0.7.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ecopipeline-0.7.7.dist-info/top_level.txt,sha256=WOPFJH2LIgKqm4lk2OnFF5cgVkYibkaBxIxgvLgO7y0,12
+ecopipeline-0.7.7.dist-info/RECORD,,

{ecopipeline-0.7.5.dist-info → ecopipeline-0.7.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.1.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any