energy-reports-pkg 0.6.14__py3-none-any.whl

mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/load_mailing_list/get_excel_file_from_link/__init__.py

@@ -0,0 +1,30 @@
+
+
+def get_excel_file_from_link(ctx, sharepoint_link):
+
+    # get the folder link from the file link
+    sharepoint_folder_link = sharepoint_link.rsplit('/', maxsplit=1)[0]
+    
+    # get sharepoint folder
+    libraryRoot = ctx.web.get_folder_by_server_relative_path(sharepoint_folder_link)
+    ctx.load(libraryRoot)
+    ctx.execute_query()
+
+    # get files in folder
+    files_in_folder = libraryRoot.files
+    ctx.load(files_in_folder)
+    ctx.execute_query()
+
+    # debug
+    # print(f'{files_in_folder = }')
+
+    # with the condition that there is only one file
+    # excel_file = files_in_folder[0]
+
+    for file in files_in_folder:
+        files_url = file.properties['ServerRelativeUrl']
+        if files_url == sharepoint_link:
+            return file
+
+    raise Exception('[ERROR]: Cannot find the specified file!')
+    # return excel_file

mindsett_energy_reports_pkg/libs/fetch_sharepoint_info/sync_with_sharepoint/extract_contacts/load_mailing_list/load_excel_as_df/__init__.py

@@ -0,0 +1,45 @@
+
+import pandas as pd
+import io
+
+def load_excel_as_df(ctx, excel_file, sheet_name=None):
+
+    if sheet_name is None:
+        sheet_name = 0
+    
+    last_update_sys = excel_file.properties['TimeLastModified']
+    last_update = pd.Timestamp(last_update_sys, tz='Europe/London').tz_convert(tz='UTC') # assume that the sharepoint system time in as London time
+
+    # print(f'{last_update = }')
+    files_url = excel_file.properties['ServerRelativeUrl']
+
+    # print(f'{last_update=}')
+    # print(f'{files_url=}')
+
+    # files_url = excel_file
+
+    # ctx.web.get().execute_query()
+
+    with io.BytesIO() as data:
+        _myfile = (ctx.web.get_file_by_server_relative_path(files_url)
+                    .download(data)
+                    .execute_query() 
+                    ) 
+        # print(f"{type(_myfile) = }")
+
+        data.seek(0)
+
+        # cols_dtype = {'board_id': str, 
+        #               'hvac_board_id': str}
+
+        df_contact = pd.read_excel(data, 
+                                #    dtype=cols_dtype,
+                            sheet_name=sheet_name,)
+        
+    # _myfile = (ctx.web.get_file_by_server_relative_path(files_url)
+    #                 .execute_query() 
+    #                 ) 
+        
+    df_contact['last_update'] = last_update
+
+    return df_contact

mindsett_energy_reports_pkg/libs/generate_dag_id/__init__.py

@@ -0,0 +1,20 @@
+import pathlib
+
+def generate_dag_id(file):
+
+    current_path = pathlib.Path(file)
+    current_dir = current_path.parents[1]
+    current_dir_str = str(current_dir)
+
+    # directory = current_dir_str.split('_dags')[-1][1:].replace('/', '_')
+
+    project = current_dir_str.split('_pkg')[0].split('/')[-1]
+
+    dag_id = project
+
+    # if directory == '':
+    #     dag_id = project
+    # else:
+    #     dag_id = f"{project}_{directory}"
+
+    return dag_id

mindsett_energy_reports_pkg/libs/get_conn_by_id/__init__.py

@@ -0,0 +1,42 @@
+
+from .get_conn_by_id_json import get_conn_by_id_json
+
+def get_conn_by_id(conn_id, 
+                   account_key=None,
+                   credential_url=None):
+    
+    # # fetching creds from local filesystem or datalake
+    # cred_conn = get_conn_by_id_json(conn_id, 
+    #                                 credential_url=credential_url)
+    
+    # if cred_conn != None:
+    #     return cred_conn
+    # else:
+    #     # fetching creds from the airflow environment
+    #     from .get_conn_by_id_airflow import get_conn_by_id_airflow
+        
+    #     cred_conn = get_conn_by_id_airflow(conn_id)
+
+    # for testing
+    cred_conn = None
+    
+    if cred_conn != None:
+        return cred_conn
+    else:
+        # fetching creds from the synapse environment
+        from .get_conn_by_id_synapse import get_conn_by_id_synapse
+
+        if account_key is None:
+            try:
+                from notebookutils import mssparkutils
+                account_key = mssparkutils.credentials.getConnectionStringOrCreds('AzureDataLakeStorageTest')
+            except:
+                raise Exception('Cannot get a valid account_key from mssparkutils, please provide one for getting credentials in the synapse environment')
+
+        cred_conn = get_conn_by_id_synapse(conn_id, account_key)
+        return cred_conn
+
+
+
+
+

mindsett_energy_reports_pkg/libs/get_conn_by_id/__version__.py

@@ -0,0 +1,8 @@
+
+__version__ = '0.0.3' # can set multiple paths for local and databricks
+
+# __version__ = '0.0.2' # being able to get conn from local, airflow, and databricks
+
+# __version__ = '0.0.1' # require two functions for getting the conn
+
+# __version__ = '0.0.0' # the basic way for getting conn only from airflow

mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_airflow/__init__.py

@@ -0,0 +1,47 @@
+
+import inspect
+from airflow.hooks.base_hook import BaseHook
+import json
+
+def get_conn_by_id_airflow(conn_id):
+
+    cred_conn = BaseHook.get_connection(conn_id)
+
+    cred_dict = {}
+    for attribute in dir(cred_conn):
+        print(cred_conn, 'attr: ', attribute)
+
+        if not attribute.startswith('__'):
+            if not inspect.ismethod(attribute):
+                cred_dict[attribute] = getattr(cred_conn, attribute)
+                if attribute == 'schema':
+                    cred_dict['database'] = getattr(cred_conn, attribute)
+                elif attribute == 'login':
+                    cred_dict['username'] = getattr(cred_conn, attribute)
+                    cred_dict['client_id'] = getattr(cred_conn, attribute)
+        
+    cred_dict['password'] = cred_conn.get_password()
+    cred_dict['token'] = cred_conn.get_password()
+    cred_dict['api_key'] = cred_conn.get_password()
+    cred_dict['secret'] = cred_conn.get_password()
+
+    cred_extra = cred_conn.get_extra()
+
+    if len(cred_extra) > 2:
+        cred_extra_json = json.loads(cred_extra)
+
+        cred_dict = {**cred_dict, **cred_extra_json}
+
+    print('cred_dict: ', cred_dict)
+    
+    return cred_dict
+
+
+
+
+
+    # server = email_conn.host
+    # port = email_conn.port
+    # send_from = json.loads(email_conn.get_extra())["send_from"]
+    # username = email_conn.login
+    # password = email_conn.get_password()

mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_json/__init__.py

@@ -0,0 +1,25 @@
+import json
+from .get_credential_url import get_credential_url
+
+def get_conn_by_id_json(conn_id, 
+                        credential_url=None):
+    
+    if credential_url == None:
+        credential_url = get_credential_url()
+
+        if credential_url == None: # if there is no credential_url from localhost or databricks
+            return None
+
+    with open(credential_url) as f:
+        credentials = json.load(f)
+
+
+        
+    for connection in credentials['connections']:
+        if connection['connection_id'] == conn_id:
+            return connection['secrets']
+    
+    print('[INFO]: cannot found the connection_id!')
+    return None # if there is no such credential id yet
+
+    # raise('cannot found the connection_id!')

mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_json/get_credential_url/__init__.py

@@ -0,0 +1,17 @@
+import os
+
+from . import databricks
+from . import local
+
+def get_credential_url():
+
+    for local_credential_url in local.credential_urls:
+        if os.path.exists(local_credential_url):
+            return local_credential_url
+        
+    for databricks_credential_url in databricks.credential_urls:
+        if os.path.exists(databricks_credential_url):
+            return databricks_credential_url
+        
+    print("[INFO]: Cannot find credentials from local server or databricks!")
+    return None

mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_json/get_credential_url/databricks/__init__.py

@@ -0,0 +1,10 @@
+
+
+# import os
+
+# from pathlib import Path
+# cache_directory = os.path.join(os.path.dirname(__file__), '_cache_/')
+
+credential_urls = ['/dbfs/FileStore/shared_uploads/x.yang@cloudfmgroup.com/credentials/credentials.json']
+
+# print('credential_url: ', credential_url)

mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_json/get_credential_url/local/__init__.py

@@ -0,0 +1,13 @@
+
+
+import os
+
+# from pathlib import Path
+# cache_directory = os.path.join(os.path.dirname(__file__), '_cache_/')
+
+# credential_url = os.path.join(os.path.dirname(__file__), 'credentials.json')
+
+credential_urls = ['/Volumes/EXT SSD/Cloudfm Backup/Documents at Cloudfm/Accounts/credentials/credentials.json',
+                   '/Volumes/macOS 1/Users/Shared/Cloudfm Backup/OneDrive - Cloudfm Integrated Services Ltd - Backup/KTP Project/Cloudfm/Accounts/credentials/credentials.json']
+
+# '/dbfs/FileStore/shared_uploads/x.yang@cloudfmgroup.com/credentials_/credentials.json'

mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_synapse/__init__.py

@@ -0,0 +1,15 @@
+from . import datalake
+from .connect_lake_and_fetch_json import connect_lake_and_fetch_json
+
+def get_conn_by_id_synapse(conn_id, account_key):
+    
+    credentials = connect_lake_and_fetch_json(datalake, account_key)
+
+    for connection in credentials['connections']:
+        if connection['connection_id'] == conn_id:
+            return connection['secrets']
+    
+    print('[INFO]: cannot found the connection_id!')
+    return None # if there is no such credential id yet
+
+    # raise('cannot found the connection_id!')

mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_synapse/connect_lake_and_fetch_json/__init__.py

@@ -0,0 +1,19 @@
+
+from azure.storage.filedatalake import DataLakeServiceClient
+import io
+import json
+
+
+def connect_lake_and_fetch_json(datalake, account_key):
+
+    # connect to the datalake and fetch the json file
+    azure_service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format(
+            "https", datalake.account_name), credential=account_key)
+        
+    azure_file_system_client = azure_service_client.get_file_system_client(file_system=datalake.container_name)
+
+    file_client = azure_file_system_client.get_file_client(datalake.file_path)
+    data = file_client.download_file().readall()
+    json_dict = json.load(io.BytesIO(data))
+
+    return json_dict

mindsett_energy_reports_pkg/libs/get_conn_by_id/get_conn_by_id_synapse/datalake/__init__.py

@@ -0,0 +1,7 @@
+
+
+account_name = f"dlcfmdsamindsettprod"
+container_name= 'uploads'
+file_path = 'energy_report_config/credentials.json'
+
+# get the account key from synapse

mindsett_energy_reports_pkg/mdl/__init__.py

@@ -0,0 +1,2 @@
+
+from .prepare_and_email_reports import prepare_and_email_reports

mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/__init__.py

@@ -0,0 +1,119 @@
+from pathlib import Path
+import os
+import ast
+
+class ClientSite:
+
+    from .get_schedule_records import get_schedule_records
+    from .get_period_range import get_period_range
+    from .get_start_time import get_start_time
+    from .get_start_time_co2_barchart import get_start_time_co2_barchart
+    from .get_end_time import get_end_time
+
+    pct_level_tobe_others = 0.03
+    pct_hide = 4 # hide the labels smaller then the value specified by pct_hide
+    currency = 'GBP'
+    conv_mwh_price = 190
+    timezone = 'UTC'
+    size_in_sqm = True
+
+    # setup default backup schedule info
+    weekend=7
+    working_start_time="08:30:00"
+    working_end_time="23:59:59"
+
+    period_offset_previous = 2 # offset between the current period and the previous finished period
+    period_freq = 'W' # M for monthly and W for weekly
+    period_offset = 1 # offset between the current period and the most recent finished period
+    period_count = 6 # number of period to show in the co2 barchart
+    group_name_column_exchange = {} # for some special (usually big) groups, to disclose further details by using its name from another column, e.g. {"Sockets": "circuit_description"}
+    group_name_modification = {} # replacement name to make the long and complex name more readable
+
+    occupancy_available = False
+    asset_group = "thing_category"
+    fillna_value = "Undefined"
+    exception = {'link_type': ['Supply', 'Load to DB', 'Load to Exclude']}
+    insight_statements = None # this will decide whether to show the insights or not
+
+    working_start_time="00:00:00" #Default start and end time, may not be used (it is needed for the functions) if the schedule information from the database
+    working_end_time="23:59:00"
+    # print('group_name_modification: ', group_name_modification) # debug
+    reviewer_emails = ['x.yang@cloudfmgroup.com', 
+                    #    's.goud@cloudfmgroup.com',
+                        's.ruthven@cloudfmgroup.com', 
+    #                     'r.patel@cloudfmgroup.com',
+                        'd.attoe@cloudfmgroup.com'
+                      ]
+    # reviewer_emails = ['x.yang@cloudfmgroup.com', 
+    #                    's.goud@cloudfmgroup.com', 
+    #                    's.ruthven@cloudfmgroup.com', 
+    #                    'r.patel@cloudfmgroup.com']
+    # cloudfm_cc_emails = ['x.yang@cloudfmgroup.com', 
+    #                     's.goud@cloudfmgroup.com', 
+    #                     's.ruthven@cloudfmgroup.com', 
+    #                     'r.patel@cloudfmgroup.com',
+    #                     'd.attoe@cloudfmgroup.com']
+    
+    # 12 March 2024: [Removed] ["d.attoe@cloudfmgroup.com","s.ruthven@cloudfmgroup.com", "r.patel@cloudfmgroup.com", 's.goud@cloudfmgroup.com']
+
+    def __init__(self, site_item):
+
+        site_dict = site_item.squeeze(axis=0).to_dict()
+
+        # print('site_item: ', site_item)
+        # print('site_dict: ', site_dict)
+
+        # file_path = Path(site_obj.__file__)
+
+        
+
+        self.site_name = site_dict['building_name'].replace("'","''")
+        self.org_name = site_dict['org'].replace("'","''")
+
+        # if site_dict['bld_name_abbr'] == '':
+        #     self.bld_name_abbr = self.site_name
+        # else:
+        #     self.bld_name_abbr = site_dict['bld_name_abbr']
+
+        print('\n' + self.org_name + ' - ' + self.site_name + ': ')
+
+        if site_dict['group_name_column_exchange'] != '':
+            self.group_name_column_exchange = ast.literal_eval(site_dict['group_name_column_exchange'])
+        
+        if site_dict['group_name_modification'] != '':
+            self.group_name_modification = ast.literal_eval(site_dict['group_name_modification'])
+
+        if site_dict['insight_statements'] != '':
+            self.insight_statements = ast.literal_eval(site_dict['insight_statements'])
+
+        # monday_config = ['site_dict']
+
+        for attribute in site_dict:
+            if attribute not in ['building_name', 
+                                 'publish', 
+                                 'testing', 
+                                 'org', 
+                                 'send_to', 
+                                 'mailing_list',
+                                 'group_name_column_exchange', 
+                                 'group_name_modification',
+                                 'insight_statements',
+                                 'last_update']:
+                
+                if 'email' not in attribute:
+                    setattr(self, attribute, site_dict[attribute])
+                    # print('debug ClientSite attr: ', attribute, site_dict[attribute])
+        
+        if self.manager_name == '':
+            self.manager_name = self.site_name + ' ' + self.org_name
+
+        
+        # for attribute in dir(site_obj):
+        #     if not attribute.startswith('__'):
+        #         setattr(self, attribute, getattr(site_obj, attribute))
+
+        # self.manager_emails += self.cloudfm_cc_emails
+
+if __name__ == '__main__':
+    a = ClientSite()
+    print(type(a))

mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/get_end_time/__init__.py

@@ -0,0 +1,8 @@
+
+
+def get_end_time(self):
+
+    period_range = self.get_period_range()
+    # end_time = period_range.max().start_time # end time of the most recent finished period
+    end_time = period_range[-(self.period_offset)].start_time # end time of the offset period
+    return end_time

mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/get_period_range/__init__.py

@@ -0,0 +1,8 @@
+import pandas as pd
+
+def get_period_range(self):
+
+    reference_time = pd.Timestamp.now(tz=self.timezone)  
+    period_range = pd.period_range(end=reference_time, freq=self.period_freq, periods=self.period_count+self.period_offset)
+
+    return period_range

mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/get_schedule_records/__init__.py

@@ -0,0 +1,34 @@
+
+import pandas as pd
+
+
+def get_schedule_records(self, db_schedule):
+
+    # print(f'self.site_name: "{self.site_name}"')
+    # print(f'self.org_name: "{self.org_name}"')
+
+    statement_list = [f"""bs.id = '{self.building_id}'"""]
+
+    if self.org_name is not None:
+        statement_new  = f"""os.id = '{self.org_id}'"""
+        statement_list.append(statement_new)
+    statement_full = " and ".join(statement_list)
+
+    engine = db_schedule.engine # new installation - The Hub - ENGINE_WH
+
+    with engine.connect() as conn:
+        query = f"""select ss.day
+                    ,ss.name
+                    ,ss.start_time
+                    ,ss.end_time
+                    ,BTRIM(bs.name) as site_name 
+                    ,BTRIM(os.name) as organisation
+                    from {db_schedule.table_schedule.name} ss
+                    join {db_schedule.table_building.name} bs on ss.space_id = bs.id
+                    join {db_schedule.table_org.name} os on bs.org_id = os.id
+                    where {statement_full};"""
+        df_schedule = pd.read_sql_query(query,
+                                        con=conn) #new installation - The Hub - table_name_wh
+        
+        # print("schedule_query: ", query)
+    return df_schedule.to_dict('records')

mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/get_start_time/__init__.py

@@ -0,0 +1,6 @@
+
+def get_start_time(self):
+
+        period_range = self.get_period_range()
+        start_time = period_range[-(self.period_offset_previous+1)].start_time # start time of the (previous) period to be compared
+        return start_time

mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/ClientSite/get_start_time_co2_barchart/__init__.py

@@ -0,0 +1,7 @@
+
+
+def get_start_time_co2_barchart(self):
+
+    period_range = self.get_period_range()
+    start_time_co2_barchart = period_range.min().start_time # start time of the first bar in the co2 barchart
+    return start_time_co2_barchart

mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/__init__.py

@@ -0,0 +1,93 @@
+
+import os
+import pandas as pd
+
+pd.options.mode.chained_assignment = None
+
+from .prepare_report import prepare_report
+from .email_report import email_report
+from .ClientSite import ClientSite
+from .load_config import load_config
+from .select_sites_and_emails import select_sites_and_emails
+
+from mdl.utils_ import (
+    delete_files_in_directory,
+    setup_table
+)
+
+def prepare_and_email_reports(cf, 
+                              mode='review', 
+                              debug=False,
+                              sync_source=None,
+                              cache_in_memory=False, 
+                              cache_directory=None):
+
+    if cache_in_memory:
+        report_file_folder = None
+        directory_to_savefile = None
+    else:
+        if cache_directory == None:
+            cache_directory = os.path.join(os.path.dirname(__file__), '_cache_/')
+
+        report_file_folder = os.path.join(cache_directory, '_cached_reports_/')
+        directory_to_savefile = os.path.join(cache_directory, '_cached_files_/')
+
+    setup_table(cf.postgresdb.engine, 
+                cf.postgresdb.table_mailing.name, 
+                cf.postgresdb.table_mailing.columns_dict,
+                hypertable=cf.postgresdb.table_mailing.hypertable)
+    
+    setup_table(cf.postgresdb.engine, 
+                cf.postgresdb.table_cache.name, 
+                cf.postgresdb.table_cache.columns_dict,
+                hypertable=cf.postgresdb.table_cache.hypertable)
+    
+    # get contacts from db
+
+    df_contact = load_config(cf.postgresdb.engine, 
+                             cf.sharepoint,
+                             cf.postgresdb.table_org.name,
+                             cf.postgresdb.table_building.name,
+                             cf.postgresdb.table_timezone.name,
+                             cf.postgresdb.table_sqm.name,
+                             cf.postgresdb.table_price.name,
+                             cf.postgresdb.table_cache.name, 
+                             list(cf.postgresdb.table_cache.columns_dict),
+                             sync_source=sync_source)
+
+    df_contact_select = select_sites_and_emails(df_contact, mode, cf.email.reviewer_emails)
+
+    # print(f'{df_contact_select=}')
+
+    # quit()
+    for _, row in df_contact_select.iterrows():
+
+        site_obj = ClientSite(row)
+        df_receivers = pd.DataFrame({'email': row['send_to']})
+
+        if df_receivers['email'].nunique() > 0:
+            
+            file_path_or_obj_dict, current_period_obj = prepare_report(cf, site_obj, 
+                                                                    debug=debug,
+                                                                    report_file_folder=report_file_folder, 
+                                                                    directory_to_savefile=directory_to_savefile)    
+
+            # print('row: ', row)
+            # print("row['send_to']: ", row['send_to'])
+
+            df_receivers = pd.DataFrame({'email': row['send_to']})
+            df_receivers['name'] = site_obj.manager_name
+
+            receivers = df_receivers.to_dict('records')
+
+            # print('receivers: ', receivers)
+
+            # quit()
+
+            email_report(receivers, cf.smtp, cf.email, current_period_obj, file_path_or_obj_dict)
+        else: 
+            print(f'{site_obj.site_name} - {site_obj.site_name}: no email address to send to!')
+        
+    if not cache_in_memory:
+        delete_files_in_directory(report_file_folder, ignore=['__init__.py'])
+        delete_files_in_directory(directory_to_savefile, ignore=['__init__.py'])

mindsett_energy_reports_pkg/mdl/prepare_and_email_reports/email_report/__init__.py

@@ -0,0 +1,47 @@
+
+from mdl.utils_ import (
+    send_email,
+)
+import jinja2
+import pandas as pd
+
+def email_report(receivers, smtp, email, current_period_obj, file_path_or_obj_dict):
+
+    for receiver in receivers:
+
+        send_to = [receiver['email']]
+
+        environment = jinja2.Environment()
+        context = {}
+
+        period_type = current_period_obj.freqstr[0]
+
+        if period_type == "W":
+            # current_period_str = (current_period_obj).strftime("Week %W, %Y") 
+            # add 1 week to the current_period_obj so that the start week is not Week 00
+            period_week_str = current_period_obj.strftime('%W')
+            period_year_str = current_period_obj.strftime('%Y')
+            current_period_str = f'Week {(int(period_week_str)+1):02d}, {period_year_str}'
+
+        else:
+            current_period_str = current_period_obj.strftime("Month %b %Y")
+        context['current_period_str'] = current_period_str
+        context['receiver_name'] = receiver['name']
+        
+
+        subject_template = environment.from_string(email.subject)
+        email_subject = subject_template.render(ct=context)
+        
+        message_template = environment.from_string(email.message)
+        email_message = message_template.render(ct=context)
+
+        # file_path_or_obj_dict = {filename: file_path/file_obj}
+
+        send_email(smtp.send_from, send_to, 
+                    email_subject, email_message, 
+                    # signature=email.signature,
+                    files=file_path_or_obj_dict,
+                    server=smtp.server,
+                    port=smtp.port,
+                    username=smtp.username,
+                    password=smtp.password)