berryworld 1.0.0.197723__py3-none-any.whl → 1.0.0.199021__py3-none-any.whl

berryworld/__init__.py

@@ -4,6 +4,7 @@ from .transportation_solver import TransportationAlgorithm
 from .allocation_solver import AllocationSolver
 from .pickle_management import PickleManagement
 from .email_logging import EmailLogging
+from .python_logs import ErrorLogs
 from .verify_keys import Verify
 from .credentials import SnowflakeCredentials, SQLCredentials, WebServiceCredentials, MicrosoftTeamsCredentials
 from .persistent_storage import PersistentStorage

berryworld/logging.py

@@ -1,6 +1,3 @@
-
-
-
 class PythonLogs:
     """ Register the Python Logs """
     def __init__(self, conn, batch_process):

berryworld/python_logs.py

@@ -0,0 +1,112 @@
+import time
+import math
+import datetime
+from uuid import uuid4
+import pandas as pd
+from threading import Thread
+
+
+class ErrorLogs:
+    """ Register Python Error Logs """
+
+    def __init__(self, project_name, pipeline, ip_address, request_url, sql_con, timeout=30*60, print_sql=False):
+        """ Initialize the class
+        :param project_name: Name of the project being run. it must be already declared in PythonEmailProjectSeverity
+        :param pipeline: Pipeline name being run. It must identify the process being executed uniquely
+        :param ip_address: IP Address
+        :param request_url: URL requested by the client
+        :param sql_con: Connection to the Database to upload the Logs
+        :param timeout: Time in seconds after which an unsuccessful log will be sent
+        :param print_sql: Print the SQL statement sent to the server
+        """
+        self.log_df = pd.DataFrame({'ProjectName': [project_name], 'Successful': [0], 'Sent': [0],
+                                    'IPAddress': [str(ip_address).replace("'", '"')],
+                                    'RequestUrl': [str(request_url).replace("'", '"')],
+                                    'StartedDate': [datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")]})
+        self.sql_con = sql_con
+        self.timeout = timeout
+        self.print_sql = print_sql
+        self.guid = str(uuid4())
+        self.pipeline = pipeline
+        self.failure_type = []
+        Thread(target=self.start_threading).start()
+
+    def start_threading(self):
+        """ Start a threading to update on failure if the script breaks or the pipeline gets blocked
+        """
+        time_range = math.ceil(self.timeout / 10)
+        for times in range(time_range):
+            time.sleep(10)
+            if len(self.failure_type) > 0:
+                break
+
+        if len(self.failure_type) == 0:
+            elapsed_time = str(datetime.timedelta(seconds=round(self.timeout)))[2:]
+            self.on_failure(error_message=f'The pipeline failed to succeed after running '
+                                          f'for {elapsed_time} minutes')
+
+    def on_success(self, pipeline=None):
+        """ Update log on success
+        :param pipeline: Pipeline name being run. It must identify the process being executed uniquely
+        """
+        if not any(self.failure_type):
+            if pipeline is not None:
+                self.pipeline = pipeline
+            successful_columns = {'Successful': 1, 'Resolved': 1, 'Pipeline': self.pipeline, 'GuidKey': self.guid,
+                                  'FinishedDate': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}
+            self.log_df = self.log_df.assign(**successful_columns)
+            self.sql_con.insert(self.log_df, 'Staging', 'Logs', print_sql=self.print_sql)
+            self.failure_type.append(True)
+
+            Thread(target=self.trigger_merge_stored_procedure).start()
+
+    def on_failure(self, error_message, pipeline=None, section=None, critical=True, proposed_solution=None):
+        """ Update log on failure
+        :param error_message: Error message to be sent in the Log
+        :param pipeline: Pipeline name being run. It must identify the process being executed uniquely
+        :param section: Indicate the script section. Useful to locate the error
+        :param critical: Indicate whether it should avoid sending successful logs
+        :param proposed_solution: Proposed solution to the error message
+        """
+        save_failure = True
+        if section is not None:
+            if ((('prod' not in str(self.sql_con.server)) | ('prd' not in str(self.sql_con.server)))
+                    & (('connection' in str(section).lower()) & ('dw' in str(section).lower()))):
+                print('Avoiding to report a connection DW error in a non-production environment')
+                save_failure = False
+
+        if save_failure:
+            if pipeline is not None:
+                self.pipeline = pipeline
+            unsuccessful_columns = {'Successful': 0, 'Section': section, 'Pipeline': self.pipeline,
+                                    'GuidKey': self.guid, 'Critical': 1 if critical is True else 0,
+                                    'FinishedDate': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
+                                    'ErrorMessage': str(error_message).replace("'", '"')}
+            self.log_df = self.log_df.assign(**unsuccessful_columns)
+            if proposed_solution is not None:
+                self.log_df = self.log_df.assign(**{'ProposedSolution': proposed_solution})
+            self.sql_con.insert(self.log_df, 'Staging', 'Logs', print_sql=self.print_sql)
+            self.failure_type.append(critical)
+
+            Thread(target=self.trigger_merge_stored_procedure).start()
+
+    def trigger_merge_stored_procedure(self):
+        """ Trigger the merge stored procedure to aggregate logs
+        """
+        stored_procedure = 'Logging.spLogsMerge'
+        self.sql_con.run_statement(f"EXEC {stored_procedure}")
+
+    @staticmethod
+    def register_failure(email_log, section, error_, solutions=None):
+        """Register a failure in the email log
+        :param email_log: Instance of PythonEmailProjectSeverity
+        :param section: Indicate the script section. Useful to locate the error
+        :param error_: Error message to be sent in the Log
+        :param solutions: List of solutions for the error messages
+        """
+        solution_header = list(filter(lambda x: section.startswith(x), list(solutions.keys())))
+        if len(solution_header) > 0:
+            proposed_solution = solutions[solution_header[0]]
+            email_log.on_failure(error_, section=section, proposed_solution=proposed_solution)
+        else:
+            email_log.on_failure(error_, section=section)

{berryworld-1.0.0.197723.dist-info → berryworld-1.0.0.199021.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: berryworld
-Version: 1.0.0.197723
+Version: 1.0.0.199021
 Summary: Handy classes to improve ETL processes
 Home-page: https://www.berryworld.com
 Author: BerryWorld ltd

{berryworld-1.0.0.197723.dist-info → berryworld-1.0.0.199021.dist-info}/RECORD

@@ -1,4 +1,4 @@
-berryworld/__init__.py,sha256=dDnmPuE2CZfPzh08BfjJjmyeM0mjs5mIP0OS7RujDco,1140
+berryworld/__init__.py,sha256=xYQwJeCZ2di85E0_O3Sx8AUsRhoT6USUxcPGJyCr-gY,1175
 berryworld/aks_logs.py,sha256=Gb2_cokiZbEX01Yoptd0MxpDociaug-GrXdwliyxFBo,2293
 berryworld/allocation_solver.py,sha256=asFtaCAze6-eHUGWXA0kAp67UBS-Upj1KKdrVLj_ttQ,8513
 berryworld/app_logs.py,sha256=MKzKPYd3JuPfOQNAapIgaeZeFHw1z_w2mbn9I6QCADE,4180
@@ -10,12 +10,13 @@ berryworld/email_con.py,sha256=CVyEDVnc_iVeRTwJgIU31ToFgyix7L2yn4Ih9Wu7I5s,15806
 berryworld/email_logging.py,sha256=LeSrTExhQhar49gJR2wGC1dS0lqsNpFl9pS3eYWqnuo,4936
 berryworld/generate_env.py,sha256=LrqUH8AjCI6P0uU6BMBRYC9cnmyKkYcpXF1KKIzxkZ8,15900
 berryworld/handy_mix.py,sha256=SqJ3UjIjmIOgjbp-_1eyKQJ3OJLN7OfpDQIASvjNZ_Q,9672
-berryworld/logging.py,sha256=tOxzaFALQm3aVVEc3I7t8MU6PwgUI9VSnzNCH9yRryo,1013
+berryworld/logging.py,sha256=mFFueg1OKudoYFDVJIzCYNY7lRipoefllJiHzytkmzk,1010
 berryworld/logic_apps.py,sha256=a0uU4tNO3v2w7grdBv-OOx4hUf7VBIerJpwZ9U-29dQ,14591
 berryworld/microsoft_teams.py,sha256=8uPo0yku-euBj2VdzBoZCeX3IcsCCOqISLqaVZUVxfA,16030
 berryworld/persistent_storage.py,sha256=L15kLyzN42T6UB1WAg8rFXJq3Mdb1M8Sw4P5YQaUN84,8711
 berryworld/pickle_management.py,sha256=5o6UuXBpTj23Jgpz6sj9V-vdcdWBK1xMEckWxT-Whj4,2436
 berryworld/power_automate.py,sha256=V86QEGG9H36DrDvod9Q6yp8OUu307hfYcXJhw06pYrA,27912
+berryworld/python_logs.py,sha256=SEWrzK5kpG6kd_7qjGXd-d5SiVgEm595Tuud14uGGzw,5810
 berryworld/sharepoint_con.py,sha256=nmyZJIcaAKJ6Y-ti4gQbvzA_rRbrMGIxTDXe4eP-tiI,44950
 berryworld/snowflake_conn.py,sha256=L0ePgTKa3tvaxj88BZmsjS6cFp3ZU3rytw7S2jkuA-U,3161
 berryworld/sql_conn.py,sha256=6kUR3oLXweakz2IBc4zI1ZMqRoN8K6nbQneHhenM-7I,47668
@@ -26,12 +27,12 @@ berryworld/verify_keys.py,sha256=X7VUHqYDklWPDO0bGVHIOXeLq5Qi4fZRZbHYw5x7UnA,435
 berryworld/vivantio.py,sha256=QfZo0UKqkzVRg_LyiwivNd3aEup4TH57x4KxLZkCJwc,10627
 berryworld/vivantio_logging.py,sha256=ciy7gA4u3FrgUIpEBnMgocbNPp6jcu9TPoy-kLcrTZU,5736
 berryworld/xml_parser.py,sha256=HWD71NaTN3DaIOGT6Wzxs4CEsroFhGQwe9iPLIL80Co,957
-berryworld-1.0.0.197723.dist-info/licenses/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
+berryworld-1.0.0.199021.dist-info/licenses/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_allocation_config.py,sha256=e12l6fE9U57eSPS35g6ekJ_hol7-RHg89JV60_m1BlE,4633
 tests/test_handy_mix_config.py,sha256=Un56mz9KJmdn4K4OwzHAHLSRzDU1Xv2nFrONNuzOG04,2594
 tests/test_xml_parser.py,sha256=3QTlhFEd6KbK6nRFKZnc35tad6wqukTbe4QrFi8mr_8,859
-berryworld-1.0.0.197723.dist-info/METADATA,sha256=Btf-waInlfkM3ovW6JgnQaM2Nytj0gy0xbMDY_ppEG8,1445
-berryworld-1.0.0.197723.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-berryworld-1.0.0.197723.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
-berryworld-1.0.0.197723.dist-info/RECORD,,
+berryworld-1.0.0.199021.dist-info/METADATA,sha256=RVDaYgMC_ojVtgoJQL2uXuIA8TsqPxLwPpgntPyeFQM,1445
+berryworld-1.0.0.199021.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+berryworld-1.0.0.199021.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
+berryworld-1.0.0.199021.dist-info/RECORD,,