berryworld 1.0.0.197234__py3-none-any.whl → 1.0.0.199020__py3-none-any.whl

berryworld/__init__.py

@@ -4,6 +4,7 @@ from .transportation_solver import TransportationAlgorithm
 from .allocation_solver import AllocationSolver
 from .pickle_management import PickleManagement
 from .email_logging import EmailLogging
+from .python_logs import ErrorLogs
 from .verify_keys import Verify
 from .credentials import SnowflakeCredentials, SQLCredentials, WebServiceCredentials, MicrosoftTeamsCredentials
 from .persistent_storage import PersistentStorage

berryworld/logging.py

@@ -1,6 +1,3 @@
-
-
-
 class PythonLogs:
     """ Register the Python Logs """
     def __init__(self, conn, batch_process):

berryworld/python_logs.py

@@ -0,0 +1,112 @@
+import time
+import math
+import datetime
+from uuid import uuid4
+import pandas as pd
+from threading import Thread
+
+
+class ErrorLogs:
+    """ Register Python Error Logs """
+
+    def __init__(self, project_name, pipeline, ip_address, request_url, sql_con, timeout=30*60, print_sql=False):
+        """ Initialize the class
+        :param project_name: Name of the project being run. it must be already declared in PythonEmailProjectSeverity
+        :param pipeline: Pipeline name being run. It must identify the process being executed uniquely
+        :param ip_address: IP Address
+        :param request_url: URL requested by the client
+        :param sql_con: Connection to the Database to upload the Logs
+        :param timeout: Time in seconds after which an unsuccessful log will be sent
+        :param print_sql: Print the SQL statement sent to the server
+        """
+        self.log_df = pd.DataFrame({'ProjectName': [project_name], 'Successful': [0], 'Sent': [0],
+                                    'IPAddress': [str(ip_address).replace("'", '"')],
+                                    'RequestUrl': [str(request_url).replace("'", '"')],
+                                    'StartedDate': [datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")]})
+        self.sql_con = sql_con
+        self.timeout = timeout
+        self.print_sql = print_sql
+        self.guid = str(uuid4())
+        self.pipeline = pipeline
+        self.failure_type = []
+        Thread(target=self.start_threading).start()
+
+    def start_threading(self):
+        """ Start a threading to update on failure if the script breaks or the pipeline gets blocked
+        """
+        time_range = math.ceil(self.timeout / 10)
+        for times in range(time_range):
+            time.sleep(10)
+            if len(self.failure_type) > 0:
+                break
+
+        if len(self.failure_type) == 0:
+            elapsed_time = str(datetime.timedelta(seconds=round(self.timeout)))[2:]
+            self.on_failure(error_message=f'The pipeline failed to succeed after running '
+                                          f'for {elapsed_time} minutes')
+
+    def on_success(self, pipeline=None):
+        """ Update log on success
+        :param pipeline: Pipeline name being run. It must identify the process being executed uniquely
+        """
+        if not any(self.failure_type):
+            if pipeline is not None:
+                self.pipeline = pipeline
+            successful_columns = {'Successful': 1, 'Resolved': 1, 'Pipeline': self.pipeline, 'GuidKey': self.guid,
+                                  'FinishedDate': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")}
+            self.log_df = self.log_df.assign(**successful_columns)
+            self.sql_con.insert(self.log_df, 'Staging', 'Logs', print_sql=self.print_sql)
+            self.failure_type.append(True)
+
+            Thread(target=self.trigger_merge_stored_procedure).start()
+
+    def on_failure(self, error_message, pipeline=None, section=None, critical=True, proposed_solution=None):
+        """ Update log on failure
+        :param error_message: Error message to be sent in the Log
+        :param pipeline: Pipeline name being run. It must identify the process being executed uniquely
+        :param section: Indicate the script section. Useful to locate the error
+        :param critical: Indicate whether it should avoid sending successful logs
+        :param proposed_solution: Proposed solution to the error message
+        """
+        save_failure = True
+        if section is not None:
+            if ((('prod' not in str(self.sql_con.server)) | ('prd' not in str(self.sql_con.server)))
+                    & (('connection' in str(section).lower()) & ('dw' in str(section).lower()))):
+                print('Avoiding to report a connection DW error in a non-production environment')
+                save_failure = False
+
+        if save_failure:
+            if pipeline is not None:
+                self.pipeline = pipeline
+            unsuccessful_columns = {'Successful': 0, 'Section': section, 'Pipeline': self.pipeline,
+                                    'GuidKey': self.guid, 'Critical': 1 if critical is True else 0,
+                                    'FinishedDate': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
+                                    'ErrorMessage': str(error_message).replace("'", '"')}
+            self.log_df = self.log_df.assign(**unsuccessful_columns)
+            if proposed_solution is not None:
+                self.log_df = self.log_df.assign(**{'ProposedSolution': proposed_solution})
+            self.sql_con.insert(self.log_df, 'Staging', 'Logs', print_sql=self.print_sql)
+            self.failure_type.append(critical)
+
+            Thread(target=self.trigger_merge_stored_procedure).start()
+
+    def trigger_merge_stored_procedure(self):
+        """ Trigger the merge stored procedure to aggregate logs
+        """
+        stored_procedure = 'Logging.spLogsMerge'
+        self.sql_con.execute(f"EXEC {stored_procedure}")
+
+    @staticmethod
+    def register_failure(email_log, section, error_, solutions=None):
+        """Register a failure in the email log
+        :param email_log: Instance of PythonEmailProjectSeverity
+        :param section: Indicate the script section. Useful to locate the error
+        :param error_: Error message to be sent in the Log
+        :param solutions: List of solutions for the error messages
+        """
+        solution_header = list(filter(lambda x: section.startswith(x), list(solutions.keys())))
+        if len(solution_header) > 0:
+            proposed_solution = solutions[solution_header[0]]
+            email_log.on_failure(error_, section=section, proposed_solution=proposed_solution)
+        else:
+            email_log.on_failure(error_, section=section)

berryworld/sql_connenction.py

@@ -2,7 +2,6 @@ import os
 import re
 import ast
 import math
-import time
 import pyodbc
 import traceback
 import numpy as np
@@ -10,7 +9,6 @@ import pandas as pd
 import sqlalchemy as sa
 from urllib import parse
 from numbers import Number
-from threading import Thread
 from sqlalchemy.pool import QueuePool
 
 
@@ -1029,7 +1027,7 @@ class SQLPoolEngine:
     """ Connect a Pool Engine to a Microsoft SQL """
 
     def __init__(self, db_reference, server, master=False, trusted_certificate=True, encrypt=True, multi_db=False,
-                 commit_as_transaction=True, pool_size=10, max_overflow=10, pool_timeout=30, timeout=300):
+                 commit_as_transaction=True, pool_size=10, max_overflow=10, pool_timeout=30):
         """ Initialize the class
         It requires the
         SQL-DBREFERENCE-PROD = 'server_name db_name user password'
@@ -1049,7 +1047,6 @@ class SQLPoolEngine:
         :param pool_size: Number of connections to keep in the pool
         :param max_overflow: Extra connections beyond pool_size
         :param pool_timeout: Timeout for getting a connection
-        :param timeout: Connection timeout in seconds
         """
         self.con_string_read = None
         self.con_string_write = None
@@ -1060,7 +1057,6 @@ class SQLPoolEngine:
         self.pool_size = pool_size
         self.max_overflow = max_overflow
         self.pool_timeout = pool_timeout
-        self.timeout = timeout
 
         self.db_reference = db_reference.replace("_", "") if "_" in db_reference else db_reference
         self.server = server
@@ -1103,9 +1099,6 @@ class SQLPoolEngine:
 
         self.create_write_engine(commit_as_transaction=self.commit_as_transaction)
 
-        # Dispose the engine after a certain timeout
-        Thread(target=self.dispose_engine, args=(self.timeout,)).start()
-
     def credentials(self):
         """ Return the credentials used to connect to the SQL Server
         :return: Dictionary with the credentials used to connect to the SQL Server
@@ -1160,8 +1153,7 @@ class SQLPoolEngine:
                                             poolclass=QueuePool,
                                             pool_size=self.pool_size,           # Number of connections to keep in the pool
                                             max_overflow=self.max_overflow,     # Extra connections beyond pool_size
-                                            pool_timeout=self.pool_timeout,     # Timeout for getting a connection
-                                            pool_recycle=self.timeout           # Recycle connections after X minutes
+                                            pool_timeout=self.pool_timeout      # Timeout for getting a connection
                                             )
 
         if not commit_as_transaction:
@@ -1182,8 +1174,7 @@ class SQLPoolEngine:
                                              poolclass=QueuePool,
                                              pool_size=10,               # Number of connections to keep in the pool
                                              max_overflow=10,            # Extra connections beyond pool_size
-                                             pool_timeout=30,            # Timeout for getting a connection
-                                             pool_recycle=self.timeout   # Recycle connections after X minutes
+                                             pool_timeout=30             # Timeout for getting a connection
                                              )
 
         if not commit_as_transaction:
@@ -1197,13 +1188,10 @@ class SQLPoolEngine:
         self.create_read_engine(commit_as_transaction=commit_as_transaction)
         self.con = self.engine_read.connect().connection
 
-    def dispose_engine(self, timeout=0):
+    def dispose_engine(self):
         """ Dispose any opened engines with the Server
         :return: None
         """
-        if timeout > 0:
-            time.sleep(timeout)
-
         if self.engine_read:
             self.engine_read.dispose()
 
@@ -1255,26 +1243,16 @@ class SQLConnectionPool:
         self.commit_as_transaction = pool_class.commit_as_transaction
         self.db_name = pool_class.db_name
         self.server = pool_class.server
-        self.timeout = pool_class.timeout
-
-        Thread(target=self.close_connection, args=(self.timeout,)).start()
 
-    def close_connection(self, timeout=0):
+    def close_connection(self):
         """ Close any opened connections with the Server
         :return: None
         """
-        if timeout > 0:
-            time.sleep(timeout)
-
         if self.con_read is not None:
             self.con_read.close()
-            if self.engine_read:
-                self.engine_read.dispose()
 
         if self.con_write is not None:
             self.con_write.close()
-            if self.engine_write:
-                self.engine_write.dispose()
 
     def query(self, sql_query, coerce_float=False):
         """ Read data from SQL according to the sql_query

{berryworld-1.0.0.197234.dist-info → berryworld-1.0.0.199020.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: berryworld
-Version: 1.0.0.197234
+Version: 1.0.0.199020
 Summary: Handy classes to improve ETL processes
 Home-page: https://www.berryworld.com
 Author: BerryWorld ltd

{berryworld-1.0.0.197234.dist-info → berryworld-1.0.0.199020.dist-info}/RECORD

@@ -1,4 +1,4 @@
-berryworld/__init__.py,sha256=dDnmPuE2CZfPzh08BfjJjmyeM0mjs5mIP0OS7RujDco,1140
+berryworld/__init__.py,sha256=xYQwJeCZ2di85E0_O3Sx8AUsRhoT6USUxcPGJyCr-gY,1175
 berryworld/aks_logs.py,sha256=Gb2_cokiZbEX01Yoptd0MxpDociaug-GrXdwliyxFBo,2293
 berryworld/allocation_solver.py,sha256=asFtaCAze6-eHUGWXA0kAp67UBS-Upj1KKdrVLj_ttQ,8513
 berryworld/app_logs.py,sha256=MKzKPYd3JuPfOQNAapIgaeZeFHw1z_w2mbn9I6QCADE,4180
@@ -10,28 +10,29 @@ berryworld/email_con.py,sha256=CVyEDVnc_iVeRTwJgIU31ToFgyix7L2yn4Ih9Wu7I5s,15806
 berryworld/email_logging.py,sha256=LeSrTExhQhar49gJR2wGC1dS0lqsNpFl9pS3eYWqnuo,4936
 berryworld/generate_env.py,sha256=LrqUH8AjCI6P0uU6BMBRYC9cnmyKkYcpXF1KKIzxkZ8,15900
 berryworld/handy_mix.py,sha256=SqJ3UjIjmIOgjbp-_1eyKQJ3OJLN7OfpDQIASvjNZ_Q,9672
-berryworld/logging.py,sha256=tOxzaFALQm3aVVEc3I7t8MU6PwgUI9VSnzNCH9yRryo,1013
+berryworld/logging.py,sha256=mFFueg1OKudoYFDVJIzCYNY7lRipoefllJiHzytkmzk,1010
 berryworld/logic_apps.py,sha256=a0uU4tNO3v2w7grdBv-OOx4hUf7VBIerJpwZ9U-29dQ,14591
 berryworld/microsoft_teams.py,sha256=8uPo0yku-euBj2VdzBoZCeX3IcsCCOqISLqaVZUVxfA,16030
 berryworld/persistent_storage.py,sha256=L15kLyzN42T6UB1WAg8rFXJq3Mdb1M8Sw4P5YQaUN84,8711
 berryworld/pickle_management.py,sha256=5o6UuXBpTj23Jgpz6sj9V-vdcdWBK1xMEckWxT-Whj4,2436
 berryworld/power_automate.py,sha256=V86QEGG9H36DrDvod9Q6yp8OUu307hfYcXJhw06pYrA,27912
+berryworld/python_logs.py,sha256=mzRSO4sywKrr_4KC5_WetRs4XJ9uiD1OaUPOGwsEib0,5804
 berryworld/sharepoint_con.py,sha256=nmyZJIcaAKJ6Y-ti4gQbvzA_rRbrMGIxTDXe4eP-tiI,44950
 berryworld/snowflake_conn.py,sha256=L0ePgTKa3tvaxj88BZmsjS6cFp3ZU3rytw7S2jkuA-U,3161
 berryworld/sql_conn.py,sha256=6kUR3oLXweakz2IBc4zI1ZMqRoN8K6nbQneHhenM-7I,47668
-berryworld/sql_connenction.py,sha256=jqXaYZ204HdaIwaQjBlKvrAiJIj6QS_FASb-RYEkT-g,105053
+berryworld/sql_connenction.py,sha256=IJAOsQ7p0tMFEEucazmmkS3lNonf_YFWftyZ_kGPHj8,104150
 berryworld/teams_logging.py,sha256=8NwXyWr4fLj7W6GzAm2nRQCGFDxibQpAHDHHD24FrP8,6997
 berryworld/transportation_solver.py,sha256=tNc1JJk71azIBccdWVHbqcvXWhalOdKffv6HmBD6tG0,5014
 berryworld/verify_keys.py,sha256=X7VUHqYDklWPDO0bGVHIOXeLq5Qi4fZRZbHYw5x7UnA,4356
 berryworld/vivantio.py,sha256=QfZo0UKqkzVRg_LyiwivNd3aEup4TH57x4KxLZkCJwc,10627
 berryworld/vivantio_logging.py,sha256=ciy7gA4u3FrgUIpEBnMgocbNPp6jcu9TPoy-kLcrTZU,5736
 berryworld/xml_parser.py,sha256=HWD71NaTN3DaIOGT6Wzxs4CEsroFhGQwe9iPLIL80Co,957
-berryworld-1.0.0.197234.dist-info/licenses/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
+berryworld-1.0.0.199020.dist-info/licenses/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_allocation_config.py,sha256=e12l6fE9U57eSPS35g6ekJ_hol7-RHg89JV60_m1BlE,4633
 tests/test_handy_mix_config.py,sha256=Un56mz9KJmdn4K4OwzHAHLSRzDU1Xv2nFrONNuzOG04,2594
 tests/test_xml_parser.py,sha256=3QTlhFEd6KbK6nRFKZnc35tad6wqukTbe4QrFi8mr_8,859
-berryworld-1.0.0.197234.dist-info/METADATA,sha256=uTCAY4_FLQRCSQZ-WQDjJlCD7bqH11YSbVRKZltYhRA,1445
-berryworld-1.0.0.197234.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-berryworld-1.0.0.197234.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
-berryworld-1.0.0.197234.dist-info/RECORD,,
+berryworld-1.0.0.199020.dist-info/METADATA,sha256=yk85pOBoxVb2NakviW8KMRrMNGzDmvE_-YHnYFnvy7E,1445
+berryworld-1.0.0.199020.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+berryworld-1.0.0.199020.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
+berryworld-1.0.0.199020.dist-info/RECORD,,