berryworld 1.0.0.179097__py3-none-any.whl → 1.0.0.180127__py3-none-any.whl

berryworld/cache_data.py

@@ -50,7 +50,7 @@ class CacheManagement:
     def dump_pickle(df, filepath):
         """ Save the data to a filepath
         :param df: DataFrame gathering the data
-        :param path: Path and filename to the file destination
+        :param filepath: Path and filename to the file destination
         """
         pickle_out = open(filepath, 'wb')
         pickle.dump(df, pickle_out)
@@ -59,7 +59,7 @@ class CacheManagement:
     @staticmethod
     def retrieve_pickle(filepath):
         """ Get the data from the indicated filepath
-        :param path: Path and filename to the file destination
+        :param filepath: Path and filename to the file destination
         """
         pickle_in = open(filepath, "rb")
         data_df = pickle.load(pickle_in)
@@ -69,7 +69,7 @@ class CacheManagement:
     @staticmethod
     def remove_pickle(filepath):
         """ Delete a file allocated in filepath
-        :param path: Path and filename to the file destination
+        :param filepath: Path and filename to the file destination
         """
         if os.path.exists(filepath):
             os.remove(filepath)

berryworld/credentials.py

@@ -209,15 +209,18 @@ class WebServiceCredentials:
 
             try:
                 user_name = os.environ.get(f"WEBSERVICE_USER_{self.service.upper()}")
-            except Exception:
+            except Exception as e:
+                print(e)
                 user_name = ''
             try:
                 password = os.environ.get(f"WEBSERVICE_PASSWORD_{self.service.upper()}")
-            except Exception:
+            except Exception as e:
+                print(e)
                 password = ''
             try:
                 access_token = os.environ.get(f"WEBSERVICE_ACCESS_TOKEN_{self.service.upper()}")
-            except Exception:
+            except Exception as e:
+                print(e)
                 access_token = ''
 
             return {'user_name': user_name,
@@ -250,3 +253,23 @@ class MicrosoftTeamsCredentials:
 
         except ValueError as e:
             raise ValueError("Variable %s not found" % str(e))
+
+
+class SnowflakeCredentials:
+    def __init__(self, db_name):
+        if db_name is None:
+            raise ValueError("Please provide a value for db_name")
+        self.db_name = db_name
+
+    def simple_creds(self):
+        try:
+            account = os.environ.get("SNOWFLAKE_" + self.db_name.upper() + '_ACCOUNT')
+            user_name = os.environ.get("SNOWFLAKE_" + self.db_name.upper() + '_USERNAME')
+            password = os.environ.get("SNOWFLAKE_" + self.db_name.upper() + '_PASSWORD')
+
+            return {
+                'account': account,
+                'user_name': user_name,
+                'password': password}
+        except ValueError as e:
+            raise ValueError("Variable %s not found" % str(e))

berryworld/email_logging.py

@@ -59,16 +59,24 @@ class EmailLogging:
         :param critical: Indicate whether it should avoid sending successful logs
         :param proposed_solution: Proposed solution to the error message
         """
-        unsuccessful_columns = {'Successful': 0,
-                                'FinishedDate': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
-                                'Section': section,
-                                'ErrorMessage': str(error_message).replace("'", '"'),
-                                'Critical': 1 if critical is True else 0}
-        self.log_df = self.log_df.assign(**unsuccessful_columns)
-        if proposed_solution is not None:
-            self.log_df = self.log_df.assign(**{'ProposedSolution': proposed_solution})
-        self.sql_con.insert(self.log_df, 'Logging', 'PythonLogs', print_sql=self.print_sql)
-        self.failure_type.append(critical)
+        save_failure = True
+        if section is not None:
+            if (('prod' not in str(self.sql_con.server)) | ('prd' not in str(self.sql_con.server))) \
+                    & (('connection' in str(section).lower()) & ('dw' in str(section).lower())):
+                print('Avoiding to report a connection DW error in a non-production environment')
+                save_failure = False
+
+        if save_failure:
+            unsuccessful_columns = {'Successful': 0,
+                                    'FinishedDate': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
+                                    'Section': section,
+                                    'ErrorMessage': str(error_message).replace("'", '"'),
+                                    'Critical': 1 if critical is True else 0}
+            self.log_df = self.log_df.assign(**unsuccessful_columns)
+            if proposed_solution is not None:
+                self.log_df = self.log_df.assign(**{'ProposedSolution': proposed_solution})
+            self.sql_con.insert(self.log_df, 'Logging', 'PythonLogs', print_sql=self.print_sql)
+            self.failure_type.append(critical)
 
     @staticmethod
     def register_failure(email_log, section, error_, solutions=None):

berryworld/handy_mix.py

@@ -65,7 +65,21 @@ class HandyMix:
         return result
 
     @staticmethod
-    def likely_match(srs, match_str, n_res=1, threshold=False):
+    def grab_n_elem(n, comp_):
+        max_list = heapq.nlargest(n, comp_)
+        like_matches = np.array([elem in max_list for elem in comp_]).sum()
+        if like_matches > n:
+            list_ = [elem in max_list for elem in comp_]
+            k = 0
+            for elem in list_:
+                if (elem is True) & (k > n - 1):
+                    list_[k] = False
+                k = + 1
+            return list_
+        else:
+            return [elem in max_list for elem in comp_]
+
+    def likely_match(self, srs, match_str, n_res=1, threshold=False):
         """ Look for the more similar element in a Series to the one provided in match_str. If there is a draw,
         It will pick the first matching elements. Threshold only consider the elements if their likelihood is
         greater than that threshold variable
@@ -88,29 +102,15 @@ class HandyMix:
             aux = SequenceMatcher(None, element, match_str)
             comp.append(aux.ratio())
 
-        def grab_n_elem(n, comp):
-            max_list = heapq.nlargest(n, comp)
-            like_matches = np.array([elem in max_list for elem in comp]).sum()
-            if like_matches > n:
-                list_ = [elem in max_list for elem in comp]
-                k = 0
-                for elem in list_:
-                    if (elem is True) & (k > n - 1):
-                        list_[k] = False
-                    k = + 1
-                return list_
-            else:
-                return [elem in max_list for elem in comp]
-
         if threshold:
             # Return n elements above the threshold
             if any(elem >= threshold for elem in comp):
-                return grab_n_elem(n_res, comp)
+                return self.grab_n_elem(n_res, comp)
             else:
                 return [False] * comp.__len__()
         else:
             # Return n elements likely to match
-            return grab_n_elem(n_res, comp)
+            return self.grab_n_elem(n_res, comp)
 
     @staticmethod
     def remove_line_chars(input_text, remove_duplicate_white_spaces=False):

berryworld/microsoft_teams.py

@@ -254,11 +254,11 @@ class MicrosoftTeams:
             vivantio_ticket_id = vivantio_ticket['Id'].values[0]
             display_id = vivantio_ticket['DisplayId'].values[0]
             vivantio_ticket_url = f'https://poupart.flex.vivantio.com/item/Ticket/{vivantio_ticket_id}'
-            html_message = html_message + f'<br><b>Vivantio Ticket Id:</b> {display_id}' + \
-                           f'<br><b>Vivantio Ticket URL:</b> {vivantio_ticket_url}<br>'
+            html_message = html_message + f'<br><b>Vivantio Ticket Id:</b> {display_id}' \
+                                          f'<br><b>Vivantio Ticket URL:</b> {vivantio_ticket_url}<br>'
 
-        html_message = html_message + '<br><b>Error Status:</b> ' + status + f'<br><b>Error Count:</b> {count}' + \
-                       '<br><b>Error Content:</b> <br>' + message
+        html_message = html_message + '<br><b>Error Status:</b> ' + status + f'<br><b>Error Count:</b> {count}' \
+                                                                             f'<br><b>Error Content:</b> <br>' + message
 
         payload = {}
         if importance:

berryworld/snowflake_conn.py

@@ -0,0 +1,77 @@
+import pandas as pd
+import snowflake.connector
+from credentials import SnowflakeCredentials
+
+pd.set_option('future.no_silent_downcasting', True)
+
+
+class SnowflakeConn:
+    def __init__(self, server_creds=None, **kwargs):
+        """ Initialize the class
+        -----------------------------
+        server_creds = {
+                        "account": "",
+                        "db_name": "",
+                        "user_name": "",
+                        "password": ""
+                        }
+
+        -----------------------------
+        :param server_creds: Dictionary containing the info to connect to the Server
+        :param kwargs: Additional parameters to be passed to the connection
+        """
+
+        self.conn_str = None
+        self.conn = None
+
+        if kwargs != {}:
+            try:
+                db = kwargs['db_name']
+                server_creds = SnowflakeCredentials(db).simple_creds()
+            except KeyError:
+                raise KeyError('Please provide a valid db_name and server_type')
+
+        self.db = kwargs['db_name']
+        self.user = server_creds['user_name']
+        self.pw = server_creds['password']
+        self.account = server_creds['account']
+
+        if self.user and self.pw and self.account:
+            self.connect()
+
+    def connect(self):
+        """ Open the connection to Snowflake """
+        self.conn = snowflake.connector.connect(
+            user=self.user,
+            password=self.pw,
+            account=self.account,
+            database=self.db)
+
+    def close(self):
+        """ Close the connection to Snowflake """
+        self.conn.close()
+
+    def query(self, sql_query):
+        """ Read data from Snowflake according to the sql_query
+        -----------------------------
+        query_str = "SELECT * FROM %s" & table
+        con_.query(query_str)
+        -----------------------------
+        :param sql_query: Query to be sent to Snowflake
+        :return: DataFrame gathering the requested data
+        """
+        cursor = None
+        self.connect()
+        try:
+            cursor = self.conn.cursor()
+            cursor.execute(sql_query)
+            rows = cursor.fetchall()
+            col_names = [desc[0] for desc in cursor.description]
+            result = pd.DataFrame(rows, columns=col_names)
+            return result
+        except Exception as e:
+            raise Exception(e)
+        finally:
+            if cursor:
+                cursor.close()
+            self.close()

berryworld/sql_conn.py

@@ -942,9 +942,11 @@ class SQLConn:
                     # Avoid converting datetime values
                     try:
                         pd.to_datetime(data[col])
-                    except:
+                    except Exception as e:
+                        print(e)
                         data[col] = data[col].str.replace('.0', '')
-            except:
+            except Exception as e:
+                print(e)
                 continue
 
         return data

berryworld/transportation_solver.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from scipy.optimize import linprog
+from scipy.optimize import linprog, _highs
 
 
 class TransportationAlgorithm:
@@ -71,13 +71,13 @@ class TransportationAlgorithm:
 
         # Run the Simplex method
         method = 'Interior-Point without constraints'
-        res = linprog(c, A_eq=a_matrix, b_eq=b, options=options, method='interior-point')
+        res = linprog(c, A_eq=a_matrix, b_eq=b, options=options, method='highs-ipm')
         res_success = res.success
 
         # Run the Interior-Point method
         if res_success is False:
             method = 'Interior-Point without constraints'
-            res = linprog(c, A_eq=a_matrix, b_eq=b, options=options, method='revised simplex')
+            res = linprog(c, A_eq=a_matrix, b_eq=b, options=options, method='highs-ds')
             res_success = res.success
 
         # Run the interior-point method with constraints
@@ -89,7 +89,7 @@ class TransportationAlgorithm:
             max_value = max(prod[0], sales[0])
             b_ub = [max_value] + [max_value]
             res = linprog(c, A_eq=a_eq, b_eq=b_eq, A_ub=a_ub, b_ub=b_ub,
-                          options=options, method='interior-point')
+                          options=options, method='highs-ipm')
             res_success = res.success
 
         # Run the Revised Simplex method with constraints
@@ -101,7 +101,7 @@ class TransportationAlgorithm:
             max_value = max(prod[0], sales[0])
             b_ub = [max_value] + [max_value]
             res = linprog(c, A_eq=a_eq, b_eq=b_eq, A_ub=a_ub, b_ub=b_ub,
-                          options=options, method='revised simplex')
+                          options=options, method='highs-ds')
             res_success = res.success
 
         result = self.ranking

{berryworld-1.0.0.179097.dist-info → berryworld-1.0.0.180127.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: berryworld
-Version: 1.0.0.179097
+Version: 1.0.0.180127
 Summary: Handy classes to improve ETL processes
 Home-page: https://www.berryworld.com
 Author: BerryWorld ltd

{berryworld-1.0.0.179097.dist-info → berryworld-1.0.0.180127.dist-info}/RECORD

@@ -3,23 +3,24 @@ berryworld/aks_logs.py,sha256=Gb2_cokiZbEX01Yoptd0MxpDociaug-GrXdwliyxFBo,2293
 berryworld/allocation_solver.py,sha256=asFtaCAze6-eHUGWXA0kAp67UBS-Upj1KKdrVLj_ttQ,8513
 berryworld/app_logs.py,sha256=MKzKPYd3JuPfOQNAapIgaeZeFHw1z_w2mbn9I6QCADE,4180
 berryworld/app_logs_query.py,sha256=U94b-z3X9cuY_KFozupUcfaYciXWBn7p_RHkoRsfROU,4376
-berryworld/cache_data.py,sha256=2cStWbFQHimon_lHMbcM_0vU7lt-FCge96D-T9YXaxQ,2242
-berryworld/credentials.py,sha256=Knxo4gssLT7sbaBjOTFe3mX5k70G2e0M_6CdtlddjtA,10200
+berryworld/cache_data.py,sha256=AWAhV4PPkilZ4Xb1pUXuuu6UdHICAx8QqDtb9rVnjDM,2254
+berryworld/credentials.py,sha256=L70ZzSNaS2mbHtSS8lb3iesd1XSwZNnUzGznlEKxlJY,11018
 berryworld/devops.py,sha256=BAsVonVwCXoApUOovkt-BCzwc6KnXjxRDGff_ejSGw8,9719
 berryworld/email_con.py,sha256=uSBzs_Ijz9pUPWt9e7U3TCB7i6q7hU1bB5vhsTB_tmw,14448
-berryworld/email_logging.py,sha256=ebZ-PHZ9tCU3q-vkNnOT3-mTG7pCAmewnzzJZ9lnOjI,4476
+berryworld/email_logging.py,sha256=LeSrTExhQhar49gJR2wGC1dS0lqsNpFl9pS3eYWqnuo,4936
 berryworld/generate_env.py,sha256=Tk9Z_u7cA4Ve8YYTyLH2qwmLVAuYoTIWoFc0h8Va8lY,7842
-berryworld/handy_mix.py,sha256=SLCAdl2xaWEewWkECzcVFUDODDEkvUgpmJjTiccyVwU,9771
+berryworld/handy_mix.py,sha256=lRUes_V4jL-3ADOzwiKqi1Dukx-HWXvmf9-4s4MLwsA,9740
 berryworld/logic_apps.py,sha256=a0uU4tNO3v2w7grdBv-OOx4hUf7VBIerJpwZ9U-29dQ,14591
-berryworld/microsoft_teams.py,sha256=Mhj83-tQWRR58YfBUPUTQKQ-qJD1Y5FBPpjU70gP6UU,15964
+berryworld/microsoft_teams.py,sha256=8uPo0yku-euBj2VdzBoZCeX3IcsCCOqISLqaVZUVxfA,16030
 berryworld/persistent_storage.py,sha256=_lGdXa7IyxfMF3xNF9y26X_z9RDb2Ah7R0oF61HR8Gc,5764
 berryworld/pickle_management.py,sha256=O49ojVtTqYCT510rVRTbZWWaur_-5q3HSVG03Azn8mQ,2393
 berryworld/postgres_connection.py,sha256=whKDnchd5Feqpmxpoh2vlyn36EKHR-dVEULYq0N_4wA,8287
 berryworld/power_automate.py,sha256=9rDuRy0v-Ttq-SThid4lOB_tD4ibkyEmobiROpa--g4,25414
 berryworld/sharepoint_con.py,sha256=TuH-Vxk1VxjTi7x80KFssf_J8YPLRXpV27RBaFZi37U,22254
-berryworld/sql_conn.py,sha256=F5DO006Hjm1VvFzo7D7pRimEMU8QlrctZN3JQWoe6Zs,47403
+berryworld/snowflake_conn.py,sha256=qd7kjeGDBUtMQqSfR0rCVz1Qn6d9U3zskBvfLRNSofc,2480
+berryworld/sql_conn.py,sha256=meDAf_92hNW5Zp5sv_PPeIUA8NDchaDtVN7zo77veIA,47491
 berryworld/teams_logging.py,sha256=8NwXyWr4fLj7W6GzAm2nRQCGFDxibQpAHDHHD24FrP8,6997
-berryworld/transportation_solver.py,sha256=AdJPekVNufweaKDZLWYIB9qSxeVti80LaoaD-4NCSjc,5038
+berryworld/transportation_solver.py,sha256=gdlHRgl2wlp5EV0uhKIOYtkkbAlHaFHGktnMgvuENWE,5022
 berryworld/verify_keys.py,sha256=J2J505PcmBsQ9bj0XSRtXjpY-8qwpPU1A5LQdFRicFU,4257
 berryworld/vivantio.py,sha256=QfZo0UKqkzVRg_LyiwivNd3aEup4TH57x4KxLZkCJwc,10627
 berryworld/vivantio_logging.py,sha256=ciy7gA4u3FrgUIpEBnMgocbNPp6jcu9TPoy-kLcrTZU,5736
@@ -28,8 +29,8 @@ tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_allocation_config.py,sha256=e12l6fE9U57eSPS35g6ekJ_hol7-RHg89JV60_m1BlE,4633
 tests/test_handy_mix_config.py,sha256=Un56mz9KJmdn4K4OwzHAHLSRzDU1Xv2nFrONNuzOG04,2594
 tests/test_xml_parser.py,sha256=3QTlhFEd6KbK6nRFKZnc35tad6wqukTbe4QrFi8mr_8,859
-berryworld-1.0.0.179097.dist-info/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
-berryworld-1.0.0.179097.dist-info/METADATA,sha256=E9GLcy3lZWJ9HNHqXqx_56WHvu5RUqoW8qy-4w7Qxes,1091
-berryworld-1.0.0.179097.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-berryworld-1.0.0.179097.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
-berryworld-1.0.0.179097.dist-info/RECORD,,
+berryworld-1.0.0.180127.dist-info/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
+berryworld-1.0.0.180127.dist-info/METADATA,sha256=k_9Yaln2DC97Fes4tsW8U2GPrfZ9t2pzXDHOOCPhCcA,1091
+berryworld-1.0.0.180127.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+berryworld-1.0.0.180127.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
+berryworld-1.0.0.180127.dist-info/RECORD,,

{berryworld-1.0.0.179097.dist-info → berryworld-1.0.0.180127.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.0.0)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any