berryworld 1.0.0.178885__py3-none-any.whl → 1.0.0.179222__py3-none-any.whl

berryworld/cache_data.py

@@ -50,7 +50,7 @@ class CacheManagement:
     def dump_pickle(df, filepath):
         """ Save the data to a filepath
         :param df: DataFrame gathering the data
-        :param path: Path and filename to the file destination
+        :param filepath: Path and filename to the file destination
         """
         pickle_out = open(filepath, 'wb')
         pickle.dump(df, pickle_out)
@@ -59,7 +59,7 @@ class CacheManagement:
     @staticmethod
     def retrieve_pickle(filepath):
         """ Get the data from the indicated filepath
-        :param path: Path and filename to the file destination
+        :param filepath: Path and filename to the file destination
         """
         pickle_in = open(filepath, "rb")
         data_df = pickle.load(pickle_in)
@@ -69,7 +69,7 @@ class CacheManagement:
     @staticmethod
     def remove_pickle(filepath):
         """ Delete a file allocated in filepath
-        :param path: Path and filename to the file destination
+        :param filepath: Path and filename to the file destination
         """
         if os.path.exists(filepath):
             os.remove(filepath)

berryworld/credentials.py

@@ -209,15 +209,18 @@ class WebServiceCredentials:
 
             try:
                 user_name = os.environ.get(f"WEBSERVICE_USER_{self.service.upper()}")
-            except Exception:
+            except Exception as e:
+                print(e)
                 user_name = ''
             try:
                 password = os.environ.get(f"WEBSERVICE_PASSWORD_{self.service.upper()}")
-            except Exception:
+            except Exception as e:
+                print(e)
                 password = ''
             try:
                 access_token = os.environ.get(f"WEBSERVICE_ACCESS_TOKEN_{self.service.upper()}")
-            except Exception:
+            except Exception as e:
+                print(e)
                 access_token = ''
 
             return {'user_name': user_name,

berryworld/email_con.py

@@ -275,8 +275,7 @@ class EmailConnection:
             return emails_df.shape[0] != 0
 
         except Exception as e:
-            print(traceback.format_exc())
-            raise Exception(e)
+            print(e)
 
     @staticmethod
     def get_mimetype(file_path):
@@ -284,8 +283,6 @@ class EmailConnection:
         :param file_path: Path of the file to be attached
         """
         # Get mimetype
-        mimetype = None
-        extension = None
         if ('.jpg' in file_path) | ('.jpeg' in file_path):
             mimetype = "image/jpeg"
             extension = '.jpeg'

berryworld/email_logging.py

@@ -59,16 +59,24 @@ class EmailLogging:
         :param critical: Indicate whether it should avoid sending successful logs
         :param proposed_solution: Proposed solution to the error message
         """
-        unsuccessful_columns = {'Successful': 0,
-                                'FinishedDate': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
-                                'Section': section,
-                                'ErrorMessage': str(error_message).replace("'", '"'),
-                                'Critical': 1 if critical is True else 0}
-        self.log_df = self.log_df.assign(**unsuccessful_columns)
-        if proposed_solution is not None:
-            self.log_df = self.log_df.assign(**{'ProposedSolution': proposed_solution})
-        self.sql_con.insert(self.log_df, 'Logging', 'PythonLogs', print_sql=self.print_sql)
-        self.failure_type.append(critical)
+        save_failure = True
+        if section is not None:
+            if (('prod' not in str(self.sql_con.server)) | ('prd' not in str(self.sql_con.server))) \
+                    & (('connection' in str(section).lower()) & ('dw' in str(section).lower())):
+                print('Avoiding to report a connection DW error in a non-production environment')
+                save_failure = False
+
+        if save_failure:
+            unsuccessful_columns = {'Successful': 0,
+                                    'FinishedDate': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
+                                    'Section': section,
+                                    'ErrorMessage': str(error_message).replace("'", '"'),
+                                    'Critical': 1 if critical is True else 0}
+            self.log_df = self.log_df.assign(**unsuccessful_columns)
+            if proposed_solution is not None:
+                self.log_df = self.log_df.assign(**{'ProposedSolution': proposed_solution})
+            self.sql_con.insert(self.log_df, 'Logging', 'PythonLogs', print_sql=self.print_sql)
+            self.failure_type.append(critical)
 
     @staticmethod
     def register_failure(email_log, section, error_, solutions=None):

berryworld/handy_mix.py

@@ -65,7 +65,21 @@ class HandyMix:
         return result
 
     @staticmethod
-    def likely_match(srs, match_str, n_res=1, threshold=False):
+    def grab_n_elem(n, comp_):
+        max_list = heapq.nlargest(n, comp_)
+        like_matches = np.array([elem in max_list for elem in comp_]).sum()
+        if like_matches > n:
+            list_ = [elem in max_list for elem in comp_]
+            k = 0
+            for elem in list_:
+                if (elem is True) & (k > n - 1):
+                    list_[k] = False
+                k = + 1
+            return list_
+        else:
+            return [elem in max_list for elem in comp_]
+
+    def likely_match(self, srs, match_str, n_res=1, threshold=False):
         """ Look for the more similar element in a Series to the one provided in match_str. If there is a draw,
         It will pick the first matching elements. Threshold only consider the elements if their likelihood is
         greater than that threshold variable
@@ -88,29 +102,15 @@ class HandyMix:
             aux = SequenceMatcher(None, element, match_str)
             comp.append(aux.ratio())
 
-        def grab_n_elem(n, comp):
-            max_list = heapq.nlargest(n, comp)
-            like_matches = np.array([elem in max_list for elem in comp]).sum()
-            if like_matches > n:
-                list_ = [elem in max_list for elem in comp]
-                k = 0
-                for elem in list_:
-                    if (elem is True) & (k > n - 1):
-                        list_[k] = False
-                    k = + 1
-                return list_
-            else:
-                return [elem in max_list for elem in comp]
-
         if threshold:
             # Return n elements above the threshold
             if any(elem >= threshold for elem in comp):
-                return grab_n_elem(n_res, comp)
+                return self.grab_n_elem(n_res, comp)
             else:
                 return [False] * comp.__len__()
         else:
             # Return n elements likely to match
-            return grab_n_elem(n_res, comp)
+            return self.grab_n_elem(n_res, comp)
 
     @staticmethod
     def remove_line_chars(input_text, remove_duplicate_white_spaces=False):

berryworld/logic_apps.py

@@ -5,6 +5,56 @@ from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
 
+def format_create_payload(payload):
+    """ Format the payload to create a workflow in Azure Portal
+    :param payload: Properties and Definition of the workflow
+    """
+    # Reshape the payload to fit the required format
+    json_dict = json.loads(payload)
+
+    # Check if definition key is in json payload
+    if not ('definition' in json_dict['properties'].keys()):
+        json_dict = {
+            "properties": {
+                "definition": json_dict['properties']
+            }
+        }
+
+    # Check if location key is in json payload
+    location_flag = False
+    location_dict = {}
+    if 'location' in json_dict['properties']['definition'].keys():
+        location_dict = json_dict['properties']['definition']['location']
+        json_dict = {
+            "properties": {
+                "definition": {key: val for key, val in json_dict['properties']['definition'].items() if
+                               key != 'location'}
+            }
+        }
+        location_flag = True
+
+    # Check if tags key is in json payload
+    tags_flag = False
+    tags_dict = {}
+    if 'tags' in json_dict['properties']['definition'].keys():
+        tags_dict = json_dict['properties']['definition']['tags']
+        json_dict = {
+            "properties": {
+                "definition": {key: val for key, val in json_dict['properties']['definition'].items() if
+                               key != 'tags'}
+            }
+        }
+        tags_flag = True
+        json_dict = {**json_dict, **{'tags': tags_dict}}
+
+    if tags_flag:
+        json_dict = {**json_dict, **{'tags': tags_dict}}
+    if location_flag:
+        json_dict = {**json_dict, **{'location': location_dict}}
+
+    return json.dumps(json_dict)
+
+
 class LogicApps:
     def __init__(self, **kwargs):
         """
@@ -187,7 +237,8 @@ class LogicApps:
         :param resource_group: Resource group Name to get the workflows from
         :param workflow_name: New name of the workflow
         :param payload: Payload to create the workflow. It usually comes under properties in the workflow response
-        which we must add "location" and "tags" keys.
+        which we must add "location" and "tags" keys. It should be in the same format that the Logic Apps are stored in
+        the Azure DevOps repository.
         Payload example:
         {
           "properties": {
@@ -246,6 +297,11 @@ class LogicApps:
                      f"providers/Microsoft.Logic/workflows/{workflow_name}?" + self.api_version
         headers_ = self.headers
         headers_['Content-Type'] = 'application/json'
+
         create_resp = req.put(create_url, headers=self.headers, data=payload)
-        if create_resp.status_code != 200:
-            raise Exception(f'Cannot create the workflow {workflow_name}. Error: {create_resp.text}')
+        if not str(create_resp.status_code).startswith('2'):
+            print('Try to reform the payload to fit the required format in the Azure Portal')
+            payload = format_create_payload(payload)
+            create_resp = req.put(create_url, headers=self.headers, data=payload)
+            if not str(create_resp.status_code).startswith('2'):
+                raise Exception(f'Cannot create the workflow {workflow_name}. Error: {create_resp.text}')

berryworld/microsoft_teams.py

@@ -254,11 +254,11 @@ class MicrosoftTeams:
             vivantio_ticket_id = vivantio_ticket['Id'].values[0]
             display_id = vivantio_ticket['DisplayId'].values[0]
             vivantio_ticket_url = f'https://poupart.flex.vivantio.com/item/Ticket/{vivantio_ticket_id}'
-            html_message = html_message + f'<br><b>Vivantio Ticket Id:</b> {display_id}' + \
-                           f'<br><b>Vivantio Ticket URL:</b> {vivantio_ticket_url}<br>'
+            html_message = html_message + f'<br><b>Vivantio Ticket Id:</b> {display_id}' \
+                                          f'<br><b>Vivantio Ticket URL:</b> {vivantio_ticket_url}<br>'
 
-        html_message = html_message + '<br><b>Error Status:</b> ' + status + f'<br><b>Error Count:</b> {count}' + \
-                       '<br><b>Error Content:</b> <br>' + message
+        html_message = html_message + '<br><b>Error Status:</b> ' + status + f'<br><b>Error Count:</b> {count}' \
+                                                                             f'<br><b>Error Content:</b> <br>' + message
 
         payload = {}
         if importance:

berryworld/sql_conn.py

@@ -942,9 +942,11 @@ class SQLConn:
                     # Avoid converting datetime values
                     try:
                         pd.to_datetime(data[col])
-                    except:
+                    except Exception as e:
+                        print(e)
                         data[col] = data[col].str.replace('.0', '')
-            except:
+            except Exception as e:
+                print(e)
                 continue
 
         return data

berryworld/transportation_solver.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from scipy.optimize import linprog
+from scipy.optimize import linprog, _highs
 
 
 class TransportationAlgorithm:
@@ -71,13 +71,13 @@ class TransportationAlgorithm:
 
         # Run the Simplex method
         method = 'Interior-Point without constraints'
-        res = linprog(c, A_eq=a_matrix, b_eq=b, options=options, method='interior-point')
+        res = linprog(c, A_eq=a_matrix, b_eq=b, options=options, method='highs-ipm')
         res_success = res.success
 
         # Run the Interior-Point method
         if res_success is False:
             method = 'Interior-Point without constraints'
-            res = linprog(c, A_eq=a_matrix, b_eq=b, options=options, method='revised simplex')
+            res = linprog(c, A_eq=a_matrix, b_eq=b, options=options, method='highs-ds')
             res_success = res.success
 
         # Run the interior-point method with constraints
@@ -89,7 +89,7 @@ class TransportationAlgorithm:
             max_value = max(prod[0], sales[0])
             b_ub = [max_value] + [max_value]
             res = linprog(c, A_eq=a_eq, b_eq=b_eq, A_ub=a_ub, b_ub=b_ub,
-                          options=options, method='interior-point')
+                          options=options, method='highs-ipm')
             res_success = res.success
 
         # Run the Revised Simplex method with constraints
@@ -101,7 +101,7 @@ class TransportationAlgorithm:
             max_value = max(prod[0], sales[0])
             b_ub = [max_value] + [max_value]
             res = linprog(c, A_eq=a_eq, b_eq=b_eq, A_ub=a_ub, b_ub=b_ub,
-                          options=options, method='revised simplex')
+                          options=options, method='highs-ds')
             res_success = res.success
 
         result = self.ranking

{berryworld-1.0.0.178885.dist-info → berryworld-1.0.0.179222.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: berryworld
-Version: 1.0.0.178885
+Version: 1.0.0.179222
 Summary: Handy classes to improve ETL processes
 Home-page: https://www.berryworld.com
 Author: BerryWorld ltd

{berryworld-1.0.0.178885.dist-info → berryworld-1.0.0.179222.dist-info}/RECORD

@@ -3,23 +3,23 @@ berryworld/aks_logs.py,sha256=Gb2_cokiZbEX01Yoptd0MxpDociaug-GrXdwliyxFBo,2293
 berryworld/allocation_solver.py,sha256=asFtaCAze6-eHUGWXA0kAp67UBS-Upj1KKdrVLj_ttQ,8513
 berryworld/app_logs.py,sha256=MKzKPYd3JuPfOQNAapIgaeZeFHw1z_w2mbn9I6QCADE,4180
 berryworld/app_logs_query.py,sha256=U94b-z3X9cuY_KFozupUcfaYciXWBn7p_RHkoRsfROU,4376
-berryworld/cache_data.py,sha256=2cStWbFQHimon_lHMbcM_0vU7lt-FCge96D-T9YXaxQ,2242
-berryworld/credentials.py,sha256=Knxo4gssLT7sbaBjOTFe3mX5k70G2e0M_6CdtlddjtA,10200
+berryworld/cache_data.py,sha256=AWAhV4PPkilZ4Xb1pUXuuu6UdHICAx8QqDtb9rVnjDM,2254
+berryworld/credentials.py,sha256=SeBLFqgadtUZclV1WbUC4uqI6xkcBlpV8miuifc8oqY,10290
 berryworld/devops.py,sha256=BAsVonVwCXoApUOovkt-BCzwc6KnXjxRDGff_ejSGw8,9719
-berryworld/email_con.py,sha256=lAadyJCTXdXJQeS9WDaTErtKxT1c9DKPJ-LpWNmWa98,14549
-berryworld/email_logging.py,sha256=ebZ-PHZ9tCU3q-vkNnOT3-mTG7pCAmewnzzJZ9lnOjI,4476
+berryworld/email_con.py,sha256=uSBzs_Ijz9pUPWt9e7U3TCB7i6q7hU1bB5vhsTB_tmw,14448
+berryworld/email_logging.py,sha256=LeSrTExhQhar49gJR2wGC1dS0lqsNpFl9pS3eYWqnuo,4936
 berryworld/generate_env.py,sha256=Tk9Z_u7cA4Ve8YYTyLH2qwmLVAuYoTIWoFc0h8Va8lY,7842
-berryworld/handy_mix.py,sha256=SLCAdl2xaWEewWkECzcVFUDODDEkvUgpmJjTiccyVwU,9771
-berryworld/logic_apps.py,sha256=hFi4wAOTgL9xirU6cR4rkOYTFBdDNes_aU3-aS9MsN0,12457
-berryworld/microsoft_teams.py,sha256=Mhj83-tQWRR58YfBUPUTQKQ-qJD1Y5FBPpjU70gP6UU,15964
+berryworld/handy_mix.py,sha256=lRUes_V4jL-3ADOzwiKqi1Dukx-HWXvmf9-4s4MLwsA,9740
+berryworld/logic_apps.py,sha256=a0uU4tNO3v2w7grdBv-OOx4hUf7VBIerJpwZ9U-29dQ,14591
+berryworld/microsoft_teams.py,sha256=8uPo0yku-euBj2VdzBoZCeX3IcsCCOqISLqaVZUVxfA,16030
 berryworld/persistent_storage.py,sha256=_lGdXa7IyxfMF3xNF9y26X_z9RDb2Ah7R0oF61HR8Gc,5764
 berryworld/pickle_management.py,sha256=O49ojVtTqYCT510rVRTbZWWaur_-5q3HSVG03Azn8mQ,2393
 berryworld/postgres_connection.py,sha256=whKDnchd5Feqpmxpoh2vlyn36EKHR-dVEULYq0N_4wA,8287
 berryworld/power_automate.py,sha256=9rDuRy0v-Ttq-SThid4lOB_tD4ibkyEmobiROpa--g4,25414
 berryworld/sharepoint_con.py,sha256=TuH-Vxk1VxjTi7x80KFssf_J8YPLRXpV27RBaFZi37U,22254
-berryworld/sql_conn.py,sha256=F5DO006Hjm1VvFzo7D7pRimEMU8QlrctZN3JQWoe6Zs,47403
+berryworld/sql_conn.py,sha256=meDAf_92hNW5Zp5sv_PPeIUA8NDchaDtVN7zo77veIA,47491
 berryworld/teams_logging.py,sha256=8NwXyWr4fLj7W6GzAm2nRQCGFDxibQpAHDHHD24FrP8,6997
-berryworld/transportation_solver.py,sha256=AdJPekVNufweaKDZLWYIB9qSxeVti80LaoaD-4NCSjc,5038
+berryworld/transportation_solver.py,sha256=gdlHRgl2wlp5EV0uhKIOYtkkbAlHaFHGktnMgvuENWE,5022
 berryworld/verify_keys.py,sha256=J2J505PcmBsQ9bj0XSRtXjpY-8qwpPU1A5LQdFRicFU,4257
 berryworld/vivantio.py,sha256=QfZo0UKqkzVRg_LyiwivNd3aEup4TH57x4KxLZkCJwc,10627
 berryworld/vivantio_logging.py,sha256=ciy7gA4u3FrgUIpEBnMgocbNPp6jcu9TPoy-kLcrTZU,5736
@@ -28,8 +28,8 @@ tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_allocation_config.py,sha256=e12l6fE9U57eSPS35g6ekJ_hol7-RHg89JV60_m1BlE,4633
 tests/test_handy_mix_config.py,sha256=Un56mz9KJmdn4K4OwzHAHLSRzDU1Xv2nFrONNuzOG04,2594
 tests/test_xml_parser.py,sha256=3QTlhFEd6KbK6nRFKZnc35tad6wqukTbe4QrFi8mr_8,859
-berryworld-1.0.0.178885.dist-info/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
-berryworld-1.0.0.178885.dist-info/METADATA,sha256=f8pKCUKdq0Dtk7T1jmtepXYhbh1I1LIuaem2lzNDKkw,1091
-berryworld-1.0.0.178885.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
-berryworld-1.0.0.178885.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
-berryworld-1.0.0.178885.dist-info/RECORD,,
+berryworld-1.0.0.179222.dist-info/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
+berryworld-1.0.0.179222.dist-info/METADATA,sha256=QgD_R8cGkJU0mNXoC8WvrwuDxfgC3RG1hFsKsA1HLMo,1091
+berryworld-1.0.0.179222.dist-info/WHEEL,sha256=uCRv0ZEik_232NlR4YDw4Pv3Ajt5bKvMH13NUU7hFuI,91
+berryworld-1.0.0.179222.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
+berryworld-1.0.0.179222.dist-info/RECORD,,

{berryworld-1.0.0.178885.dist-info → berryworld-1.0.0.179222.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (73.0.1)
+Generator: setuptools (74.1.1)
 Root-Is-Purelib: true
 Tag: py3-none-any