berryworld 1.0.0.161217__py3-none-any.whl → 1.0.0.161412__py3-none-any.whl

berryworld/power_automate.py

@@ -65,6 +65,9 @@ class PowerAutomate:
         # Base URL to get the flows
         self.base_flow_url = f'https://api.flow.microsoft.com/providers/Microsoft.ProcessSimple'
 
+        # Base URL to edit the flows
+        self.base_edit_url = 'https://make.powerautomate.com/'
+
         self.headers = self.generate_bearer_token(self.grant_type)
 
     def generate_bearer_token(self, grant_type, scope=None):
@@ -193,7 +196,7 @@ class PowerAutomate:
         # Get Flow Owners
         flows_df['Owners'] = None
         for i, flow in flows_df.iterrows():
-            flow_user_ids, flow_group_ids = self.list_flow_owners(flow['name'])
+            flow_user_ids, flow_group_ids = self.list_flow_owners(flow['FlowId'])
             flows_df.at[i, 'Owners'] = (flow_user_ids + flow_group_ids)
 
             user_ids = set(list(user_ids) + flow_user_ids)
@@ -215,33 +218,64 @@ class PowerAutomate:
                 batch_group_ids = group_ids[batch * 15:(batch + 1) * 15]
                 accounts_df = pd.concat([accounts_df, self.list_graph_accounts('groups', batch_group_ids)])
 
+        # Get Creator
+        flows_df['CreatedById'] = flows_df['FlowProperties'].apply(lambda x: x['creator']['userId'])
+        flows_df = flows_df.merge(
+            accounts_df[['displayName', 'id']].rename(columns={'id': 'CreatedById'}).drop_duplicates(),
+            how='left', on='CreatedById')
+        flows_df.rename(columns={'displayName': 'CreatedBy'}, inplace=True)
+
         if accounts_df.shape[0] > 0:
             flows_df = flows_df.explode('Owners')
             flows_df = flows_df.merge(accounts_df[['displayName', 'id']], how='left',
-                                      left_on='Owners',
-                                      right_on='id', suffixes=('', '_account'))
+                                      left_on='Owners', right_on='id', suffixes=('', '_account'))
 
             # Remove null owners
             null_account_mask = flows_df['displayName'].isnull()
-            flow_name_mask = flows_df.groupby('name')['name'].transform('count') > 1
+            flow_name_mask = flows_df.groupby('FlowId')['FlowId'].transform('count') > 1
             flows_df = flows_df[~(null_account_mask & flow_name_mask)]
 
             flows_df = flows_df[flows_df['displayName'].notna()]
-            flows_df = flows_df.groupby(['name', 'id', 'type']).agg({
-                'displayName': list,
-                'properties': list
-            }).reset_index()
+            flows_df = flows_df.groupby(['FlowId', 'id', 'type', 'CreatedBy']).agg({
+                'displayName': list, 'FlowProperties': list}).reset_index()
             flows_df.rename(columns={'displayName': 'Owners'}, inplace=True)
 
         return flows_df
 
+    @staticmethod
+    def enrich_info(flows_df):
+        """ Enrich the flows with the information
+        :param flows_df: DataFrame containing the flows information
+        """
+        if 'FlowProperties' in flows_df.columns:
+            flows_info_df = pd.DataFrame()
+            flows_list = []
+            for _, prop in flows_df[['FlowId', 'FlowProperties']].iterrows():
+                flow_info_response = prop['FlowProperties']
+                if isinstance(flow_info_response, list) & (len(flow_info_response) > 0):
+                    flow_info_response = flow_info_response[0]
+                flow_info = {'FlowId': prop['FlowId']}
+                for key in flow_info_response.keys():
+                    if isinstance(flow_info_response[key], (dict, list, bool)):
+                        flow_info[key] = [flow_info_response[key]]
+                    else:
+                        if len(flow_info_response[key]) > 0:
+                            flow_info[key] = flow_info_response[key]
+
+                flows_list.append(flow_info)
+                flows_info_df = pd.DataFrame(flows_list)
+
+            flows_df = pd.merge(flows_df, flows_info_df, on='FlowId', how='left')
+
+        return flows_df
+
     def enrich_run_history(self, flows_df):
         """ Enrich the flows with the run history
         :param flows_df: DataFrame containing the flows information
         """
         flows_runs_df = pd.DataFrame()
         for _, flow in flows_df.iterrows():
-            flow_url = f'{self.base_flow_url}/scopes/admin/environments/{self.environment_id}/flows/{flow["name"]}/' \
+            flow_url = f'{self.base_flow_url}/scopes/admin/environments/{self.environment_id}/flows/{flow["FlowId"]}/' \
                        f'runs?{self.api_version}'
 
             run_info_df = self.session_request("GET", flow_url)
@@ -255,11 +289,9 @@ class PowerAutomate:
                 run_info_df['ErrorMessage'] = run_info_df['RunProperties'].apply(
                     lambda x: x['error']['message'] if 'error' in x else None)
 
-                run_info_df['name'] = flow['name']
-                run_info_df = run_info_df.groupby(['name', 'ErrorCount', 'RunCount', 'LastRun', 'LastRunStatus']).agg({
-                    'RunProperties': list,
-                    'ErrorMessage': list
-                }).reset_index()
+                run_info_df['FlowId'] = flow['FlowId']
+                run_info_df = run_info_df.groupby(['FlowId', 'ErrorCount', 'RunCount', 'LastRun', 'LastRunStatus']).agg(
+                    {'RunProperties': list, 'ErrorMessage': list}).reset_index()
 
                 run_info_df['ErrorMessage'] = run_info_df['ErrorMessage'].apply(
                     lambda x: [e for e in x if not pd.isna(e)])
@@ -267,7 +299,7 @@ class PowerAutomate:
                 flows_runs_df = pd.concat([flows_runs_df, run_info_df])
 
         if flows_runs_df.shape[0] > 0:
-            flows_df = pd.merge(flows_df, flows_runs_df, how='left', on='name')
+            flows_df = pd.merge(flows_df, flows_runs_df, how='left', on='FlowId')
             flows_df[['ErrorCount', 'RunCount']] = flows_df[['ErrorCount', 'RunCount']].fillna(0).astype(int)
 
         return flows_df
@@ -278,19 +310,18 @@ class PowerAutomate:
         """
         flows_connections_df = pd.DataFrame()
         for i, flow in flows_df.iterrows():
-            flow_url = f'{self.base_flow_url}/scopes/admin/environments/{self.environment_id}/flows/{flow["name"]}/' \
+            flow_url = f'{self.base_flow_url}/scopes/admin/environments/{self.environment_id}/flows/{flow["FlowId"]}/' \
                        f'connections?{self.api_version}'
 
             connections_df = self.session_request("GET", flow_url)
             if connections_df.shape[0] > 0:
                 connections_df.rename(columns={'name': 'ConnectionName'}, inplace=True)
-                connections_df['name'] = flow['name']
+                connections_df['FlowId'] = flow['FlowId']
 
                 if 'properties' in connections_df.columns:
-                    connections_df_properties = pd.concat(
-                        connections_df.apply(lambda row: pd.json_normalize(row['properties']).assign(name=row['name']),
-                                             axis=1).tolist(),
-                        ignore_index=True)
+                    connections_df_properties = pd.concat(connections_df.apply(
+                        lambda row: pd.json_normalize(row['properties']).assign(name=row['FlowId']), axis=1).tolist(),
+                                                          ignore_index=True)
 
                     expanded_statuses_df = pd.concat(connections_df_properties.apply(
                         lambda row: pd.json_normalize(row['statuses']).assign(original_index=row.name),
@@ -337,7 +368,8 @@ class PowerAutomate:
                     flows_connections_df = pd.concat([flows_connections_df, connections_df_properties_group])
 
         if flows_connections_df.shape[0] > 0:
-            flows_df = pd.merge(flows_df, flows_connections_df, how='left', on='name')
+            flows_connections_df.rename(columns={'name': 'FlowId'}, inplace=True)
+            flows_df = pd.merge(flows_df, flows_connections_df, how='left', on='FlowId')
             flows_df['ConnectionsCount'] = flows_df['ConnectionsCount'].fillna(0).astype(int)
 
             if 'ConnectionErrorCode' in flows_df.columns:
@@ -348,67 +380,61 @@ class PowerAutomate:
 
         return flows_df
 
-    def list_flows(self, **kwargs):
+    def enrich_flow_url(self, flows_df):
+        """ Enrich the flows with the flow URL
+        :param flows_df: DataFrame containing the flows information
+        """
+        replacement_str = '/providers/Microsoft.ProcessSimple/'
+        flows_df.loc[:, 'FlowUrl'] = flows_df['id'].apply(lambda x: self.base_edit_url + x.replace(replacement_str, ''))
+
+        return flows_df
+
+    def list_flows(self, relevant_columns=True, **kwargs):
         """ List the flows of an environment
+        :param relevant_columns: Whether to return all the columns or only the relevant ones
         """
         enrich = False
         enrich_info = False
         enrich_owners = False
         enrich_run_history = False
         enrich_connections = False
+        enrich_flow_url = False
         if kwargs != {}:
             try:
                 if 'enrich' in kwargs.keys():
                     enrich = True
 
-                if ('enrich_info' in kwargs.keys()) | (enrich is True):
-                    enrich_info = True
-
-                if ('enrich_owners' in kwargs.keys()) | (enrich is True):
+                if enrich:
                     enrich_owners = True
-
-                if ('enrich_run_history' in kwargs.keys()) | (enrich is True):
+                    enrich_info = True
                     enrich_run_history = True
-
-                if ('enrich_connections' in kwargs.keys()) | (enrich is True):
                     enrich_connections = True
+                    enrich_flow_url = True
+                else:
+                    if ('enrich_info' in kwargs.keys()) | (enrich is True):
+                        enrich_info = True
+                    if ('enrich_owners' in kwargs.keys()) | (enrich is True):
+                        enrich_owners = True
+                    if ('enrich_run_history' in kwargs.keys()) | (enrich is True):
+                        enrich_run_history = True
+                    if ('enrich_connections' in kwargs.keys()) | (enrich is True):
+                        enrich_connections = True
+                    if ('enrich_flow_url' in kwargs.keys()) | (enrich is True):
+                        enrich_flow_url = True
 
             except KeyError:
                 raise KeyError('Please provide a value to enrich the response with')
 
         flow_url = f'{self.base_flow_url}/scopes/admin/environments/{self.environment_id}/flows?{self.api_version}'
         flows_df = self.session_request("GET", flow_url)
-
-        if enrich:
-            enrich_owners = True
-            enrich_info = True
-            enrich_run_history = True
-            enrich_connections = True
+        flows_df.rename(columns={'name': 'FlowId', 'properties': 'FlowProperties'}, inplace=True)
 
         if enrich_owners:
             flows_df = self.enrich_owners(flows_df)
-            flows_df.drop_duplicates(subset=['name'], inplace=True)
+            flows_df.drop_duplicates(subset=['FlowId'], inplace=True)
 
         if enrich_info:
-            if 'properties' in flows_df.columns:
-                flows_info_df = pd.DataFrame()
-                flows_list = []
-                for _, prop in flows_df[['name', 'properties']].iterrows():
-                    flow_info_response = prop['properties']
-                    if isinstance(flow_info_response, list) & (len(flow_info_response) > 0):
-                        flow_info_response = flow_info_response[0]
-                    flow_info = {'name': prop['name']}
-                    for key in flow_info_response.keys():
-                        if isinstance(flow_info_response[key], (dict, list, bool)):
-                            flow_info[key] = [flow_info_response[key]]
-                        else:
-                            if len(flow_info_response[key]) > 0:
-                                flow_info[key] = flow_info_response[key]
-
-                    flows_list.append(flow_info)
-                    flows_info_df = pd.DataFrame(flows_list)
-
-                flows_df = pd.merge(flows_df, flows_info_df, on='name', how='left')
+            flows_df = self.enrich_info(flows_df)
 
         if enrich_run_history:
             flows_df = self.enrich_run_history(flows_df)
@@ -416,7 +442,41 @@ class PowerAutomate:
         if enrich_connections:
             flows_df = self.enrich_connections(flows_df)
 
-        return flows_df.reset_index(drop=True)
+        if enrich_flow_url:
+            flows_df = self.enrich_flow_url(flows_df)
+
+        flows_df.rename(
+            columns={
+                'displayName': 'FlowName', 'definition': 'JsonDefinition', 'state': 'State',
+                'createdTime': 'CreatedDate', 'lastModifiedTime': 'LastModifiedDate',
+                'flowSuspensionReason': 'FlowSuspensionReason', 'flowSuspensionTime': 'FlowSuspensionTime',
+                'flowSuspensionReasonDetails': 'FlowSuspensionReasonDetails',
+                'AuthenticatedUserName': 'ConnectionAuthenticatedUserName'
+            }, inplace=True
+        )
+
+        if relevant_columns:
+            flows_final_df = flows_df[['FlowId', 'FlowName', 'id', 'FlowUrl', 'JsonDefinition', 'State', 'CreatedBy',
+                                       'Owners', 'CreatedDate', 'LastModifiedDate', 'FlowSuspensionReason',
+                                       'ErrorCount', 'ErrorMessage', 'RunCount', 'RunProperties', 'LastRun',
+                                       'LastRunStatus', 'ConnectionAccountName', 'ConnectionStatus',
+                                       'ConnectionLastModifiedTime', 'ConnectionsCount',
+                                       'ConnectionAuthenticatedUserName', 'ConnectionErrorCode']]
+
+            if 'FlowSuspensionReasonDetails' in flows_df.columns:
+                flows_final_df.loc[:, 'FlowSuspensionReasonDetails'] = flows_df['FlowSuspensionReasonDetails']
+            if 'FlowSuspensionTime' in flows_df.columns:
+                flows_final_df.loc[:, 'FlowSuspensionTime'] = flows_df['FlowSuspensionTime']
+            if 'ConnectionErrorCode' in flows_df.columns:
+                flows_final_df.loc[:, 'ConnectionErrorCode'] = flows_df['ConnectionErrorCode']
+            if 'ConnectionErrorMessage' in flows_df.columns:
+                flows_final_df.loc[:, 'ConnectionErrorMessage'] = flows_df['ConnectionErrorMessage']
+        else:
+            flows_final_df = flows_df
+
+        flows_final_df.sort_values(by=['FlowName'], inplace=True)
+
+        return flows_final_df.reset_index(drop=True)
 
     def update_flow(self, flow_id, payload):
         """ Update a flow

{berryworld-1.0.0.161217.dist-info → berryworld-1.0.0.161412.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: berryworld
-Version: 1.0.0.161217
+Version: 1.0.0.161412
 Summary: Handy classes to improve ETL processes
 Home-page: https://www.berryworld.com
 Author: BerryWorld ltd

{berryworld-1.0.0.161217.dist-info → berryworld-1.0.0.161412.dist-info}/RECORD

@@ -14,7 +14,7 @@ berryworld/microsoft_teams.py,sha256=Mhj83-tQWRR58YfBUPUTQKQ-qJD1Y5FBPpjU70gP6UU
 berryworld/persistent_storage.py,sha256=_lGdXa7IyxfMF3xNF9y26X_z9RDb2Ah7R0oF61HR8Gc,5764
 berryworld/pickle_management.py,sha256=O49ojVtTqYCT510rVRTbZWWaur_-5q3HSVG03Azn8mQ,2393
 berryworld/postgres_connection.py,sha256=L4x1wM0g4HkkkaJhqOP4NV9klWsaQ_F8EucD0Q2Rr60,8788
-berryworld/power_automate.py,sha256=h8Qn6NiXgLTGV-00vUYovR7joUgIV9_qu6MAKamXQJE,21957
+berryworld/power_automate.py,sha256=PY7Sj7t6Td8P54qUfnKSMpSkV9leITadtgpP9o5KZhM,25575
 berryworld/sharepoint_con.py,sha256=72CZSVpAoHxocAWFuSSeUkNHa4Vu8FdM09NX-QC5-eE,22011
 berryworld/sql_conn.py,sha256=_q7FSrVHUuNWmqLjMR4is2o9Tns2jWNxtmi3tahmkSw,45226
 berryworld/sql_connection.py,sha256=qxSMZKKUhjX-c2BOE14qJFfvHhDZ_CNM_w4XbLec-y0,44344
@@ -28,8 +28,8 @@ tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_allocation_config.py,sha256=e12l6fE9U57eSPS35g6ekJ_hol7-RHg89JV60_m1BlE,4633
 tests/test_handy_mix_config.py,sha256=Un56mz9KJmdn4K4OwzHAHLSRzDU1Xv2nFrONNuzOG04,2594
 tests/test_xml_parser.py,sha256=3QTlhFEd6KbK6nRFKZnc35tad6wqukTbe4QrFi8mr_8,859
-berryworld-1.0.0.161217.dist-info/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
-berryworld-1.0.0.161217.dist-info/METADATA,sha256=hgaWo5pZCPCfRcdn_Yp4JaIdteYFu3XEpQFsrnu0p3I,1107
-berryworld-1.0.0.161217.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-berryworld-1.0.0.161217.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
-berryworld-1.0.0.161217.dist-info/RECORD,,
+berryworld-1.0.0.161412.dist-info/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
+berryworld-1.0.0.161412.dist-info/METADATA,sha256=jVfB9cUzbjuRoc2N6zPmhS5eHo0s395Vw9g8HOrfhqs,1107
+berryworld-1.0.0.161412.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+berryworld-1.0.0.161412.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
+berryworld-1.0.0.161412.dist-info/RECORD,,