msfabricpysdkcore 0.0.8__py3-none-any.whl → 0.0.10__py3-none-any.whl

msfabricpysdkcore/deployment_pipeline.py

@@ -0,0 +1,240 @@
+import json
+from time import sleep
+
+import requests 
+from msfabricpysdkcore.long_running_operation import check_long_running_operation
+
+
+class DeploymentPipeline:
+    """Class to represent a deployment pipeline in Microsoft Fabric"""
+
+    def __init__(self, id, display_name, description, auth) -> None:
+        self.id = id
+        self.display_name = display_name
+        self.description = description
+        self.auth = auth
+
+    
+    def from_dict(dict,  auth):
+        """Create a Workspace object from a dictionary"""
+        if dict["displayName"] == None:
+            dict["displayName"] = dict["display_name"]
+
+        
+        return DeploymentPipeline(
+            id=dict["id"],
+            display_name=dict["displayName"],
+            description=dict["description"],
+            auth=auth
+        )
+
+    
+    def __str__(self) -> str:
+        """Return a string representation of the workspace object"""
+        dict_ = {"id": self.id, "display_name": self.display_name, "description": self.description}
+
+        return json.dumps(dict_, indent=2)
+    
+    def __repr__(self) -> str:
+        return self.__str__()
+    
+    def get_pipeline(deployment_pipeline_id, auth):
+        """Get a deployment pipeline"""
+        # GET https://api.fabric.microsoft.com/v1/deploymentPipelines/{deploymentPipelineId}
+
+        url = f"https://api.fabric.microsoft.com/v1/deploymentPipelines/{deployment_pipeline_id}"
+
+        for _ in range(10):
+            response = requests.get(url=url, headers=auth.get_headers())
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                raise Exception(f"Error getting item: {response.text}")
+            break
+        
+        item_dict = json.loads(response.text)
+        return DeploymentPipeline.from_dict(item_dict, auth)
+    
+    def get_stages(self, continuationToken = None):
+        """Get stages in a deployment pipeline"""
+        # GET https://api.fabric.microsoft.com/v1/deploymentPipelines/{deploymentPipelineId}/stages
+
+        url = f"https://api.fabric.microsoft.com/v1/deploymentPipelines/{self.id}/stages"
+
+        if continuationToken:
+            url += f"?continuationToken={continuationToken}"
+
+        for _ in range(10):
+            response = requests.get(url=url, headers=self.auth.get_headers())
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                raise Exception(f"Error getting stages: {response.text}")
+            break
+        
+        resp_dict = json.loads(response.text)
+        items = resp_dict["value"]
+        for item in items:
+            item["deploymentPipelineId"] = self.id
+        stages = [Deployment_Pipeline_Stage.from_dict(item, self.auth) for item in items]
+
+        if "continuationToken" in resp_dict:
+            stages_next = self.get_stages(continuationToken=resp_dict["continuationToken"])
+            stages.extend(stages_next)
+        
+        return stages
+    
+
+    def deploy(self, source_stage_id, target_stage_id, created_workspace_details = None,
+               items = None, note = None, wait_for_completion = True):
+        # POST https://api.fabric.microsoft.com/v1/deploymentPipelines/{deploymentPipelineId}/deploy
+
+        url = f"https://api.fabric.microsoft.com/v1/deploymentPipelines/{self.id}/deploy"
+
+        body = {
+            "sourceStageId": source_stage_id,
+            "targetStageId": target_stage_id
+        }
+
+        if created_workspace_details:
+            body["createdWorkspaceDetails"] = created_workspace_details
+        if items:
+            body["items"] = items
+        if note:
+            body["note"] = note
+            
+
+        for _ in range(10):
+            response = requests.post(url=url, headers=self.auth.get_headers(), json=body)
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code == 202 and wait_for_completion:
+                print("successfully started the operation")
+                try:
+                    operation_result = check_long_running_operation( response.headers, self.auth)
+                    return operation_result
+                except Exception as e:
+                    print("Problem waiting for long running operation. Returning initial response.")
+                    print(e)
+                    return response
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                raise Exception(f"Error deploying: {response.text}")
+            break
+
+        return response.json()
+
+
+
+    def get_deployment_pipeline_stages_items(self, stage_id = None, stage_name = None):
+        """Get items in a deployment pipeline stage"""
+
+        if stage_id == None and stage_name == None:
+            raise Exception("Please provide either stage_id or stage_name")
+        stages = self.get_stages()
+        if stage_id is None:
+            dep_pip_stages = [stage for stage in stages if stage.display_name == stage_name]
+            if len(dep_pip_stages) == 0:
+                raise Exception(f"Stage with name {stage_name} not found")
+        else:
+            dep_pip_stages = [stage for stage in stages if stage.id == stage_id]
+            if len(dep_pip_stages) == 0:
+                raise Exception(f"Stage with id {stage_id} not found")
+        dep_pip_stage = dep_pip_stages[0]
+        return dep_pip_stage.get_items()
+
+class Deployment_Pipeline_Stage():
+
+    """Class to represent a deployment pipeline stage in Microsoft Fabric"""
+
+    def __init__(self, id, order, display_name, description, workspace_id, workspace_name, is_public, deployment_pipeline_id, auth) -> None:
+        self.id = id
+        self.order = order
+        self.display_name = display_name
+        self.description = description
+        self.workspace_id = workspace_id
+        self.workspace_name = workspace_name
+        self.is_public = is_public
+        self.deployment_pipeline_id = deployment_pipeline_id
+        self.auth = auth
+
+    
+    def from_dict(dict,  auth):
+        """Create a Workspace object from a dictionary"""
+        if dict["displayName"] is None:
+            dict["displayName"] = dict["display_name"]
+
+        if dict.get("workspaceId", None) is None:
+            dict["workspaceId"] = dict.get("workspace_id", None)
+        
+        if dict.get("workspaceName", None)  is None:
+            dict["workspaceName"] = dict.get("workspace_name", None)
+        
+        if dict["deploymentPipelineId"] is None:
+            dict["deploymentPipelineId"] = dict["deployment_pipeline_id"]
+
+        if dict["isPublic"] is None:
+            dict["isPublic"] = dict["is_public"]
+
+        
+        return Deployment_Pipeline_Stage(id=dict["id"],
+                                         order=dict["order"],
+                                         display_name=dict["displayName"],
+                                         description=dict["description"],
+                                         workspace_id=dict["workspaceId"],
+                                         workspace_name=dict["workspaceName"],
+                                         is_public=dict["isPublic"],
+                                         deployment_pipeline_id=dict["deploymentPipelineId"],
+                                         auth=auth
+                                        )
+    
+    def __str__(self) -> str:
+        """Return a string representation of the workspace object"""
+        dict_ = {"id": self.id, "order": self.order, "display_name": self.display_name, 
+                 "description": self.description, "workspace_id": self.workspace_id, 
+                 "workspace_name": self.workspace_name, 
+                 "deployment_pipeline_id": self.deployment_pipeline_id,
+                 "is_public": self.is_public}
+
+        return json.dumps(dict_, indent=2)
+    
+    def __repr__(self) -> str:
+        return self.__str__()
+    
+
+    def get_items(self, continuationToken = None):
+        """Get items in a deployment pipeline stage"""
+        # GET https://api.fabric.microsoft.com/v1/deploymentPipelines/{deploymentPipelineId}/stages/{stageId}/items
+
+        url = f"https://api.fabric.microsoft.com/v1/deploymentPipelines/{self.deployment_pipeline_id}/stages/{self.id}/items"
+        if continuationToken is not None:
+            url += f"?continuationToken={continuationToken}"
+
+        for _ in range(10):
+            response = requests.get(url=url, headers=self.auth.get_headers())
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                raise Exception(f"Error getting items: {response.text}")
+            break
+
+        resp_dict = json.loads(response.text)
+        items = resp_dict["value"]
+        return items
+
+        

msfabricpysdkcore/environment.py

@@ -0,0 +1,222 @@
+import json 
+import requests
+from time import sleep
+
+from msfabricpysdkcore.item import Item
+from msfabricpysdkcore.long_running_operation import check_long_running_operation
+
+class Environment(Item):
+    """Class to represent a item in Microsoft Fabric"""
+
+    def __init__(self, id, display_name, type, workspace_id, auth, properties = None, definition=None, description="", 
+                 sparkcompute = None, staging_sparkcompute = None, libraries = None, staging_libraries = None):
+        super().__init__(id, display_name, type, workspace_id, auth, properties, definition, description)
+
+        self.sparkcompute = sparkcompute
+        self.staging_sparkcompute = staging_sparkcompute
+        self.libraries = libraries
+        self.staging_libraries = staging_libraries
+
+    def from_dict(item_dict, auth):
+        return Environment(id=item_dict['id'], display_name=item_dict['displayName'], type=item_dict['type'], workspace_id=item_dict['workspaceId'],
+            properties=item_dict.get('properties', None),
+            definition=item_dict.get('definition', None), description=item_dict.get('description', ""), 
+            sparkcompute=item_dict.get('sparkcompute', None),
+            staging_sparkcompute=item_dict.get('staging_sparkcompute', None),
+            libraries=item_dict.get('libraries', None),
+            staging_libraries=item_dict.get('staging_libraries', None),
+            auth=auth)
+
+    # GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/environments/{environmentId}/sparkcompute
+    def get_published_settings(self):
+        """Get the published settings of the environment"""
+        url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/environments/{self.id}/sparkcompute"
+
+        for _ in range(10):
+            response = requests.get(url=url, headers=self.auth.get_headers())
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                print(self)
+                raise Exception(f"Error getting published settings: {response.text}")
+            break
+
+        resp_json = json.loads(response.text)        
+        self.sparkcompute = resp_json
+        return resp_json
+    
+    # GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/environments/{environmentId}/staging/sparkcompute
+
+    def get_staging_settings(self):
+        """Get the staging settings of the environment"""
+        url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/environments/{self.id}/staging/sparkcompute"
+
+        for _ in range(10):
+            response = requests.get(url=url, headers=self.auth.get_headers())
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                print(self)
+                raise Exception(f"Error getting staging settings: {response.text}")
+            break
+
+        resp_json = json.loads(response.text)        
+        self.staging_sparkcompute = resp_json
+        return resp_json
+
+    def update_staging_settings(self, instance_pool, driver_cores, driver_memory, executor_cores, executor_memory,
+                                dynamic_executor_allocation, spark_properties, runtime_version):
+        """Update the staging settings of the environment"""
+        url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/environments/{self.id}/staging/sparkcompute"
+        body = {
+            "instancePool": instance_pool,
+            "driverCores": driver_cores,
+            "driverMemory": driver_memory,
+            "executorCores": executor_cores,
+            "executorMemory": executor_memory,
+            "dynamicExecutorAllocation": dynamic_executor_allocation,
+            "sparkProperties": spark_properties,
+            "runtimeVersion": runtime_version
+        }
+        for _ in range(10):
+            response = requests.patch(url=url, headers=self.auth.get_headers(), json=body)
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                print(self)
+                raise Exception(f"Error updating staging settings: {response.text}")
+            break
+
+        self.staging_sparkcompute = body
+        return body
+
+#  GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/environments/{environmentId}/libraries
+
+    def get_published_libraries(self):
+        """Get the published libraries of the environment"""
+        url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/environments/{self.id}/libraries"
+
+        for _ in range(10):
+            response = requests.get(url=url, headers=self.auth.get_headers())
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                print(self)
+                raise Exception(f"Error getting published libraries: {response.text}")
+            break
+
+        resp_json = json.loads(response.text)
+        self.libraries = resp_json
+        return resp_json
+    
+# GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/environments/{environmentId}/staging/libraries
+
+    def get_staging_libraries(self):
+        """Get the staging libraries of the environment"""
+        url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/environments/{self.id}/staging/libraries"
+
+        for _ in range(10):
+            response = requests.get(url=url, headers=self.auth.get_headers())
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                print(self)
+                raise Exception(f"Error getting staging libraries: {response.text}")
+            break
+
+        resp_json = json.loads(response.text)
+        self.staging_libraries = resp_json
+        return resp_json
+    
+
+    def update_staging_libraries(self):
+        raise NotImplementedError("This method is not implemented yet because the REST API is not complete")
+    
+# DELETE https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/environments/{environmentId}/staging/libraries?libraryToDelete={libraryToDelete}
+
+    def delete_staging_library(self, library_to_delete):
+        """Delete a library from the staging libraries of the environment"""
+        url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/environments/{self.id}/staging/libraries?libraryToDelete={library_to_delete}"
+
+        for _ in range(10):
+            response = requests.delete(url=url, headers=self.auth.get_headers())
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                print(self)
+                raise Exception(f"Error deleting staging libraries: {response.text}")
+            break
+
+        return response.text
+    
+# POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/environments/{environmentId}/staging/publish
+
+    def publish_environment(self):
+        """Publish the staging settings and libraries of the environment"""
+        url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/environments/{self.id}/staging/publish"
+
+        for _ in range(10):
+            response = requests.post(url=url, headers=self.auth.get_headers())
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code == 202:
+                publish_info = check_long_running_operation(response.headers, self.auth)
+                return publish_info
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                print(self)
+                raise Exception(f"Error publishing staging: {response.text}")
+            break
+
+        resp_dict = json.loads(response.text)
+        return resp_dict
+
+
+# POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/environments/{environmentId}/staging/cancelPublish
+
+    def cancel_publish(self):
+        """Cancel the publishing of the staging settings and libraries of the environment"""
+        url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/environments/{self.id}/staging/cancelPublish"
+
+        for _ in range(10):
+            response = requests.post(url=url, headers=self.auth.get_headers())
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code not in (200, 429):
+                print(response.status_code)
+                print(response.text)
+                print(self)
+                raise Exception(f"Error canceling publishing: {response.text}")
+            break
+
+        resp_dict = json.loads(response.text)
+        return resp_dict

msfabricpysdkcore/item.py

@@ -41,11 +41,6 @@ class Item:
     def from_dict(item_dict, auth):
         """Create Item object from dictionary"""
         
-        if item_dict['type'] == "Lakehouse":
-            from msfabricpysdkcore.lakehouse import Lakehouse
-            return Lakehouse(id=item_dict['id'], display_name=item_dict['displayName'], type=item_dict['type'], workspace_id=item_dict['workspaceId'],
-                    properties=item_dict.get('properties', None),
-                    definition=item_dict.get('definition', None), description=item_dict.get('description', ""), auth=auth)
         return Item(id=item_dict['id'], display_name=item_dict['displayName'], type=item_dict['type'], workspace_id=item_dict['workspaceId'],
                     properties=item_dict.get('properties', None),
                     definition=item_dict.get('definition', None), description=item_dict.get('description', ""), auth=auth)
@@ -71,10 +66,15 @@ class Item:
 
         return response.status_code
     
-    def get_definition(self):
+    def get_definition(self, type = None, format = None):
         """Get the definition of the item"""
         
         url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/getDefinition"
+        if type:
+            url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/{type}/{self.id}/getDefinition"
+
+        if format:
+            url += f"?format={format}"
 
         for _ in range(10):
             response = requests.post(url=url, headers=self.auth.get_headers())
@@ -127,9 +127,13 @@ class Item:
 
         return self
 
-    def update_definition(self, definition):
+    def update_definition(self, definition, type = None):
         """Update the item definition"""
         url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/updateDefinition"
+
+        if type:
+            url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/{type}/{self.id}/updateDefinition"
+            
         payload = {
             'definition': definition
         }

msfabricpysdkcore/lakehouse.py

@@ -3,6 +3,7 @@ import requests
 from time import sleep
 
 from msfabricpysdkcore.item import Item
+from msfabricpysdkcore.long_running_operation import check_long_running_operation
 
 class Lakehouse(Item):
     """Class to represent a item in Microsoft Fabric"""
@@ -10,6 +11,10 @@ class Lakehouse(Item):
     def __init__(self, id, display_name, type, workspace_id, auth, properties = None, definition=None, description=""):
         super().__init__(id, display_name, type, workspace_id, auth, properties, definition, description)
 
+    def from_dict(item_dict, auth):
+        return Lakehouse(id=item_dict['id'], display_name=item_dict['displayName'], type=item_dict['type'], workspace_id=item_dict['workspaceId'],
+            properties=item_dict.get('properties', None),
+            definition=item_dict.get('definition', None), description=item_dict.get('description', ""), auth=auth)
 
     def list_tables(self, continuationToken = None):
         """List all tables in the lakehouse"""
@@ -92,4 +97,40 @@ class Lakehouse(Item):
             
             sleep(3)
         return False
-    
+    
+    # run on demand table maintenance
+    # POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/lakehouses/{lakehouseId}/jobs/instances?jobType={jobType}
+
+    def run_on_demand_table_maintenance(self, execution_data, job_type = "TableMaintenance", wait_for_completion = True):
+        """Run on demand table maintenance"""
+        url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/lakehouses/{self.id}/jobs/instances?jobType={job_type}"
+
+        body = {
+                "executionData": execution_data
+              }
+
+        for _ in range(10):
+            response = requests.post(url=url, headers=self.auth.get_headers(), json=body)
+            if response.status_code == 429:
+                print("Too many requests, waiting 10 seconds")
+                sleep(10)
+                continue
+            if response.status_code == 202 and wait_for_completion:
+                print("successfully started the operation")
+                try:
+                    operation_result = check_long_running_operation( response.headers, self.auth)
+                    return operation_result
+                except Exception as e:
+                    
+                    print("Problem waiting for long running operation. Returning initial response.")
+                    print(e)
+                    return response
+                
+            if response.status_code not in (200, 202, 429):
+                print(response.status_code)
+                print(response.text)
+
+                raise Exception(f"Error at run on demand table maintenance: {response.text}")
+            break
+
+        return response

msfabricpysdkcore/long_running_operation.py

@@ -25,9 +25,7 @@ class LongRunningOperation:
             if response.status_code == 400:
                 return None
             if response.status_code not in (200, 429):
-                print(response.status_code)
-                print(response.text)
-                raise Exception(f"Error getting operation results: {response.text}")
+                raise Exception(f"Error getting operation results: {response.status_code}, {response.text}")
             break
 
         return json.loads(response.text)
@@ -43,9 +41,7 @@ class LongRunningOperation:
                 sleep(10)
                 continue
             if response.status_code not in (200, 429):
-                print(response.status_code)
-                print(response.text)
-                raise Exception(f"Error getting operation state: {response.text}")
+                raise Exception(f"Error getting operation state: {response.status_code},  {response.text}")
             break
 
         return json.loads(response.text)