openmetadata-ingestion 1.7.0.0rc2__py3-none-any.whl → 1.7.1.0__py3-none-any.whl

metadata/ingestion/models/patch_request.py

@@ -369,6 +369,9 @@ def build_patch(
                 array_entity_fields=array_entity_fields,
             )
 
+        # special handler for tableConstraints
+        _table_constraints_handler(source, destination)
+
         # Get the difference between source and destination
         if allowed_fields:
             patch = jsonpatch.make_patch(
@@ -424,6 +427,69 @@ def build_patch(
         return None
 
 
+def _get_attribute_name(attr: T) -> str:
+    """Get the attribute name from the attribute."""
+    if hasattr(attr, "name"):
+        return model_str(attr.name)
+    return model_str(attr)
+
+
+def rearrange_attributes(final_attributes: List[T], source_attributes: List[T]):
+    source_staging_list = []
+    destination_staging_list = []
+    for attribute in final_attributes or []:
+        if attribute in source_attributes:
+            source_staging_list.append(attribute)
+        else:
+            destination_staging_list.append(attribute)
+    return source_staging_list + destination_staging_list
+
+
+def _table_constraints_handler(source: T, destination: T):
+    """
+    Handle table constraints patching properly.
+    This ensures we only perform allowed operations on constraints and maintain the structure.
+    """
+    if not hasattr(source, "tableConstraints") or not hasattr(
+        destination, "tableConstraints"
+    ):
+        return
+
+    source_table_constraints = getattr(source, "tableConstraints")
+    destination_table_constraints = getattr(destination, "tableConstraints")
+
+    if not source_table_constraints or not destination_table_constraints:
+        return
+
+    # Create a dictionary of source constraints for easy lookup
+    source_constraints_dict = {}
+    for constraint in source_table_constraints:
+        # Create a unique key based on constraintType and columns
+        key = f"{constraint.constraintType}:{','.join(sorted(constraint.columns))}"
+        source_constraints_dict[key] = constraint
+
+    # Rearrange destination constraints to match source order when possible
+    rearranged_constraints = []
+
+    # First add constraints that exist in both source and destination (preserving order from source)
+    for source_constraint in source_table_constraints:
+        key = f"{source_constraint.constraintType}:{','.join(sorted(source_constraint.columns))}"
+        for dest_constraint in destination_table_constraints:
+            dest_key = f"{dest_constraint.constraintType}:{','.join(sorted(dest_constraint.columns))}"
+            if key == dest_key:
+                rearranged_constraints.append(dest_constraint)
+                break
+
+    # Then add new constraints from destination that don't exist in source
+    for dest_constraint in destination_table_constraints:
+        dest_key = f"{dest_constraint.constraintType}:{','.join(sorted(dest_constraint.columns))}"
+        if dest_key not in source_constraints_dict:
+            rearranged_constraints.append(dest_constraint)
+
+    # Update the destination constraints with the rearranged list
+    setattr(destination, "tableConstraints", rearranged_constraints)
+
+
 def _sort_array_entity_fields(
     source: T,
     destination: T,
@@ -439,19 +505,21 @@ def _sort_array_entity_fields(
 
             # Create a dictionary of destination attributes for easy lookup
             destination_dict = {
-                model_str(attr.name): attr for attr in destination_attributes
+                _get_attribute_name(attr): attr for attr in destination_attributes
             }
 
             updated_attributes = []
             for source_attr in source_attributes or []:
                 # Update the destination attribute with the source attribute
-                destination_attr = destination_dict.get(model_str(source_attr.name))
+                destination_attr = destination_dict.get(
+                    _get_attribute_name(source_attr)
+                )
                 if destination_attr:
                     updated_attributes.append(
                         source_attr.model_copy(update=destination_attr.__dict__)
                     )
                     # Remove the updated attribute from the destination dictionary
-                    del destination_dict[model_str(source_attr.name)]
+                    del destination_dict[_get_attribute_name(source_attr)]
                 else:
                     updated_attributes.append(None)
 

metadata/ingestion/ometa/mixins/es_mixin.py

@@ -97,7 +97,7 @@ class ESMixin(Generic[T]):
 
     fqdn_search = (
         "/search/fieldQuery?fieldName={field_name}&fieldValue={field_value}&from={from_}"
-        "&size={size}&index={index}"
+        "&size={size}&index={index}&deleted=false"
     )
 
     # sort_field needs to be unique for the pagination to work, so we can use the FQN
@@ -125,14 +125,20 @@ class ESMixin(Generic[T]):
         if response:
             if fields:
                 fields = fields.split(",")
-            return [
-                self.get_by_name(
+
+            entities = []
+            for hit in response["hits"]["hits"]:
+                entity = self.get_by_name(
                     entity=entity_type,
                     fqn=hit["_source"]["fullyQualifiedName"],
                     fields=fields,
                 )
-                for hit in response["hits"]["hits"]
-            ] or None
+                if entity is None:
+                    continue
+
+                entities.append(entity)
+
+            return entities or None
 
         return None
 

metadata/ingestion/source/api/rest/connection.py

@@ -30,8 +30,6 @@ from metadata.ingestion.connections.test_connections import test_connection_step
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils.constants import THREE_MIN
 
-ACCEPTED_CONTENT_TYPES = ["application/json", "application/vnd.oai.openapi+json"]
-
 
 class SchemaURLError(Exception):
     """
@@ -68,21 +66,25 @@ def test_connection(
     """
 
     def custom_url_exec():
-        if client.status_code == 200 and any(
-            content_type in client.headers.get("content-type")
-            for content_type in ACCEPTED_CONTENT_TYPES
-        ):
+        if client.status_code == 200:
             return []
         raise SchemaURLError(
-            "Failed to parse JSON schema url. Please check if provided url is valid JSON schema."
+            "Failed to connect to the JSON schema url. Please check the url and credentials. Status Code was: "
+            + str(client.status_code)
         )
 
     def custom_schema_exec():
-        if client.json().get("openapi") is not None:
-            return []
-        raise InvalidOpenAPISchemaError(
-            "Provided schema is not valid OpenAPI JSON schema"
-        )
+        try:
+            if client.json() is not None and client.json().get("openapi") is not None:
+                return []
+
+            raise InvalidOpenAPISchemaError(
+                "Provided schema is not valid OpenAPI JSON schema"
+            )
+        except:
+            raise InvalidOpenAPISchemaError(
+                "Provided schema is not valid OpenAPI JSON schema"
+            )
 
     test_fn = {"CheckURL": custom_url_exec, "CheckSchema": custom_schema_exec}
 

metadata/ingestion/source/api/rest/metadata.py

@@ -197,8 +197,9 @@ class RestSource(ApiServiceSource):
     ) -> Optional[RESTEndpoint]:
         try:
             endpoint = RESTEndpoint(**info)
-            endpoint.name = f"{path}/{method_type}"
-            endpoint.display_name = f"{path}"
+            path_clean_name = clean_uri(path)
+            endpoint.name = f"{path_clean_name}/{method_type}"
+            endpoint.display_name = f"{path_clean_name}"
             endpoint.url = self._generate_endpoint_url(collection, endpoint)
             return endpoint
         except Exception as err:
@@ -279,6 +280,18 @@ class RestSource(ApiServiceSource):
                 .get("schema", {})
                 .get("$ref", {})
             )
+            if not schema_ref:
+                logger.debug("Trying to parse response schema from schema property")
+                schema_ref = (
+                    info.get("responses", {})
+                    .get("200", {})
+                    .get("content", {})
+                    .get("application/json", {})
+                    .get("schema", {})
+                    .get("properties", {})
+                    .get("data", {})
+                    .get("$ref")
+                )
             if not schema_ref:
                 logger.debug("No response schema found for the endpoint")
                 return None

metadata/ingestion/source/dashboard/dashboard_service.py

@@ -612,6 +612,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
         patch_request = PatchRequest(
             original_entity=original_entity,
             new_entity=original_entity.model_copy(update=create_request.__dict__),
+            override_metadata=self.source_config.overrideMetadata,
         )
         if isinstance(original_entity, Dashboard):
             # For patch the charts need to be entity ref instead of fqn

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -59,6 +59,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.ometa.utils import model_str
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
 from metadata.ingestion.source.dashboard.powerbi.models import (
+    Dataflow,
     Dataset,
     Group,
     PowerBIDashboard,
@@ -287,7 +288,7 @@ class PowerbiSource(DashboardServiceSource):
         """
         return (
             f"{clean_uri(self.service_connection.hostPort)}/groups/"
-            f"{workspace_id}/dashboards/{dashboard_id}"
+            f"{workspace_id}/dashboards/{dashboard_id}?experience=power-bi"
         )
 
     def _get_report_url(self, workspace_id: str, dashboard_id: str) -> str:
@@ -296,7 +297,7 @@ class PowerbiSource(DashboardServiceSource):
         """
         return (
             f"{clean_uri(self.service_connection.hostPort)}/groups/"
-            f"{workspace_id}/reports/{dashboard_id}"
+            f"{workspace_id}/reports/{dashboard_id}/ReportSection?experience=power-bi"
         )
 
     def _get_chart_url(
@@ -506,11 +507,14 @@ class PowerbiSource(DashboardServiceSource):
                 logger.warning(f"Error to yield datamodel column: {exc}")
         return datasource_columns
 
-    def _get_datamodels_list(self) -> List[Dataset]:
+    def _get_datamodels_list(self) -> List[Union[Dataset, Dataflow]]:
         """
         Get All the Powerbi Datasets
         """
-        return self.context.get().workspace.datasets
+        return (
+            self.context.get().workspace.datasets
+            + self.context.get().workspace.dataflows
+        )
 
     def yield_datamodel(
         self, dashboard_details: Group
@@ -526,6 +530,17 @@ class PowerbiSource(DashboardServiceSource):
                     ):
                         self.status.filter(dataset.name, "Data model filtered out.")
                         continue
+                    if isinstance(dataset, Dataset):
+                        data_model_type = DataModelType.PowerBIDataModel.value
+                        datamodel_columns = self._get_column_info(dataset)
+                    elif isinstance(dataset, Dataflow):
+                        data_model_type = DataModelType.PowerBIDataFlow.value
+                        datamodel_columns = []
+                    else:
+                        logger.warning(
+                            f"Unknown dataset type: {type(dataset)}, name: {dataset.name}"
+                        )
+                        continue
                     data_model_request = CreateDashboardDataModelRequest(
                         name=EntityName(dataset.id),
                         displayName=dataset.name,
@@ -535,9 +550,9 @@ class PowerbiSource(DashboardServiceSource):
                         service=FullyQualifiedEntityName(
                             self.context.get().dashboard_service
                         ),
-                        dataModelType=DataModelType.PowerBIDataModel.value,
+                        dataModelType=data_model_type,
                         serviceType=DashboardServiceType.PowerBI.value,
-                        columns=self._get_column_info(dataset),
+                        columns=datamodel_columns,
                         project=self.get_project_name(dashboard_details=dataset),
                         owners=self.get_owner_ref(dashboard_details=dataset),
                     )
@@ -677,6 +692,30 @@ class PowerbiSource(DashboardServiceSource):
             logger.debug(f"Error to get data_model_column_fqn {exc}")
             logger.debug(traceback.format_exc())
 
+    def _parse_snowflake_regex_exp(
+        self, match: re.Match, datamodel_entity: DashboardDataModel
+    ) -> Optional[str]:
+        """parse snowflake regex expression"""
+        try:
+            if not match:
+                return None
+            elif match.group(1):
+                return match.group(1)
+            elif match.group(2):
+                dataset = self._fetch_dataset_from_workspace(datamodel_entity.name.root)
+                if dataset and dataset.expressions:
+                    # find keyword from dataset expressions
+                    for dexpression in dataset.expressions:
+                        if dexpression.name == match.group(2):
+                            pattern = r'DefaultValue="([^"]+)"'
+                            kw_match = re.search(pattern, dexpression.expression)
+                            if kw_match:
+                                return kw_match.group(1)
+        except Exception as exc:
+            logger.debug(f"Error to parse snowflake regex expression: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
     def _parse_redshift_source(self, source_expression: str) -> Optional[dict]:
         try:
             db_match = re.search(
@@ -693,9 +732,7 @@ class PowerbiSource(DashboardServiceSource):
                 schema = schema_table_match[0] if len(schema_table_match) > 0 else None
                 table = schema_table_match[1] if len(schema_table_match) > 1 else None
 
-            if (
-                schema and table
-            ):  # atlease 2 entities schema and table should be fetched
+            if table:  # atlease table should be fetched
                 return {"database": database, "schema": schema, "table": table}
             return None
         except Exception as exc:
@@ -703,30 +740,32 @@ class PowerbiSource(DashboardServiceSource):
             logger.debug(traceback.format_exc())
         return None
 
-    def _parse_snowflake_source(self, source_expression: str) -> Optional[dict]:
+    def _parse_snowflake_source(
+        self, source_expression: str, datamodel_entity: DashboardDataModel
+    ) -> Optional[dict]:
         try:
             if "Snowflake.Databases" not in source_expression:
                 # Not a snowflake valid expression
                 return None
             db_match = re.search(
-                r'\[Name="([^"]+)",Kind="Database"\]', source_expression
+                r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="Database"\]', source_expression
             )
             schema_match = re.search(
-                r'\[Name="([^"]+)",Kind="Schema"\]', source_expression
+                r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="Schema"\]', source_expression
             )
-            view_match = re.search(r'\[Name="([^"]+)",Kind="View"\]', source_expression)
             table_match = re.search(
-                r'\[Name="([^"]+)",Kind="Table"\]', source_expression
+                r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="Table"\]', source_expression
+            )
+            view_match = re.search(
+                r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="View"\]', source_expression
             )
 
-            database = db_match.group(1) if db_match else None
-            schema = schema_match.group(1) if schema_match else None
-            view = view_match.group(1) if view_match else None
-            table = table_match.group(1) if table_match else None
+            database = self._parse_snowflake_regex_exp(db_match, datamodel_entity)
+            schema = self._parse_snowflake_regex_exp(schema_match, datamodel_entity)
+            table = self._parse_snowflake_regex_exp(table_match, datamodel_entity)
+            view = self._parse_snowflake_regex_exp(view_match, datamodel_entity)
 
-            if schema and (
-                table or view
-            ):  # atlease 2 entities schema and table should be fetched
+            if table or view:  # atlease table or view should be fetched
                 return {
                     "database": database,
                     "schema": schema,
@@ -738,14 +777,18 @@ class PowerbiSource(DashboardServiceSource):
             logger.debug(traceback.format_exc())
         return None
 
-    def _parse_table_info_from_source_exp(self, table: PowerBiTable) -> dict:
+    def _parse_table_info_from_source_exp(
+        self, table: PowerBiTable, datamodel_entity: DashboardDataModel
+    ) -> dict:
         try:
             if not isinstance(table.source, list):
                 return {}
             source_expression = table.source[0].expression
 
             # parse snowflake source
-            table_info = self._parse_snowflake_source(source_expression)
+            table_info = self._parse_snowflake_source(
+                source_expression, datamodel_entity
+            )
             if isinstance(table_info, dict):
                 return table_info
             # parse redshift source
@@ -768,12 +811,12 @@ class PowerbiSource(DashboardServiceSource):
         Method to create lineage between table and datamodels
         """
         try:
-            table_info = self._parse_table_info_from_source_exp(table)
+            table_info = self._parse_table_info_from_source_exp(table, datamodel_entity)
             fqn_search_string = build_es_fqn_search_string(
-                database_name=table_info.get("database"),
-                schema_name=table_info.get("schema"),
                 service_name=db_service_name or "*",
                 table_name=table_info.get("table") or table.name,
+                schema_name=table_info.get("schema") or "*",
+                database_name=table_info.get("database") or "*",
             )
             table_entity = self.metadata.search_in_any_service(
                 entity_type=Table,
@@ -955,27 +998,68 @@ class PowerbiSource(DashboardServiceSource):
         try:
             owner_ref_list = []  # to assign multiple owners to entity if they exist
             for owner in dashboard_details.users or []:
+                owner_ref = None
+                # put filtering conditions
+                if isinstance(dashboard_details, Dataset):
+                    access_right = owner.datasetUserAccessRight
+                elif isinstance(dashboard_details, Dataflow):
+                    access_right = owner.dataflowUserAccessRight
+                elif isinstance(dashboard_details, PowerBIReport):
+                    access_right = owner.reportUserAccessRight
+                elif isinstance(dashboard_details, PowerBIDashboard):
+                    access_right = owner.dashboardUserAccessRight
+
+                if owner.userType != "Member" or (
+                    isinstance(
+                        dashboard_details, (Dataflow, PowerBIReport, PowerBIDashboard)
+                    )
+                    and access_right != "Owner"
+                ):
+                    logger.warning(
+                        f"User is not a member and has no access to the {dashboard_details.id}: ({owner.displayName}, {owner.email})"
+                    )
+                    continue
                 if owner.email:
-                    owner_ref = None
                     try:
                         owner_email = EmailStr._validate(owner.email)
-                        owner_ref = self.metadata.get_reference_by_email(
-                            owner_email.lower()
-                        )
                     except PydanticCustomError:
-                        logger.warning(
-                            f"Could not fetch owner data for email: {owner.email}"
-                        )
-                        if owner.displayName:
-                            owner_ref = self.metadata.get_reference_by_name(
-                                name=owner.displayName
+                        logger.warning(f"Invalid email for owner: {owner.email}")
+                        owner_email = None
+                    if owner_email:
+                        try:
+                            owner_ref = self.metadata.get_reference_by_email(
+                                owner_email.lower()
+                            )
+                        except Exception as err:
+                            logger.warning(
+                                f"Could not fetch owner data with email {owner.email} in {dashboard_details.id}: {err}"
                             )
+                elif owner.displayName:
+                    try:
+                        owner_ref = self.metadata.get_reference_by_name(
+                            name=owner.displayName
+                        )
                     except Exception as err:
                         logger.warning(
-                            f"Error processing current owner data in {dashboard_details.id}: {err}"
+                            f"Could not process owner data with name {owner.displayName} in {dashboard_details.id}: {err}"
                         )
-                    if owner_ref:
+                if owner_ref:
+                    owner_ref_list.append(owner_ref.root[0])
+            # check for last modified, configuredBy user
+            current_active_user = None
+            if isinstance(dashboard_details, Dataset):
+                current_active_user = dashboard_details.configuredBy
+            elif isinstance(dashboard_details, (Dataflow, PowerBIReport)):
+                current_active_user = dashboard_details.modifiedBy
+            if current_active_user:
+                try:
+                    owner_ref = self.metadata.get_reference_by_email(
+                        current_active_user.lower()
+                    )
+                    if owner_ref and owner_ref.root[0] not in owner_ref_list:
                         owner_ref_list.append(owner_ref.root[0])
+                except Exception as err:
+                    logger.warning(f"Could not fetch owner data due to {err}")
             if len(owner_ref_list) > 0:
                 logger.debug(
                     f"Successfully fetched owners data for {dashboard_details.id}"

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -39,6 +39,11 @@ class PowerBIUser(BaseModel):
 
     displayName: Optional[str] = None
     email: Optional[str] = Field(alias="emailAddress", default=None)
+    userType: Optional[str] = None
+    reportUserAccessRight: Optional[str] = None
+    datasetUserAccessRight: Optional[str] = None
+    dataflowUserAccessRight: Optional[str] = None
+    dashboardUserAccessRight: Optional[str] = None
 
 
 class PowerBIDashboard(BaseModel):
@@ -65,6 +70,7 @@ class PowerBIReport(BaseModel):
     name: str
     datasetId: Optional[str] = None
     users: Optional[List[PowerBIUser]] = []
+    modifiedBy: Optional[str] = None
 
 
 class DashboardsResponse(BaseModel):
@@ -163,6 +169,11 @@ class TablesResponse(BaseModel):
     value: List[PowerBiTable]
 
 
+class DatasetExpression(BaseModel):
+    name: str
+    expression: str
+
+
 class Dataset(BaseModel):
     """
     PowerBI Dataset Model
@@ -174,6 +185,8 @@ class Dataset(BaseModel):
     tables: Optional[List[PowerBiTable]] = []
     description: Optional[str] = None
     users: Optional[List[PowerBIUser]] = []
+    expressions: Optional[List[DatasetExpression]] = []
+    configuredBy: Optional[str] = None
 
 
 class DatasetResponse(BaseModel):
@@ -186,6 +199,14 @@ class DatasetResponse(BaseModel):
     value: List[Dataset]
 
 
+class Dataflow(BaseModel):
+    id: str = Field(alias="objectId")
+    name: str
+    description: Optional[str] = None
+    users: Optional[List[PowerBIUser]] = []
+    modifiedBy: Optional[str] = None
+
+
 class Group(BaseModel):
     """
     PowerBI Group Model
@@ -199,6 +220,7 @@ class Group(BaseModel):
     dashboards: Optional[List[PowerBIDashboard]] = []
     reports: Optional[List[PowerBIReport]] = []
     datasets: Optional[List[Dataset]] = []
+    dataflows: Optional[List[Dataflow]] = []
 
 
 class GroupsResponse(BaseModel):