openmetadata-ingestion 1.5.15.0__py3-none-any.whl → 1.5.15.1__py3-none-any.whl

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-12-17T04:30:29+00:00
+#   timestamp: 2024-12-18T16:09:51+00:00
 
 from __future__ import annotations
 

metadata/ingestion/ometa/mixins/user_mixin.py

@@ -13,6 +13,7 @@ Mixin class containing User specific methods
 
 To be used by OpenMetadata class
 """
+import json
 from functools import lru_cache
 from typing import Optional, Type
 
@@ -46,16 +47,29 @@ class OMetaUserMixin:
         )
 
     @staticmethod
-    def name_search_query_es(entity: Type[T]) -> str:
+    def name_search_query_es(entity: Type[T], name: str, from_: int, size: int) -> str:
         """
         Allow for more flexible lookup following what the UI is doing when searching users.
 
         We don't want to stick to `q=name:{name}` since in case a user is named `random.user`
         but looked as `Random User`, we want to find this match.
+
+        Search should only look in name and displayName fields and should not return bots.
         """
+        query_filter = {
+            "query": {
+                "query_string": {
+                    "query": f"{name} AND isBot:false",
+                    "fields": ["name", "displayName"],
+                    "default_operator": "AND",
+                    "fuzziness": "AUTO",
+                }
+            }
+        }
+
         return (
-            "/search/query?q={name} AND isBot:false&from={from_}&size={size}&index="
-            + ES_INDEX_MAP[entity.__name__]
+            f"""/search/query?query_filter={json.dumps(query_filter)}"""
+            f"&from={from_}&size={size}&index=" + ES_INDEX_MAP[entity.__name__]
         )
 
     def _search_by_email(
@@ -103,8 +117,8 @@ class OMetaUserMixin:
             fields: Optional field list to pass to ES request
         """
         if name:
-            query_string = self.name_search_query_es(entity=entity).format(
-                name=name, from_=from_count, size=size
+            query_string = self.name_search_query_es(
+                entity=entity, name=name, from_=from_count, size=size
             )
             return self.get_entity_from_es(
                 entity=entity, query_string=query_string, fields=fields

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -127,7 +127,13 @@ class PowerBiApiClient:
             List[PowerBIDashboard]
         """
         try:
-            response_data = self.client.get(f"/myorg/groups/{group_id}/dashboards")
+            admin = "admin/" if self.config.useAdminApis else ""
+            response_data = self.client.get(
+                f"/myorg/{admin}groups/{group_id}/dashboards"
+            )
+            if not response_data:
+                logger.debug(f"No dashboards found for workspace_id: {group_id}")
+                return None
             response = DashboardsResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -142,7 +148,11 @@ class PowerBiApiClient:
             List[PowerBIReport]
         """
         try:
-            response_data = self.client.get(f"/myorg/groups/{group_id}/reports")
+            admin = "admin/" if self.config.useAdminApis else ""
+            response_data = self.client.get(f"/myorg/{admin}groups/{group_id}/reports")
+            if not response_data:
+                logger.debug(f"No reports found for workspace_id: {group_id}")
+                return None
             response = ReportsResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -157,7 +167,11 @@ class PowerBiApiClient:
             List[Dataset]
         """
         try:
-            response_data = self.client.get(f"/myorg/groups/{group_id}/datasets")
+            admin = "admin/" if self.config.useAdminApis else ""
+            response_data = self.client.get(f"/myorg/{admin}groups/{group_id}/datasets")
+            if not response_data:
+                logger.debug(f"No datasets found for workspace_id: {group_id}")
+                return None
             response = DatasetResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -174,9 +188,13 @@ class PowerBiApiClient:
             List[Tile]
         """
         try:
+            admin = "admin/" if self.config.useAdminApis else ""
             response_data = self.client.get(
-                f"/myorg/groups/{group_id}/dashboards/{dashboard_id}/tiles"
+                f"/myorg/{admin}dashboards/{dashboard_id}/tiles"
             )
+            if not response_data:
+                logger.debug(f"No dashboard tiles found for workspace_id: {group_id}")
+                return None
             response = TilesResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -92,12 +92,6 @@ class PowerbiSource(DashboardServiceSource):
         self.datamodel_file_mappings = []
 
     def prepare(self):
-        if self.service_connection.useAdminApis:
-            groups = self.get_admin_workspace_data()
-        else:
-            groups = self.get_org_workspace_data()
-        if groups:
-            self.workspace_data = self.get_filtered_workspaces(groups)
         return super().prepare()
 
     def close(self):
@@ -123,91 +117,6 @@ class PowerbiSource(DashboardServiceSource):
             filtered_groups.append(group)
         return filtered_groups
 
-    def get_org_workspace_data(self) -> Optional[List[Group]]:
-        """
-        fetch all the group workspace ids
-        """
-        groups = self.client.api_client.fetch_all_workspaces()
-        for group in groups:
-            # add the dashboards to the groups
-            group.dashboards.extend(
-                self.client.api_client.fetch_all_org_dashboards(group_id=group.id) or []
-            )
-            for dashboard in group.dashboards:
-                # add the tiles to the dashboards
-                dashboard.tiles.extend(
-                    self.client.api_client.fetch_all_org_tiles(
-                        group_id=group.id, dashboard_id=dashboard.id
-                    )
-                    or []
-                )
-
-            # add the reports to the groups
-            group.reports.extend(
-                self.client.api_client.fetch_all_org_reports(group_id=group.id) or []
-            )
-
-            # add the datasets to the groups
-            group.datasets.extend(
-                self.client.api_client.fetch_all_org_datasets(group_id=group.id) or []
-            )
-            for dataset in group.datasets:
-                # add the tables to the datasets
-                dataset.tables.extend(
-                    self.client.api_client.fetch_dataset_tables(
-                        group_id=group.id, dataset_id=dataset.id
-                    )
-                    or []
-                )
-        return groups
-
-    def get_admin_workspace_data(self) -> Optional[List[Group]]:
-        """
-        fetch all the workspace ids
-        """
-        groups = []
-        workspaces = self.client.api_client.fetch_all_workspaces()
-        if workspaces:
-            workspace_id_list = [workspace.id for workspace in workspaces]
-
-            # Start the scan of the available workspaces for dashboard metadata
-            workspace_paginated_list = [
-                workspace_id_list[i : i + self.pagination_entity_per_page]
-                for i in range(
-                    0, len(workspace_id_list), self.pagination_entity_per_page
-                )
-            ]
-            count = 1
-            for workspace_ids_chunk in workspace_paginated_list:
-                logger.info(
-                    f"Scanning {count}/{len(workspace_paginated_list)} set of workspaces"
-                )
-                workspace_scan = self.client.api_client.initiate_workspace_scan(
-                    workspace_ids_chunk
-                )
-
-                # Keep polling the scan status endpoint to check if scan is succeeded
-                workspace_scan_status = self.client.api_client.wait_for_scan_complete(
-                    scan_id=workspace_scan.id
-                )
-                if workspace_scan_status:
-                    response = self.client.api_client.fetch_workspace_scan_result(
-                        scan_id=workspace_scan.id
-                    )
-                    groups.extend(
-                        [
-                            active_workspace
-                            for active_workspace in response.workspaces
-                            if active_workspace.state == "Active"
-                        ]
-                    )
-                else:
-                    logger.error("Error in fetching dashboards and charts")
-                count += 1
-        else:
-            logger.error("Unable to fetch any PowerBI workspaces")
-        return groups or None
-
     @classmethod
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
@@ -220,12 +129,51 @@ class PowerbiSource(DashboardServiceSource):
             )
         return cls(config, metadata)
 
+    def prepare_workspace_data(self, workspace: Group):
+        """prepare one workspace data at a time"""
+        # add the dashboards to the groups
+        workspace.dashboards.extend(
+            self.client.api_client.fetch_all_org_dashboards(group_id=workspace.id) or []
+        )
+        for dashboard in workspace.dashboards:
+            # add the tiles to the dashboards
+            dashboard.tiles.extend(
+                self.client.api_client.fetch_all_org_tiles(
+                    group_id=workspace.id, dashboard_id=dashboard.id
+                )
+                or []
+            )
+
+        # add the reports to the groups
+        workspace.reports.extend(
+            self.client.api_client.fetch_all_org_reports(group_id=workspace.id) or []
+        )
+
+        # add the datasets to the groups
+        workspace.datasets.extend(
+            self.client.api_client.fetch_all_org_datasets(group_id=workspace.id) or []
+        )
+        for dataset in workspace.datasets:
+            # add the tables to the datasets
+            dataset.tables.extend(
+                self.client.api_client.fetch_dataset_tables(
+                    group_id=workspace.id, dataset_id=dataset.id
+                )
+                or []
+            )
+
     def get_dashboard(self) -> Any:
         """
         Method to iterate through dashboard lists filter dashboards & yield dashboard details
         """
-        for workspace in self.workspace_data:
+        # fetch all workspaces/groups & apply filter pattern
+        all_workspaces = self.client.api_client.fetch_all_workspaces()
+        all_workspaces = self.get_filtered_workspaces(all_workspaces)
+        for workspace in all_workspaces:
+            # prepare additional data for specific workspace (datasets, reports, dashboards)
+            self.prepare_workspace_data(workspace)
             self.context.get().workspace = workspace
+            self.workspace_data.append(workspace)
             for dashboard in self.get_dashboards_list():
                 try:
                     dashboard_details = self.get_dashboard_details(dashboard)
@@ -309,54 +257,45 @@ class PowerbiSource(DashboardServiceSource):
             f"{workspace_id}/{chart_url_postfix}"
         )
 
-    def list_datamodels(self) -> Iterable[Dataset]:
+    def yield_datamodel(
+        self, dashboard_details: Union[PowerBIDashboard, PowerBIReport]
+    ) -> Iterable[Either[CreateDashboardRequest]]:
         """
-        Get All the Powerbi Datasets
+        Method to yield datamodel for each workspace
         """
-        if self.source_config.includeDataModels:
+        workspace_datasets = self.context.get().workspace.datasets
+        for dataset in workspace_datasets:
+            if filter_by_datamodel(
+                self.source_config.dataModelFilterPattern, dataset.name
+            ):
+                self.status.filter(dataset.name, "Data model filtered out.")
+                continue
             try:
-                for workspace in self.workspace_data:
-                    for dataset in workspace.datasets or []:
-                        if filter_by_datamodel(
-                            self.source_config.dataModelFilterPattern, dataset.name
-                        ):
-                            self.status.filter(dataset.name, "Data model filtered out.")
-                            continue
-                        yield dataset
-            except Exception as err:
-                logger.debug(traceback.format_exc())
-                logger.error(f"Unexpected error fetching PowerBI datasets - {err}")
-
-    def yield_bulk_datamodel(
-        self, dataset: Dataset
-    ) -> Iterable[Either[CreateDashboardDataModelRequest]]:
-        """
-        Method to fetch DataModels in bulk
-        """
-        try:
-            data_model_request = CreateDashboardDataModelRequest(
-                name=EntityName(dataset.id),
-                displayName=dataset.name,
-                description=Markdown(dataset.description)
-                if dataset.description
-                else None,
-                service=FullyQualifiedEntityName(self.context.get().dashboard_service),
-                dataModelType=DataModelType.PowerBIDataModel.value,
-                serviceType=DashboardServiceType.PowerBI.value,
-                columns=self._get_column_info(dataset),
-                project=self._fetch_dataset_workspace(dataset_id=dataset.id),
-            )
-            yield Either(right=data_model_request)
-            self.register_record_datamodel(datamodel_request=data_model_request)
+                data_model_request = CreateDashboardDataModelRequest(
+                    name=EntityName(dataset.id),
+                    displayName=dataset.name,
+                    description=Markdown(dataset.description)
+                    if dataset.description
+                    else None,
+                    service=FullyQualifiedEntityName(
+                        self.context.get().dashboard_service
+                    ),
+                    dataModelType=DataModelType.PowerBIDataModel.value,
+                    serviceType=DashboardServiceType.PowerBI.value,
+                    columns=self._get_column_info(dataset),
+                    project=self.get_project_name(dashboard_details),
+                )
+                yield Either(right=data_model_request)
+                self.register_record_datamodel(datamodel_request=data_model_request)
 
-        except Exception as exc:
-            yield Either(
-                left=StackTraceError(
-                    name=dataset.name,
-                    error=f"Error yielding Data Model [{dataset.name}]: {exc}",
-                    stackTrace=traceback.format_exc(),
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name=dataset.name,
+                        error=f"Error yielding Data Model [{dataset.name}]: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
                 )
-            )
 
     def _get_child_columns(self, table: PowerBiTable) -> List[Column]:
         """

metadata/ingestion/source/database/common_db_source.py

@@ -225,6 +225,7 @@ class CommonDbSourceService(
             yield self.service_connection.databaseSchema
         else:
             for schema_name in self.inspector.get_schema_names():
+                logger.info(f"Processing Schema: {schema_name}")
                 yield schema_name
 
     def get_database_schema_names(self) -> Iterable[str]:

metadata/ingestion/source/database/dbt/metadata.py

@@ -61,6 +61,7 @@ from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.models.table_metadata import ColumnDescription
+from metadata.ingestion.ometa.client import APIError
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.ingestion.source.database.database_service import DataModelLink
@@ -437,7 +438,7 @@ class DbtSource(DbtServiceSource):
                         table_name=model_name,
                     )
 
-                    table_entity: Optional[
+                    table_entities: Optional[
                         Union[Table, List[Table]]
                     ] = get_entity_from_es_result(
                         entity_list=self.metadata.es_search_from_fqn(
@@ -445,10 +446,20 @@ class DbtSource(DbtServiceSource):
                             fqn_search_string=table_fqn,
                             fields="sourceHash",
                         ),
-                        fetch_multiple_entities=False,
+                        fetch_multiple_entities=True,
+                    )
+                    logger.debug(
+                        f"Found table entities from {table_fqn}: {table_entities}"
+                    )
+
+                    table_entity = next(
+                        iter(filter(lambda x: x is not None, table_entities)), None
                     )
 
                     if table_entity:
+                        logger.debug(
+                            f"Using Table Entity for datamodel: {table_entity}"
+                        )
                         data_model_link = DataModelLink(
                             table_entity=table_entity,
                             datamodel=DataModel(
@@ -881,6 +892,7 @@ class DbtSource(DbtServiceSource):
                         self.metadata, entity_link_str
                     )
                     table_fqn = get_table_fqn(entity_link_str)
+                    logger.debug(f"Table fqn found: {table_fqn}")
                     source_elements = table_fqn.split(fqn.FQN_SEPARATOR)
                     test_case_fqn = fqn.build(
                         self.metadata,
@@ -916,6 +928,7 @@ class DbtSource(DbtServiceSource):
                                 owners=None,
                             )
                         )
+                    logger.debug(f"Test case Already Exists: {test_case_fqn}")
         except Exception as err:  # pylint: disable=broad-except
             yield Either(
                 left=StackTraceError(
@@ -1005,10 +1018,17 @@ class DbtSource(DbtServiceSource):
                         else None,
                         test_case_name=manifest_node.name,
                     )
-                    self.metadata.add_test_case_results(
-                        test_results=test_case_result,
-                        test_case_fqn=test_case_fqn,
-                    )
+
+                    logger.debug(f"Adding test case results to {test_case_fqn} ")
+                    try:
+                        self.metadata.add_test_case_results(
+                            test_results=test_case_result,
+                            test_case_fqn=test_case_fqn,
+                        )
+                    except APIError as err:
+                        if err.code != 409:
+                            raise APIError(err) from err
+
         except Exception as err:  # pylint: disable=broad-except
             logger.debug(traceback.format_exc())
             logger.error(

metadata/ingestion/source/database/external_table_lineage_mixin.py

@@ -43,6 +43,7 @@ class ExternalTableLineageMixin(ABC):
         """
         Yield external table lineage
         """
+        logger.info("Processing Lineage for External Tables")
         for table_qualified_tuple, location in self.external_location_map.items() or []:
             try:
                 location_entity = self.metadata.es_search_container_by_path(

metadata/ingestion/source/database/snowflake/metadata.py

@@ -316,7 +316,7 @@ class SnowflakeSource(
                 ):
                     self.status.filter(database_fqn, "Database Filtered Out")
                     continue
-
+                logger.debug(f"Processing database {new_database}")
                 try:
                     self.set_inspector(database_name=new_database)
                     self.set_session_query_tag()

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -147,6 +147,7 @@ class StoredProcedureMixin(ABC):
         self, query_by_procedure: QueryByProcedure, procedure: StoredProcedure
     ) -> Iterable[Either[AddLineageRequest]]:
         """Add procedure lineage from its query"""
+        logger.debug("Processing Lineage for Stored Procedure Query")
         self.context.get().stored_procedure_query_lineage = False
         if self.is_lineage_query(
             query_type=query_by_procedure.query_type,
@@ -203,6 +204,7 @@ class StoredProcedureMixin(ABC):
         self,
     ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
         """Get all the queries and procedures list and yield them"""
+        logger.debug("Processing Lineage for Stored Procedure")
         if self.context.get().stored_procedures:
             logger.info("Processing Lineage for Stored Procedures")
             # First, get all the query history