openmetadata-ingestion 1.8.0.0__py3-none-any.whl → 1.8.2.0__py3-none-any.whl

metadata/ingestion/source/dashboard/metabase/metadata.py

@@ -210,9 +210,11 @@ class MetabaseSource(DashboardServiceSource):
                 name=EntityName(str(dashboard_details.id)),
                 sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.name,
-                description=Markdown(dashboard_details.description)
-                if dashboard_details.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.description)
+                    if dashboard_details.description
+                    else None
+                ),
                 project=self.context.get().project_name,
                 charts=[
                     FullyQualifiedEntityName(
@@ -286,7 +288,7 @@ class MetabaseSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: MetabaseDashboardDetails,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """Get lineage method
 
@@ -308,7 +310,7 @@ class MetabaseSource(DashboardServiceSource):
                 if chart_details.dataset_query.type == "native":
                     yield from self._yield_lineage_from_query(
                         chart_details=chart_details,
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         dashboard_name=dashboard_name,
                     ) or []
 
@@ -319,7 +321,7 @@ class MetabaseSource(DashboardServiceSource):
                         continue
                     yield from self._yield_lineage_from_api(
                         chart_details=chart_details,
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         dashboard_name=dashboard_name,
                     ) or []
 
@@ -341,11 +343,18 @@ class MetabaseSource(DashboardServiceSource):
     def _yield_lineage_from_query(
         self,
         chart_details: MetabaseChart,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
         dashboard_name: str,
     ) -> Iterable[Either[AddLineageRequest]]:
         database = self.client.get_database(chart_details.database_id)
 
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         query = None
         if (
             chart_details.dataset_query
@@ -359,23 +368,54 @@ class MetabaseSource(DashboardServiceSource):
 
         database_name = database.details.db if database and database.details else None
 
-        db_service = self._get_database_service(db_service_name)
+        db_service = self._get_database_service(prefix_service_name)
 
         lineage_parser = LineageParser(
             query,
-            ConnectionTypeDialectMapper.dialect_of(db_service.serviceType.value)
-            if db_service
-            else Dialect.ANSI,
+            (
+                ConnectionTypeDialectMapper.dialect_of(db_service.serviceType.value)
+                if db_service
+                else Dialect.ANSI
+            ),
         )
 
+        if (
+            prefix_database_name
+            and database_name
+            and prefix_database_name.lower() != database_name.lower()
+        ):
+            logger.debug(
+                f"Database {database_name} does not match prefix {prefix_database_name}"
+            )
+            return
+
         for table in lineage_parser.source_tables:
             database_schema_name, table = fqn.split(str(table))[-2:]
             database_schema_name = self.check_database_schema_name(database_schema_name)
+
+            if (
+                prefix_table_name
+                and table
+                and prefix_table_name.lower() != table.lower()
+            ):
+                logger.debug(f"Table {table} does not match prefix {prefix_table_name}")
+                continue
+
+            if (
+                prefix_schema_name
+                and database_schema_name
+                and prefix_schema_name.lower() != database_schema_name.lower()
+            ):
+                logger.debug(
+                    f"Schema {database_schema_name} does not match prefix {prefix_schema_name}"
+                )
+                continue
+
             fqn_search_string = build_es_fqn_search_string(
-                database_name=database_name,
-                schema_name=database_schema_name,
-                service_name=db_service_name or "*",
-                table_name=table,
+                database_name=prefix_database_name or database_name,
+                schema_name=prefix_schema_name or database_schema_name,
+                service_name=prefix_service_name or "*",
+                table_name=prefix_table_name or table,
             )
             from_entities = self.metadata.search_in_any_service(
                 entity_type=Table,
@@ -401,7 +441,7 @@ class MetabaseSource(DashboardServiceSource):
     def _yield_lineage_from_api(
         self,
         chart_details: MetabaseChart,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
         dashboard_name: str,
     ) -> Iterable[Either[AddLineageRequest]]:
         table = self.client.get_table(chart_details.table_id)
@@ -410,12 +450,50 @@ class MetabaseSource(DashboardServiceSource):
         if table is None or table_name is None:
             return
 
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         database_name = table.db.details.db if table.db and table.db.details else None
+
+        if (
+            prefix_table_name
+            and table_name
+            and prefix_table_name.lower() != table_name.lower()
+        ):
+            logger.debug(
+                f"Table {table_name} does not match prefix {prefix_table_name}"
+            )
+            return
+
+        if (
+            prefix_schema_name
+            and table.table_schema
+            and prefix_schema_name.lower() != table.table_schema.lower()
+        ):
+            logger.debug(
+                f"Schema {table.table_schema} does not match prefix {prefix_schema_name}"
+            )
+            return
+
+        if (
+            prefix_database_name
+            and database_name
+            and prefix_database_name.lower() != database_name.lower()
+        ):
+            logger.debug(
+                f"Database {database_name} does not match prefix {prefix_database_name}"
+            )
+            return
+
         fqn_search_string = build_es_fqn_search_string(
-            database_name=database_name,
-            schema_name=table.table_schema,
-            service_name=db_service_name or "*",
-            table_name=table_name,
+            service_name=prefix_service_name or "*",
+            database_name=prefix_database_name or database_name,
+            schema_name=prefix_schema_name or table.table_schema,
+            table_name=prefix_table_name or table_name,
         )
         from_entities = self.metadata.search_in_any_service(
             entity_type=Table,

metadata/ingestion/source/dashboard/microstrategy/metadata.py

@@ -184,16 +184,23 @@ class MicrostrategySource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: MstrDashboardDetails,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Optional[Iterable[AddLineageRequest]]:
         """
-        Get lineage between datamodel and data sources
+        Get lineage between dashboard and data sources
         """
-        if not db_service_name:
+        if not db_service_prefix:
             return
 
+        (
+            prefix_db_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         database_service = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=db_service_name
+            entity=DatabaseService, fqn=prefix_db_service_name
         )
         dialect = ConnectionTypeDialectMapper.dialect_of(
             database_service.serviceType.value
@@ -221,7 +228,7 @@ class MicrostrategySource(DashboardServiceSource):
                 for table in lineage_parser.source_tables:
                     table_entities = get_table_entities_from_query(
                         metadata=self.metadata,
-                        service_name=db_service_name,
+                        service_name=prefix_db_service_name,
                         database_name="*",
                         database_schema="*",
                         table_name=str(table),
@@ -230,6 +237,29 @@ class MicrostrategySource(DashboardServiceSource):
                         logger.debug(f"Table not found in metadata: {str(table)}")
                         continue
                     for table_entity in table_entities or []:
+                        if (
+                            prefix_table_name
+                            and prefix_table_name.lower()
+                            != str(table_entity.name.root).lower()
+                        ):
+                            continue
+
+                        if (
+                            prefix_schema_name
+                            and getattr(table_entity.databaseSchema, "name", None)
+                            and prefix_schema_name.lower()
+                            != str(table_entity.databaseSchema.name).lower()
+                        ):
+                            continue
+
+                        if (
+                            prefix_database_name
+                            and getattr(table_entity.database, "name", None)
+                            and prefix_database_name.lower()
+                            != str(table_entity.database.name).lower()
+                        ):
+                            continue
+
                         yield Either(
                             right=AddLineageRequest(
                                 edge=EntitiesEdge(

metadata/ingestion/source/dashboard/mode/metadata.py

@@ -110,9 +110,11 @@ class ModeSource(DashboardServiceSource):
             name=EntityName(dashboard_details.get(client.TOKEN)),
             sourceUrl=SourceUrl(dashboard_url),
             displayName=dashboard_details.get(client.NAME),
-            description=Markdown(dashboard_details.get(client.DESCRIPTION))
-            if dashboard_details.get(client.DESCRIPTION)
-            else None,
+            description=(
+                Markdown(dashboard_details.get(client.DESCRIPTION))
+                if dashboard_details.get(client.DESCRIPTION)
+                else None
+            ),
             charts=[
                 FullyQualifiedEntityName(
                     fqn.build(
@@ -130,51 +132,95 @@ class ModeSource(DashboardServiceSource):
         yield Either(right=dashboard_request)
         self.register_record(dashboard_request=dashboard_request)
 
+    # pylint: disable=too-many-locals
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: dict,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """Get lineage method"""
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         try:
             response_queries = self.client.get_all_queries(
                 workspace_name=self.workspace_name,
                 report_token=dashboard_details[client.TOKEN],
             )
             queries = response_queries[client.EMBEDDED][client.QUERIES]
+
             for query in queries:
                 if not query.get("data_source_id"):
                     continue
                 data_source = self.data_sources.get(query.get("data_source_id"))
                 if not data_source:
                     continue
+
+                database_name = data_source.get(client.DATABASE)
+                if (
+                    prefix_database_name
+                    and database_name
+                    and prefix_database_name.lower() != str(database_name).lower()
+                ):
+                    logger.debug(
+                        f"Database {database_name} does not match prefix {prefix_database_name}"
+                    )
+                    continue
+
                 lineage_parser = LineageParser(query.get("raw_query"))
                 for table in lineage_parser.source_tables:
                     database_schema_name, table = fqn.split(str(table))[-2:]
                     database_schema_name = self.check_database_schema_name(
                         database_schema_name
                     )
+
+                    if (
+                        prefix_table_name
+                        and table
+                        and prefix_table_name.lower() != str(table).lower()
+                    ):
+                        logger.debug(
+                            f"Table {table} does not match prefix {prefix_table_name}"
+                        )
+                        continue
+
+                    if (
+                        prefix_schema_name
+                        and database_schema_name
+                        and prefix_schema_name.lower()
+                        != str(database_schema_name).lower()
+                    ):
+                        logger.debug(
+                            f"Schema {database_schema_name} does not match prefix {prefix_schema_name}"
+                        )
+                        continue
+
                     fqn_search_string = build_es_fqn_search_string(
-                        database_name=data_source.get(client.DATABASE),
-                        schema_name=database_schema_name,
-                        service_name=db_service_name or "*",
-                        table_name=table,
+                        database_name=prefix_database_name or database_name,
+                        schema_name=prefix_schema_name or database_schema_name,
+                        service_name=prefix_service_name or "*",
+                        table_name=prefix_table_name or table,
                     )
                     from_entities = self.metadata.search_in_any_service(
                         entity_type=Table,
                         fqn_search_string=fqn_search_string,
                         fetch_multiple_entities=True,
                     )
+
+                    to_entity = self.metadata.get_by_name(
+                        entity=Lineage_Dashboard,
+                        fqn=fqn.build(
+                            self.metadata,
+                            Lineage_Dashboard,
+                            service_name=self.config.serviceName,
+                            dashboard_name=dashboard_details.get(client.TOKEN),
+                        ),
+                    )
                     for from_entity in from_entities or []:
-                        to_entity = self.metadata.get_by_name(
-                            entity=Lineage_Dashboard,
-                            fqn=fqn.build(
-                                self.metadata,
-                                Lineage_Dashboard,
-                                service_name=self.config.serviceName,
-                                dashboard_name=dashboard_details.get(client.TOKEN),
-                            ),
-                        )
                         yield self._get_add_lineage_request(
                             to_entity=to_entity, from_entity=from_entity
                         )
@@ -182,7 +228,7 @@ class ModeSource(DashboardServiceSource):
             yield Either(
                 left=StackTraceError(
                     name="Lineage",
-                    error=f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}",
+                    error=f"Error to yield dashboard lineage details for service name [{prefix_service_name}]: {exc}",
                     stackTrace=traceback.format_exc(),
                 )
             )

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -164,6 +164,10 @@ class PowerBiApiClient:
             List[PowerBIDashboard]
         """
         if self.config.useAdminApis:
+            logger.debug(
+                f"Calling the API({str(self.client._base_url)}/myorg/admin/dashboards)"  # pylint: disable=protected-access
+                " to get dashboards"
+            )
             response_data = self.client.get("/myorg/admin/dashboards")
             response = DashboardsResponse(**response_data)
             return response.value
@@ -178,6 +182,10 @@ class PowerBiApiClient:
             List[PowerBIDashboard]
         """
         try:
+            logger.debug(
+                f"Calling the API({str(self.client._base_url)}/myorg/groups/{group_id}/dashboards)"  # pylint: disable=protected-access
+                " to get group dashboards"
+            )
             response_data = self.client.get(f"/myorg/groups/{group_id}/dashboards")
             response = DashboardsResponse(**response_data)
             return response.value
@@ -193,6 +201,10 @@ class PowerBiApiClient:
             List[PowerBIReport]
         """
         try:
+            logger.debug(
+                f"Calling the API({str(self.client._base_url)}/myorg/groups/{group_id}/reports)"  # pylint: disable=protected-access
+                " to get group reports"
+            )
             response_data = self.client.get(f"/myorg/groups/{group_id}/reports")
             response = ReportsResponse(**response_data)
             return response.value
@@ -208,6 +220,10 @@ class PowerBiApiClient:
             List[Dataset]
         """
         try:
+            logger.debug(
+                f"Calling the API({str(self.client._base_url)}/myorg/groups/{group_id}/datasets)"  # pylint: disable=protected-access
+                " to get group datasets"
+            )
             response_data = self.client.get(f"/myorg/groups/{group_id}/datasets")
             response = DatasetResponse(**response_data)
             return response.value
@@ -225,6 +241,10 @@ class PowerBiApiClient:
             List[Tile]
         """
         try:
+            logger.debug(
+                f"Calling the API({str(self.client._base_url)}/myorg/groups/{group_id}/dashboards/{dashboard_id}/tiles)"  # pylint: disable=protected-access
+                " to get dashboard tiles"
+            )
             response_data = self.client.get(
                 f"/myorg/groups/{group_id}/dashboards/{dashboard_id}/tiles"
             )
@@ -244,6 +264,10 @@ class PowerBiApiClient:
             List[PowerBiTable]
         """
         try:
+            logger.debug(
+                f"Calling the API({str(self.client._base_url)}/myorg/groups/{group_id}/datasets/{dataset_id}/tables)"  # pylint: disable=protected-access
+                " to get dataset tables"
+            )
             response_data = self.client.get(
                 f"/myorg/groups/{group_id}/datasets/{dataset_id}/tables"
             )
@@ -357,6 +381,11 @@ class PowerBiApiClient:
             params_data = deepcopy(GETGROUPS_DEFAULT_PARAMS)
             if parsed_filter_query:
                 params_data["$filter"] = parsed_filter_query
+            logger.debug(
+                f"Calling the API({str(self.client._base_url)}/myorg/{admin}groups)"  # pylint: disable=protected-access
+                " to get workspaces(initial call to get count of workspaces and then"
+                " further paginate all workspace calls)"
+            )
             response = self.client.get(api_url, data=params_data)
             if (
                 not response
@@ -391,17 +420,20 @@ class PowerBiApiClient:
                 }
                 if parsed_filter_query:
                     params_data["$filter"] = parsed_filter_query
-
+                index_range = (
+                    int(params_data.get("$skip")),
+                    int(params_data.get("$skip")) + int(params_data.get("$top")),
+                )
+                logger.debug(
+                    f"Calling the API({str(self.client._base_url)}/myorg/{admin}groups)"  # pylint: disable=protected-access
+                    f" to get workspaces between results: {str(index_range)}"
+                )
                 response = self.client.get(api_url, data=params_data)
                 if (
                     not response
                     or API_RESPONSE_MESSAGE_KEY in response
                     or len(response) != len(GroupsResponse.__annotations__)
                 ):
-                    index_range = (
-                        int(params_data.get("$skip")),
-                        int(params_data.get("$skip")) + int(params_data.get("$top")),
-                    )
                     logger.warning(
                         f"Error fetching workspaces between results: {str(index_range)}"
                     )
@@ -423,17 +455,20 @@ class PowerBiApiClient:
                     "Retrying one more time on failed indexes to get workspaces"
                 )
                 for params_data in failed_indexes:
+                    index_range = (
+                        int(params_data.get("$skip")),
+                        int(params_data.get("$skip")) + int(params_data.get("$top")),
+                    )
+                    logger.debug(
+                        f"Calling the API({str(self.client._base_url)}/myorg/{admin}groups)"  # pylint: disable=protected-access
+                        f" to get workspaces between results: {str(index_range)}"
+                    )
                     response = self.client.get(api_url, data=params_data)
                     if (
                         not response
                         or API_RESPONSE_MESSAGE_KEY in response
                         or len(response) != len(GroupsResponse.__annotations__)
                     ):
-                        index_range = (
-                            int(params_data.get("$skip")),
-                            int(params_data.get("$skip"))
-                            + int(params_data.get("$top")),
-                        )
                         logger.warning(
                             f"Workspaces between results {str(index_range)} "
                             "could not be fetched on multiple attempts"
@@ -471,6 +506,10 @@ class PowerBiApiClient:
                 "datasetExpressions=True&datasetSchema=True"
                 "&datasourceDetails=True&getArtifactUsers=True&lineage=True"
             )
+            logger.debug(
+                f"Calling the API({str(self.client._base_url)}{path})"  # pylint: disable=protected-access
+                " to initiate workspace scan"
+            )
             response_data = self.client.post(path=path, data=data)
             return WorkSpaceScanResponse(**response_data)
         except Exception as exc:  # pylint: disable=broad-except
@@ -489,6 +528,10 @@ class PowerBiApiClient:
             WorkSpaceScanResponse
         """
         try:
+            logger.debug(
+                f"Calling the API({str(self.client._base_url)}/myorg/admin/workspaces/scanStatus/{scan_id})"  # pylint: disable=protected-access
+                " to get workspace scan status"
+            )
             response_data = self.client.get(
                 f"/myorg/admin/workspaces/scanStatus/{scan_id}"
             )
@@ -507,6 +550,10 @@ class PowerBiApiClient:
             Workspaces
         """
         try:
+            logger.debug(
+                f"Calling the API({str(self.client._base_url)}/myorg/admin/workspaces/scanResult/{scan_id})"  # pylint: disable=protected-access
+                " to get workspace scan result"
+            )
             response_data = self.client.get(
                 f"/myorg/admin/workspaces/scanResult/{scan_id}"
             )