openmetadata-ingestion 1.8.0.0__py3-none-any.whl → 1.8.2.0__py3-none-any.whl

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -11,6 +11,7 @@
 """
 Tableau source module
 """
+# pylint: disable=too-many-lines
 import traceback
 from datetime import datetime
 from typing import Any, Iterable, List, Optional, Set
@@ -62,7 +63,10 @@ from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
 from metadata.ingestion.lineage.parser import LineageParser
-from metadata.ingestion.lineage.sql_lineage import get_column_fqn
+from metadata.ingestion.lineage.sql_lineage import (
+    get_column_fqn,
+    get_table_fqn_from_query_name,
+)
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import (
@@ -157,13 +161,13 @@ class TableauSource(DashboardServiceSource):
         return None
 
     @staticmethod
-    def _get_data_models_tags(dataModels: List[DataSource]) -> Set[str]:
+    def _get_data_models_tags(data_models: List[DataSource]) -> Set[str]:
         """
         Get the tags from the data model in the upstreamDatasources
         """
         tags = set()
         try:
-            for data_model in dataModels:
+            for data_model in data_models:
                 # tags seems to be available for upstreamDatasources only, not for dataModels
                 for upstream_source in data_model.upstreamDatasources or []:
                     for tag in upstream_source.tags:
@@ -191,7 +195,7 @@ class TableauSource(DashboardServiceSource):
             _all_tags = tags.union(_data_models_tags)
 
             yield from get_ometa_tag_and_classification(
-                tags=[tag for tag in _all_tags],
+                tags=list(_all_tags),
                 classification_name=TABLEAU_TAG_CATEGORY,
                 tag_description="Tableau Tag",
                 classification_description="Tags associated with tableau entities",
@@ -238,9 +242,9 @@ class TableauSource(DashboardServiceSource):
             data_model_request = CreateDashboardDataModelRequest(
                 name=EntityName(data_model.id),
                 displayName=data_model_name,
-                description=Markdown(data_model.description)
-                if data_model.description
-                else None,
+                description=(
+                    Markdown(data_model.description) if data_model.description else None
+                ),
                 service=FullyQualifiedEntityName(self.context.get().dashboard_service),
                 dataModelType=data_model_type.value,
                 serviceType=DashboardServiceType.Tableau.value,
@@ -307,9 +311,11 @@ class TableauSource(DashboardServiceSource):
             dashboard_request = CreateDashboardRequest(
                 name=EntityName(dashboard_details.id),
                 displayName=dashboard_details.name,
-                description=Markdown(dashboard_details.description)
-                if dashboard_details.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.description)
+                    if dashboard_details.description
+                    else None
+                ),
                 project=self.get_project_name(dashboard_details=dashboard_details),
                 charts=[
                     FullyQualifiedEntityName(
@@ -370,6 +376,7 @@ class TableauSource(DashboardServiceSource):
                     columns.append(child_column.fullyQualifiedName.root)
         return columns
 
+    # pylint: disable=arguments-differ
     def _get_column_lineage(
         self,
         upstream_table: UpstreamTable,
@@ -453,7 +460,7 @@ class TableauSource(DashboardServiceSource):
         self,
         upstream_data_model: DataSource,
         datamodel: DataSource,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
         upstream_data_model_entity: DashboardDataModel,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
@@ -467,7 +474,7 @@ class TableauSource(DashboardServiceSource):
                 if column is not None
             }
             for table in datamodel.upstreamTables or []:
-                om_tables = self._get_database_tables(db_service_name, table)
+                om_tables = self._get_database_tables(db_service_prefix, table)
                 for om_table_and_query in om_tables or []:
                     column_lineage = self._get_column_lineage(
                         table,
@@ -562,15 +569,22 @@ class TableauSource(DashboardServiceSource):
 
         return datamodel_column_lineage or None
 
+    # pylint: disable=too-many-locals,too-many-nested-blocks
     def _get_datamodel_table_lineage(
         self,
         datamodel: DataSource,
         data_model_entity: DashboardDataModel,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
     ) -> Iterable[Either[AddLineageRequest]]:
         """ "
         Method to create lineage between tables<->published datasource<->embedded datasource
         """
+        (
+            db_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
         for upstream_data_model in datamodel.upstreamDatasources or []:
             try:
                 upstream_data_model_entity = self._get_datamodel(
@@ -590,7 +604,7 @@ class TableauSource(DashboardServiceSource):
                     yield from self._get_table_datamodel_lineage(
                         upstream_data_model=upstream_data_model,
                         datamodel=datamodel,
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         upstream_data_model_entity=upstream_data_model_entity,
                     )
 
@@ -607,11 +621,13 @@ class TableauSource(DashboardServiceSource):
                                 )
                             lineage_parser = LineageParser(
                                 query,
-                                ConnectionTypeDialectMapper.dialect_of(
-                                    db_service_entity.serviceType.value
-                                )
-                                if db_service_entity
-                                else Dialect.ANSI,
+                                (
+                                    ConnectionTypeDialectMapper.dialect_of(
+                                        db_service_entity.serviceType.value
+                                    )
+                                    if db_service_entity
+                                    else Dialect.ANSI
+                                ),
                             )
                             for source_table in lineage_parser.source_tables or []:
                                 database_schema_table = fqn.split_table_name(
@@ -631,11 +647,44 @@ class TableauSource(DashboardServiceSource):
                                     database_schema_table.get("database_schema")
                                 )
                                 table_name = database_schema_table.get("table")
+
+                                if (
+                                    prefix_database_name
+                                    and database_name
+                                    and prefix_database_name.lower()
+                                    != database_name.lower()
+                                ):
+                                    logger.debug(
+                                        f"Database {database_name} does not match prefix {prefix_database_name}"
+                                    )
+                                    continue
+
+                                if (
+                                    prefix_schema_name
+                                    and schema_name
+                                    and prefix_schema_name.lower()
+                                    != schema_name.lower()
+                                ):
+                                    logger.debug(
+                                        f"Schema {schema_name} does not match prefix {prefix_schema_name}"
+                                    )
+                                    continue
+
+                                if (
+                                    prefix_table_name
+                                    and table_name
+                                    and prefix_table_name.lower() != table_name.lower()
+                                ):
+                                    logger.debug(
+                                        f"Table {table_name} does not match prefix {prefix_table_name}"
+                                    )
+                                    continue
+
                                 fqn_search_string = build_es_fqn_search_string(
-                                    database_name=database_name,
-                                    schema_name=schema_name,
+                                    database_name=prefix_database_name or database_name,
+                                    schema_name=prefix_schema_name or schema_name,
                                     service_name=db_service_name or "*",
-                                    table_name=table_name,
+                                    table_name=prefix_table_name or table_name,
                                 )
                                 from_entities = self.metadata.search_in_any_service(
                                     entity_type=Table,
@@ -644,7 +693,8 @@ class TableauSource(DashboardServiceSource):
                                 )
                                 if not from_entities:
                                     logger.debug(
-                                        f"No table entities found for custom SQL lineage. fqn_search_string={fqn_search_string}, table_name={table_name}, query={query}"
+                                        "No table entities found for custom SQL lineage."
+                                        f"fqn_search_string={fqn_search_string}, table_name={table_name}, query={query}"
                                     )
                                 for table_entity in from_entities:
                                     yield self._get_add_lineage_request(
@@ -668,7 +718,7 @@ class TableauSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: TableauDashboard,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         This method creates the lineage between tables and datamodels
@@ -690,7 +740,7 @@ class TableauSource(DashboardServiceSource):
                         yield from self._get_datamodel_table_lineage(
                             datamodel=datamodel,
                             data_model_entity=data_model_entity,
-                            db_service_name=db_service_name,
+                            db_service_prefix=db_service_prefix,
                         )
                     else:
                         # else we'll create lineage only using Embedded Datasources in below format
@@ -698,7 +748,7 @@ class TableauSource(DashboardServiceSource):
                         yield from self._get_table_datamodel_lineage(
                             upstream_data_model=datamodel,
                             datamodel=datamodel,
-                            db_service_name=db_service_name,
+                            db_service_prefix=db_service_prefix,
                             upstream_data_model_entity=data_model_entity,
                         )
 
@@ -708,7 +758,7 @@ class TableauSource(DashboardServiceSource):
                         name="Lineage",
                         error=(
                             "Error to yield dashboard lineage details for DB "
-                            f"service name [{db_service_name}]: {err}"
+                            f"service prefix [{db_service_prefix}]: {err}"
                         ),
                         stackTrace=traceback.format_exc(),
                     )
@@ -778,18 +828,26 @@ class TableauSource(DashboardServiceSource):
         self.metadata.close()
 
     def _get_table_entities_from_api(
-        self, db_service_name: Optional[str], table: UpstreamTable
+        self, db_service_prefix: Optional[str], table: UpstreamTable
     ) -> Optional[List[TableAndQuery]]:
         """
         In case we get the table details from the Graphql APIs we process them
         """
         try:
+            (
+                db_service_name,
+                prefix_database_name,
+                prefix_schema_name,
+                prefix_table_name,
+            ) = self.parse_db_service_prefix(db_service_prefix)
+
             database_schema_table = fqn.split_table_name(table.name)
             database_name = (
                 table.database.name
                 if table.database and table.database.name
                 else database_schema_table.get("database")
             )
+
             if db_service_name:
                 db_service_entity = self.metadata.get_by_name(
                     entity=DatabaseService, fqn=db_service_name
@@ -805,11 +863,38 @@ class TableauSource(DashboardServiceSource):
                 else database_schema_table.get("database_schema")
             )
             table_name = database_schema_table.get("table")
+
+            if (
+                prefix_database_name
+                and database_name
+                and prefix_database_name.lower() != database_name.lower()
+            ):
+                logger.debug(
+                    f"Database {database_name} does not match prefix {prefix_database_name}"
+                )
+                return None
+            if (
+                prefix_schema_name
+                and schema_name
+                and prefix_schema_name.lower() != schema_name.lower()
+            ):
+                logger.debug(
+                    f"Schema {schema_name} does not match prefix {prefix_schema_name}"
+                )
+            if (
+                prefix_table_name
+                and table_name
+                and prefix_table_name.lower() != table_name.lower()
+            ):
+                logger.debug(
+                    f"Table {table_name} does not match prefix {prefix_table_name}"
+                )
+
             fqn_search_string = build_es_fqn_search_string(
-                database_name=database_name,
-                schema_name=schema_name,
+                database_name=prefix_database_name or database_name,
+                schema_name=prefix_schema_name or schema_name,
                 service_name=db_service_name or "*",
-                table_name=table_name,
+                table_name=prefix_table_name or table_name,
             )
             table_entity = self.metadata.search_in_any_service(
                 entity_type=Table,
@@ -817,8 +902,36 @@ class TableauSource(DashboardServiceSource):
             )
             if table_entity:
                 return [TableAndQuery(table=table_entity)]
+
+            # fallback to fullName if schema name is not appropriate
+            # TODO: We should make this a primary approach for lineage instead
+            # using this strategy as a fallback.
+            if not table.fullName:
+                logger.debug(
+                    "No table entity found for lineage using GraphQL APIs."
+                    f"fqn_search_string={fqn_search_string}, table_name={table_name}"
+                )
+                return None
+            # fullname is in the format of [database].[schema].[table] or [database].[table] or [table]
+            database_query, schema_query, table_query = get_table_fqn_from_query_name(
+                table.fullName.replace("[", "").replace("]", "")
+            )
+            if table_query:
+                fqn_search_string = build_es_fqn_search_string(
+                    database_name=database_query,
+                    schema_name=schema_query,
+                    service_name=db_service_name or "*",
+                    table_name=table_query,
+                )
+                table_entity = self.metadata.search_in_any_service(
+                    entity_type=Table,
+                    fqn_search_string=fqn_search_string,
+                )
+                if table_entity:
+                    return [TableAndQuery(table=table_entity)]
             logger.debug(
-                f"No table entity found for lineage using GraphQL APIs. fqn_search_string={fqn_search_string}, table_name={table_name}"
+                "No table entity found for lineage using GraphQL APIs."
+                f"fqn_search_string={fqn_search_string}, table_name={table_name}"
             )
         except Exception as exc:
             logger.debug(traceback.format_exc())
@@ -826,13 +939,20 @@ class TableauSource(DashboardServiceSource):
         return None
 
     def _get_table_entities_from_query(
-        self, db_service_name: Optional[str], table: UpstreamTable
+        self, db_service_prefix: Optional[str], table: UpstreamTable
     ) -> Optional[List[TableAndQuery]]:
         """
         In case we get the table details from the Graphql APIs we process them
         """
         tables_list = []
         try:
+            (
+                db_service_name,
+                prefix_database_name,
+                prefix_schema_name,
+                prefix_table_name,
+            ) = self.parse_db_service_prefix(db_service_prefix)
+
             for custom_sql_table in table.referencedByQueries or []:
                 db_service_entity = None
                 if db_service_name:
@@ -841,11 +961,13 @@ class TableauSource(DashboardServiceSource):
                     )
                 lineage_parser = LineageParser(
                     custom_sql_table.query,
-                    ConnectionTypeDialectMapper.dialect_of(
-                        db_service_entity.serviceType.value
-                    )
-                    if db_service_entity
-                    else Dialect.ANSI,
+                    (
+                        ConnectionTypeDialectMapper.dialect_of(
+                            db_service_entity.serviceType.value
+                        )
+                        if db_service_entity
+                        else Dialect.ANSI
+                    ),
                 )
                 for source_table in lineage_parser.source_tables or []:
                     database_schema_table = fqn.split_table_name(str(source_table))
@@ -862,11 +984,40 @@ class TableauSource(DashboardServiceSource):
                         database_schema_table.get("database_schema")
                     )
                     table_name = database_schema_table.get("table")
+
+                    if (
+                        prefix_database_name
+                        and database_name
+                        and prefix_database_name.lower() != database_name.lower()
+                    ):
+                        logger.debug(
+                            f"Database {database_name} does not match prefix {prefix_database_name}"
+                        )
+                        continue
+                    if (
+                        prefix_schema_name
+                        and schema_name
+                        and prefix_schema_name.lower() != schema_name.lower()
+                    ):
+                        logger.debug(
+                            f"Schema {schema_name} does not match prefix {prefix_schema_name}"
+                        )
+                        continue
+                    if (
+                        prefix_table_name
+                        and table_name
+                        and prefix_table_name.lower() != table_name.lower()
+                    ):
+                        logger.debug(
+                            f"Table {table_name} does not match prefix {prefix_table_name}"
+                        )
+                        continue
+
                     fqn_search_string = build_es_fqn_search_string(
-                        database_name=database_name,
-                        schema_name=schema_name,
+                        database_name=prefix_database_name or database_name,
+                        schema_name=prefix_schema_name or schema_name,
                         service_name=db_service_name or "*",
-                        table_name=table_name,
+                        table_name=prefix_table_name or table_name,
                     )
                     from_entities = self.metadata.search_in_any_service(
                         entity_type=Table,
@@ -875,7 +1026,9 @@ class TableauSource(DashboardServiceSource):
                     )
                     if not from_entities:
                         logger.debug(
-                            f"No table entities found for lineage using SQL Queries. fqn_search_string={fqn_search_string}, table_name={table_name}, query={custom_sql_table.query}"
+                            "No table entities found for lineage using SQL Queries."
+                            f"fqn_search_string={fqn_search_string}, "
+                            f"table_name={table_name}, query={custom_sql_table.query}"
                         )
                     tables_list.extend(
                         [
@@ -891,7 +1044,7 @@ class TableauSource(DashboardServiceSource):
         return tables_list or []
 
     def _get_database_tables(
-        self, db_service_name: Optional[str], table: UpstreamTable
+        self, db_service_prefix: Optional[str], table: UpstreamTable
     ) -> Optional[List[TableAndQuery]]:
         """
         Get the table entities for lineage
@@ -899,12 +1052,12 @@ class TableauSource(DashboardServiceSource):
         # If we get the table details from the Graphql APIs we process them directly
         if table.name:
             return self._get_table_entities_from_api(
-                db_service_name=db_service_name, table=table
+                db_service_prefix=db_service_prefix, table=table
             )
         # Else we get the table details from the SQL queries and process them using SQL lineage parser
         if table.referencedByQueries:
             return self._get_table_entities_from_query(
-                db_service_name=db_service_name, table=table
+                db_service_prefix=db_service_prefix, table=table
             )
         return None
 
@@ -934,9 +1087,11 @@ class TableauSource(DashboardServiceSource):
             try:
                 if column:
                     parsed_column = {
-                        "dataTypeDisplay": column.remoteType
-                        if column.remoteType
-                        else DataType.UNKNOWN.value,
+                        "dataTypeDisplay": (
+                            column.remoteType
+                            if column.remoteType
+                            else DataType.UNKNOWN.value
+                        ),
                         "dataType": ColumnTypeParser.get_column_type(
                             column.remoteType if column.remoteType else None
                         ),

metadata/ingestion/source/database/bigquery/service_spec.py

@@ -1,9 +1,9 @@
 from metadata.ingestion.source.database.bigquery.lineage import BigqueryLineageSource
 from metadata.ingestion.source.database.bigquery.metadata import BigquerySource
-from metadata.ingestion.source.database.bigquery.profiler.profiler import (
-    BigQueryProfiler,
-)
 from metadata.ingestion.source.database.bigquery.usage import BigqueryUsageSource
+from metadata.profiler.interface.sqlalchemy.bigquery.profiler_interface import (
+    BigQueryProfilerInterface,
+)
 from metadata.sampler.sqlalchemy.bigquery.sampler import BigQuerySampler
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
@@ -11,6 +11,6 @@ ServiceSpec = DefaultDatabaseSpec(
     metadata_source_class=BigquerySource,
     lineage_source_class=BigqueryLineageSource,
     usage_source_class=BigqueryUsageSource,
-    profiler_class=BigQueryProfiler,
+    profiler_class=BigQueryProfilerInterface,
     sampler_class=BigQuerySampler,
 )