openmetadata-ingestion 1.6.7.0__py3-none-any.whl → 1.7.0.0rc1__py3-none-any.whl

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -12,6 +12,7 @@
 Tableau source module
 """
 import traceback
+from datetime import datetime
 from typing import Any, Iterable, List, Optional, Set
 
 from requests.utils import urlparse
@@ -56,14 +57,18 @@ from metadata.generated.schema.type.basic import (
 )
 from metadata.generated.schema.type.entityLineage import ColumnLineage
 from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
+from metadata.generated.schema.type.usageRequest import UsageRequest
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
-from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
+from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
 from metadata.ingestion.lineage.parser import LineageParser
-from metadata.ingestion.lineage.sql_lineage import get_column_fqn, search_table_entities
+from metadata.ingestion.lineage.sql_lineage import get_column_fqn
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
+from metadata.ingestion.source.dashboard.dashboard_service import (
+    DashboardServiceSource,
+    DashboardUsage,
+)
 from metadata.ingestion.source.dashboard.tableau.client import TableauClient
 from metadata.ingestion.source.dashboard.tableau.models import (
     ChartUrl,
@@ -77,6 +82,7 @@ from metadata.ingestion.source.dashboard.tableau.models import (
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_chart, filter_by_datamodel
+from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.helpers import (
     clean_uri,
     get_database_name_for_lineage,
@@ -99,6 +105,14 @@ class TableauSource(DashboardServiceSource):
     metadata_config: OpenMetadataConnection
     client: TableauClient
 
+    def __init__(
+        self,
+        config: WorkflowSource,
+        metadata: OpenMetadata,
+    ):
+        super().__init__(config, metadata)
+        self.today = datetime.now().strftime("%Y-%m-%d")
+
     @classmethod
     def create(
         cls,
@@ -205,15 +219,20 @@ class TableauSource(DashboardServiceSource):
                 include_tags=self.source_config.includeTags,
             )
 
-    def _get_datamodel_sql_query(self, data_model: DataSource) -> Optional[str]:
+    def _get_datamodel_sql_query(
+        self, data_model: DataSource, dashboard_details: TableauDashboard
+    ) -> Optional[str]:
         """
         Method to fetch the custom sql query from the tableau datamodels
         """
         try:
-            sql_queries = []
+            sql_queries = set()
             for table in data_model.upstreamTables or []:
                 for referenced_query in table.referencedByQueries or []:
-                    sql_queries.append(referenced_query.query)
+                    sql_queries.add(referenced_query.query)
+            if not sql_queries:
+                if dashboard_details and dashboard_details.custom_sql_queries:
+                    sql_queries.update(dashboard_details.custom_sql_queries)
             return "\n\n".join(sql_queries) or None
         except Exception as exc:
             logger.debug(traceback.format_exc())
@@ -255,7 +274,9 @@ class TableauSource(DashboardServiceSource):
                     classification_name=TABLEAU_TAG_CATEGORY,
                     include_tags=self.source_config.includeTags,
                 ),
-                sql=self._get_datamodel_sql_query(data_model=data_model),
+                sql=self._get_datamodel_sql_query(
+                    data_model=data_model, dashboard_details=dashboard_details
+                ),
                 owners=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=data_model_request)
@@ -456,7 +477,7 @@ class TableauSource(DashboardServiceSource):
         self,
         upstream_data_model: DataSource,
         datamodel: DataSource,
-        db_service_entity: DatabaseService,
+        db_service_name: Optional[str],
         upstream_data_model_entity: DashboardDataModel,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
@@ -470,7 +491,7 @@ class TableauSource(DashboardServiceSource):
                 if column is not None
             }
             for table in datamodel.upstreamTables or []:
-                om_tables = self._get_database_tables(db_service_entity, table)
+                om_tables = self._get_database_tables(db_service_name, table)
                 for om_table_and_query in om_tables or []:
                     column_lineage = self._get_column_lineage(
                         table,
@@ -569,7 +590,7 @@ class TableauSource(DashboardServiceSource):
         self,
         datamodel: DataSource,
         data_model_entity: DashboardDataModel,
-        db_service_entity: DatabaseService,
+        db_service_name: Optional[str],
     ) -> Iterable[Either[AddLineageRequest]]:
         """ "
         Method to create lineage between tables<->published datasource<->embedded datasource
@@ -593,7 +614,7 @@ class TableauSource(DashboardServiceSource):
                     yield from self._get_table_datamodel_lineage(
                         upstream_data_model=upstream_data_model,
                         datamodel=datamodel,
-                        db_service_entity=db_service_entity,
+                        db_service_name=db_service_name,
                         upstream_data_model_entity=upstream_data_model_entity,
                     )
             except Exception as err:
@@ -602,14 +623,16 @@ class TableauSource(DashboardServiceSource):
                         name="Lineage",
                         error=(
                             "Error to yield datamodel table lineage details for DB "
-                            f"service name [{db_service_entity.name}]: {err}"
+                            f"service name [{db_service_name}]: {err}"
                         ),
                         stackTrace=traceback.format_exc(),
                     )
                 )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: TableauDashboard, db_service_name: str
+        self,
+        dashboard_details: TableauDashboard,
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         This method creates the lineage between tables and datamodels
@@ -621,56 +644,51 @@ class TableauSource(DashboardServiceSource):
         Returns:
             Lineage request between Data Models and Database tables
         """
-        db_service_entity = self.metadata.get_by_name(
-            entity=DatabaseService, fqn=db_service_name
-        )
-        if db_service_entity:
-            for datamodel in dashboard_details.dataModels or []:
-                try:
-                    data_model_entity = self._get_datamodel(datamodel=datamodel)
-                    if data_model_entity:
-                        if datamodel.upstreamDatasources:
-                            # if we have upstreamDatasources(Published Datasources), create lineage in below format
-                            # Table<->Published Datasource<->Embedded Datasource
-                            yield from self._get_datamodel_table_lineage(
-                                datamodel=datamodel,
-                                data_model_entity=data_model_entity,
-                                db_service_entity=db_service_entity,
-                            )
-                        else:
-                            # else we'll create lineage only using Embedded Datasources in below format
-                            # Table<->Embedded Datasource
-                            yield from self._get_table_datamodel_lineage(
-                                upstream_data_model=datamodel,
-                                datamodel=datamodel,
-                                db_service_entity=db_service_entity,
-                                upstream_data_model_entity=data_model_entity,
-                            )
+        for datamodel in dashboard_details.dataModels or []:
+            try:
+                data_model_entity = self._get_datamodel(datamodel=datamodel)
+                if data_model_entity:
+                    if datamodel.upstreamDatasources:
+                        # if we have upstreamDatasources(Published Datasources), create lineage in below format
+                        # Table<->Published Datasource<->Embedded Datasource
+                        yield from self._get_datamodel_table_lineage(
+                            datamodel=datamodel,
+                            data_model_entity=data_model_entity,
+                            db_service_name=db_service_name,
+                        )
+                    else:
+                        # else we'll create lineage only using Embedded Datasources in below format
+                        # Table<->Embedded Datasource
+                        yield from self._get_table_datamodel_lineage(
+                            upstream_data_model=datamodel,
+                            datamodel=datamodel,
+                            db_service_name=db_service_name,
+                            upstream_data_model_entity=data_model_entity,
+                        )
 
-                    # Process custom SQL queries if available
-                    if dashboard_details.custom_sql_queries:
-                        for query in dashboard_details.custom_sql_queries:
-                            try:
-                                db_service_entity = None
-                                if db_service_name:
-                                    db_service_entity = self.metadata.get_by_name(
-                                        entity=DatabaseService, fqn=db_service_name
-                                    )
-                                lineage_parser = LineageParser(
-                                    query,
-                                    ConnectionTypeDialectMapper.dialect_of(
-                                        db_service_entity.serviceType.value
-                                    )
-                                    if db_service_entity
-                                    else None,
+                # Process custom SQL queries if available
+                if dashboard_details.custom_sql_queries:
+                    for query in dashboard_details.custom_sql_queries:
+                        try:
+                            db_service_entity = None
+                            if db_service_name:
+                                db_service_entity = self.metadata.get_by_name(
+                                    entity=DatabaseService, fqn=db_service_name
                                 )
-                                for source_table in lineage_parser.source_tables or []:
-                                    database_schema_table = fqn.split_table_name(
-                                        str(source_table)
-                                    )
-                                    database_name = database_schema_table.get(
-                                        "database"
-                                    )
+                            lineage_parser = LineageParser(
+                                query,
+                                ConnectionTypeDialectMapper.dialect_of(
+                                    db_service_entity.serviceType.value
+                                )
+                                if db_service_entity
+                                else Dialect.ANSI,
+                            )
+                            for source_table in lineage_parser.source_tables or []:
+                                database_schema_table = fqn.split_table_name(
+                                    str(source_table)
+                                )
+                                database_name = database_schema_table.get("database")
+                                if db_service_entity:
                                     if isinstance(
                                         db_service_entity.connection.config,
                                         BigQueryConnection,
@@ -679,40 +697,44 @@ class TableauSource(DashboardServiceSource):
                                     database_name = get_database_name_for_lineage(
                                         db_service_entity, database_name
                                     )
-                                    schema_name = self.check_database_schema_name(
-                                        database_schema_table.get("database_schema")
-                                    )
-                                    table_name = database_schema_table.get("table")
-                                    from_entities = search_table_entities(
-                                        metadata=self.metadata,
-                                        database=database_name,
-                                        service_name=db_service_entity.fullyQualifiedName.root,
-                                        database_schema=schema_name,
-                                        table=table_name,
-                                    )
-                                    for table_entity in from_entities:
-                                        yield self._get_add_lineage_request(
-                                            to_entity=data_model_entity,
-                                            from_entity=table_entity,
-                                            sql=query,
-                                        )
-                            except Exception as err:
-                                logger.debug(traceback.format_exc())
-                                logger.error(
-                                    f"Error processing custom SQL query lineage: {err}"
+                                schema_name = self.check_database_schema_name(
+                                    database_schema_table.get("database_schema")
+                                )
+                                table_name = database_schema_table.get("table")
+                                fqn_search_string = build_es_fqn_search_string(
+                                    database_name=database_name,
+                                    schema_name=schema_name,
+                                    service_name=db_service_name or "*",
+                                    table_name=table_name,
                                 )
+                                from_entities = self.metadata.search_in_any_service(
+                                    entity_type=Table,
+                                    fqn_search_string=fqn_search_string,
+                                    fetch_multiple_entities=True,
+                                )
+                                for table_entity in from_entities:
+                                    yield self._get_add_lineage_request(
+                                        to_entity=data_model_entity,
+                                        from_entity=table_entity,
+                                        sql=query,
+                                    )
+                        except Exception as err:
+                            logger.debug(traceback.format_exc())
+                            logger.error(
+                                f"Error processing custom SQL query lineage: {err}"
+                            )
 
-                except Exception as err:
-                    yield Either(
-                        left=StackTraceError(
-                            name="Lineage",
-                            error=(
-                                "Error to yield dashboard lineage details for DB "
-                                f"service name [{db_service_name}]: {err}"
-                            ),
-                            stackTrace=traceback.format_exc(),
-                        )
+            except Exception as err:
+                yield Either(
+                    left=StackTraceError(
+                        name="Lineage",
+                        error=(
+                            "Error to yield dashboard lineage details for DB "
+                            f"service name [{db_service_name}]: {err}"
+                        ),
+                        stackTrace=traceback.format_exc(),
                     )
+                )
 
     def yield_dashboard_chart(
         self, dashboard_details: TableauDashboard
@@ -773,8 +795,11 @@ class TableauSource(DashboardServiceSource):
         except ConnectionError as err:
             logger.debug(f"Error closing connection - {err}")
 
+        self.metadata.compute_percentile(Dashboard, self.today)
+        self.metadata.close()
+
     def _get_table_entities_from_api(
-        self, db_service_entity: DatabaseService, table: UpstreamTable
+        self, db_service_name: Optional[str], table: UpstreamTable
     ) -> Optional[List[TableAndQuery]]:
         """
         In case we get the table details from the Graphql APIs we process them
@@ -786,39 +811,40 @@ class TableauSource(DashboardServiceSource):
                 if table.database and table.database.name
                 else database_schema_table.get("database")
             )
-            if isinstance(db_service_entity.connection.config, BigQueryConnection):
-                database_name = None
-            database_name = get_database_name_for_lineage(
-                db_service_entity, database_name
-            )
+            if db_service_name:
+                db_service_entity = self.metadata.get_by_name(
+                    entity=DatabaseService, fqn=db_service_name
+                )
+                if isinstance(db_service_entity.connection.config, BigQueryConnection):
+                    database_name = None
+                database_name = get_database_name_for_lineage(
+                    db_service_entity, database_name
+                )
             schema_name = (
                 table.schema_
                 if table.schema_
                 else database_schema_table.get("database_schema")
             )
             table_name = database_schema_table.get("table")
-            table_fqn = fqn.build(
-                self.metadata,
-                entity_type=Table,
-                service_name=db_service_entity.name.root,
+            fqn_search_string = build_es_fqn_search_string(
+                database_name=database_name,
                 schema_name=schema_name,
+                service_name=db_service_name or "*",
                 table_name=table_name,
-                database_name=database_name,
             )
-            if table_fqn:
-                table_entity = self.metadata.get_by_name(
-                    entity=Table,
-                    fqn=table_fqn,
-                )
-                if table_entity:
-                    return [TableAndQuery(table=table_entity)]
+            table_entity = self.metadata.search_in_any_service(
+                entity_type=Table,
+                fqn_search_string=fqn_search_string,
+            )
+            if table_entity:
+                return [TableAndQuery(table=table_entity)]
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(f"Error to get tables for lineage using GraphQL Apis: {exc}")
         return None
 
     def _get_table_entities_from_query(
-        self, db_service_entity: DatabaseService, table: UpstreamTable
+        self, db_service_name: Optional[str], table: UpstreamTable
     ) -> Optional[List[TableAndQuery]]:
         """
         In case we get the table details from the Graphql APIs we process them
@@ -826,34 +852,44 @@ class TableauSource(DashboardServiceSource):
         tables_list = []
         try:
             for custom_sql_table in table.referencedByQueries or []:
+                db_service_entity = None
+                if db_service_name:
+                    db_service_entity = self.metadata.get_by_name(
+                        entity=DatabaseService, fqn=db_service_name
+                    )
                 lineage_parser = LineageParser(
                     custom_sql_table.query,
                     ConnectionTypeDialectMapper.dialect_of(
                         db_service_entity.serviceType.value
                     )
                     if db_service_entity
-                    else None,
+                    else Dialect.ANSI,
                 )
                 for source_table in lineage_parser.source_tables or []:
                     database_schema_table = fqn.split_table_name(str(source_table))
                     database_name = database_schema_table.get("database")
-                    if isinstance(
-                        db_service_entity.connection.config, BigQueryConnection
-                    ):
-                        database_name = None
-                    database_name = get_database_name_for_lineage(
-                        db_service_entity, database_name
-                    )
+                    if db_service_entity:
+                        if isinstance(
+                            db_service_entity.connection.config, BigQueryConnection
+                        ):
+                            database_name = None
+                        database_name = get_database_name_for_lineage(
+                            db_service_entity, database_name
+                        )
                     schema_name = self.check_database_schema_name(
                         database_schema_table.get("database_schema")
                     )
                     table_name = database_schema_table.get("table")
-                    from_entities = search_table_entities(
-                        metadata=self.metadata,
-                        database=database_name,
-                        service_name=db_service_entity.fullyQualifiedName.root,
-                        database_schema=schema_name,
-                        table=table_name,
+                    fqn_search_string = build_es_fqn_search_string(
+                        database_name=database_name,
+                        schema_name=schema_name,
+                        service_name=db_service_name or "*",
+                        table_name=table_name,
+                    )
+                    from_entities = self.metadata.search_in_any_service(
+                        entity_type=Table,
+                        fqn_search_string=fqn_search_string,
+                        fetch_multiple_entities=True,
                     )
                     tables_list.extend(
                         [
@@ -868,7 +904,7 @@ class TableauSource(DashboardServiceSource):
         return tables_list or []
 
     def _get_database_tables(
-        self, db_service_entity: DatabaseService, table: UpstreamTable
+        self, db_service_name: Optional[str], table: UpstreamTable
     ) -> Optional[List[TableAndQuery]]:
         """
         Get the table entities for lineage
@@ -876,12 +912,12 @@ class TableauSource(DashboardServiceSource):
         # If we get the table details from the Graphql APIs we process them directly
         if table.name:
             return self._get_table_entities_from_api(
-                db_service_entity=db_service_entity, table=table
+                db_service_name=db_service_name, table=table
             )
         # Else we get the table details from the SQL queries and process them using SQL lineage parser
         if table.referencedByQueries:
             return self._get_table_entities_from_query(
-                db_service_entity=db_service_entity, table=table
+                db_service_name=db_service_name, table=table
             )
         return None
 
@@ -969,3 +1005,82 @@ class TableauSource(DashboardServiceSource):
                 f"Error fetching project name for {dashboard_details.id}: {exc}"
             )
         return None
+
+    def yield_dashboard_usage(  # pylint: disable=W0221
+        self, dashboard_details: TableauDashboard
+    ) -> Iterable[Either[DashboardUsage]]:
+        """
+        Yield the usage of the dashboard
+        """
+        try:
+            dashboard_fqn = fqn.build(
+                metadata=self.metadata,
+                entity_type=Dashboard,
+                service_name=self.context.get().dashboard_service,
+                dashboard_name=dashboard_details.id,
+            )
+
+            dashboard: Dashboard = self.metadata.get_by_name(
+                entity=Dashboard,
+                fqn=dashboard_fqn,
+                fields=["usageSummary"],
+            )
+
+            current_views = self.client.get_workbook_view_count_by_id(
+                workbook_id=dashboard_details.id
+            )
+
+            if not current_views:
+                logger.debug(f"No usage to report for {dashboard_details.name}")
+
+            if not dashboard.usageSummary:
+                logger.info(
+                    f"Yielding fresh usage for {dashboard.fullyQualifiedName.root}"
+                )
+                yield Either(
+                    right=DashboardUsage(
+                        dashboard=dashboard,
+                        usage=UsageRequest(date=self.today, count=current_views),
+                    )
+                )
+
+            elif (
+                str(dashboard.usageSummary.date.root) != self.today
+                or not dashboard.usageSummary.dailyStats.count
+            ):
+                latest_usage = dashboard.usageSummary.dailyStats.count
+
+                new_usage = current_views - latest_usage
+                if new_usage < 0:
+                    raise ValueError(
+                        f"Wrong computation of usage difference. Got new_usage={new_usage}."
+                    )
+
+                logger.info(
+                    f"Yielding new usage for {dashboard.fullyQualifiedName.root}"
+                )
+                yield Either(
+                    right=DashboardUsage(
+                        dashboard=dashboard,
+                        usage=UsageRequest(
+                            date=self.today, count=current_views - latest_usage
+                        ),
+                    )
+                )
+
+            else:
+                logger.debug(
+                    f"Latest usage {dashboard.usageSummary} vs. today {self.today}. Nothing to compute."
+                )
+                logger.info(
+                    f"Usage already informed for {dashboard.fullyQualifiedName.root}"
+                )
+
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=f"{dashboard_details.name} Usage",
+                    error=f"Exception computing dashboard usage for {dashboard_details.name}: {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )

metadata/ingestion/source/database/bigquery/helper.py

@@ -12,8 +12,9 @@
 """
 Source connection helper
 """
+import re
 import traceback
-from typing import Any
+from typing import Any, List, Tuple
 
 from pydantic import BaseModel
 from sqlalchemy import inspect
@@ -142,3 +143,10 @@ def get_foreign_keys(
             f"Error while fetching foreign key constraint error for table [{schema}.{table_name}]: {exc}"
         )
         return []
+
+
+def parse_bigqeury_labels(labels: str) -> List[Tuple[str, str]]:
+    """
+    This function is used to parse BigQuery label string into a list of tuples.
+    """
+    return re.findall(r'STRUCT\("([^"]+)",\s*"([^"]+)"\)', labels)