acryl-datahub 1.0.0rc18__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

datahub/ingestion/source/tableau/tableau.py

@@ -3,6 +3,7 @@ import logging
 import re
 import time
 from collections import OrderedDict, defaultdict
+from copy import deepcopy
 from dataclasses import dataclass, field as dataclass_field
 from datetime import datetime, timedelta, timezone
 from functools import lru_cache
@@ -12,6 +13,7 @@ from typing import (
     Dict,
     Iterable,
     List,
+    Literal,
     Optional,
     Set,
     Tuple,
@@ -79,6 +81,7 @@ from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import (
     BIContainerSubTypes,
     DatasetSubTypes,
+    SourceCapabilityModifier,
 )
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
@@ -117,7 +120,6 @@ from datahub.ingestion.source.tableau.tableau_common import (
 )
 from datahub.ingestion.source.tableau.tableau_server_wrapper import UserInfo
 from datahub.ingestion.source.tableau.tableau_validation import check_user_role
-from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport
 from datahub.metadata.com.linkedin.pegasus2avro.common import (
     AuditStamp,
     ChangeAuditStamps,
@@ -147,7 +149,6 @@ from datahub.metadata.com.linkedin.pegasus2avro.schema import (
 )
 from datahub.metadata.schema_classes import (
     BrowsePathsClass,
-    ChangeTypeClass,
     ChartInfoClass,
     ChartUsageStatisticsClass,
     DashboardInfoClass,
@@ -473,6 +474,13 @@ class TableauPageSizeConfig(ConfigModel):
         return self.database_table_page_size or self.page_size
 
 
+_IngestHiddenAssetsOptionsType = Literal["worksheet", "dashboard"]
+_IngestHiddenAssetsOptions: List[_IngestHiddenAssetsOptionsType] = [
+    "worksheet",
+    "dashboard",
+]
+
+
 class TableauConfig(
     DatasetLineageProviderConfigBase,
     StatefulIngestionConfigBase,
@@ -523,10 +531,22 @@ class TableauConfig(
         default=False,
         description="Ingest Owner from source. This will override Owner info entered from UI",
     )
+    use_email_as_username: bool = Field(
+        default=False,
+        description="Use email address instead of username for entity owners. Requires ingest_owner to be True.",
+    )
     ingest_tables_external: bool = Field(
         default=False,
         description="Ingest details for tables external to (not embedded in) tableau as entities.",
     )
+    emit_all_published_datasources: bool = Field(
+        default=False,
+        description="Ingest all published data sources. When False (default), only ingest published data sources that belong to an ingested workbook.",
+    )
+    emit_all_embedded_datasources: bool = Field(
+        default=False,
+        description="Ingest all embedded data sources. When False (default), only ingest embedded data sources that belong to an ingested workbook.",
+    )
 
     env: str = Field(
         default=builder.DEFAULT_ENV,
@@ -573,13 +593,13 @@ class TableauConfig(
     )
 
     extract_lineage_from_unsupported_custom_sql_queries: bool = Field(
-        default=False,
-        description="[Experimental] Whether to extract lineage from unsupported custom sql queries using SQL parsing",
+        default=True,
+        description="[Experimental] Extract lineage from Custom SQL queries using DataHub's SQL parser in cases where the Tableau Catalog API fails to return lineage for the query.",
     )
 
     force_extraction_of_lineage_from_custom_sql_queries: bool = Field(
         default=False,
-        description="[Experimental] Force extraction of lineage from custom sql queries using SQL parsing, ignoring Tableau metadata",
+        description="[Experimental] Force extraction of lineage from Custom SQL queries using DataHub's SQL parser, even when the Tableau Catalog API returns lineage already.",
     )
 
     sql_parsing_disable_schema_awareness: bool = Field(
@@ -612,10 +632,14 @@ class TableauConfig(
         description="Configuration settings for ingesting Tableau groups and their capabilities as custom properties.",
     )
 
-    ingest_hidden_assets: bool = Field(
-        True,
-        description="When enabled, hidden views and dashboards are ingested into Datahub. "
-        "If a dashboard or view is hidden in Tableau the luid is blank. Default of this config field is True.",
+    ingest_hidden_assets: Union[List[_IngestHiddenAssetsOptionsType], bool] = Field(
+        _IngestHiddenAssetsOptions,
+        description=(
+            "When enabled, hidden worksheets and dashboards are ingested into Datahub."
+            " If a dashboard or worksheet is hidden in Tableau the luid is blank."
+            " A list of asset types can also be specified, to only ingest those hidden assets."
+            " Current options supported are 'worksheet' and 'dashboard'."
+        ),
     )
 
     tags_for_hidden_assets: List[str] = Field(
@@ -631,6 +655,11 @@ class TableauConfig(
     # pre = True because we want to take some decision before pydantic initialize the configuration to default values
     @root_validator(pre=True)
     def projects_backward_compatibility(cls, values: Dict) -> Dict:
+        # In-place update of the input dict would cause state contamination. This was discovered through test failures
+        # in test_hex.py where the same dict is reused.
+        # So a copy is performed first.
+        values = deepcopy(values)
+
         projects = values.get("projects")
         project_pattern = values.get("project_pattern")
         project_path_pattern = values.get("project_path_pattern")
@@ -642,6 +671,7 @@ class TableauConfig(
             values["project_pattern"] = AllowDenyPattern(
                 allow=[f"^{prj}$" for prj in projects]
             )
+            values.pop("projects")
         elif (project_pattern or project_path_pattern) and projects:
             raise ValueError(
                 "projects is deprecated. Please use project_path_pattern only."
@@ -653,7 +683,7 @@ class TableauConfig(
 
         return values
 
-    @root_validator()
+    @root_validator(skip_on_failure=True)
     def validate_config_values(cls, values: Dict) -> Dict:
         tags_for_hidden_assets = values.get("tags_for_hidden_assets")
         ingest_tags = values.get("ingest_tags")
@@ -665,6 +695,14 @@ class TableauConfig(
             raise ValueError(
                 "tags_for_hidden_assets is only allowed with ingest_tags enabled. Be aware that this will overwrite tags entered from the UI."
             )
+
+        use_email_as_username = values.get("use_email_as_username")
+        ingest_owner = values.get("ingest_owner")
+        if use_email_as_username and not ingest_owner:
+            raise ValueError(
+                "use_email_as_username requires ingest_owner to be enabled."
+            )
+
         return values
 
 
@@ -756,7 +794,6 @@ class SiteIdContentUrl:
 @dataclass
 class TableauSourceReport(
     StaleEntityRemovalSourceReport,
-    IngestionStageReport,
 ):
     get_all_datasources_query_failed: bool = False
     num_get_datasource_query_failures: int = 0
@@ -826,6 +863,9 @@ class TableauSourceReport(
         default_factory=(lambda: defaultdict(int))
     )
 
+    # Owner extraction statistics
+    num_email_fallback_to_username: int = 0
+
 
 def report_user_role(report: TableauSourceReport, server: Server) -> None:
     title: str = "Insufficient Permissions"
@@ -856,16 +896,29 @@ def report_user_role(report: TableauSourceReport, server: Server) -> None:
 @platform_name("Tableau")
 @config_class(TableauConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(
+    SourceCapability.CONTAINERS,
+    "Enabled by default",
+    subtype_modifier=[
+        SourceCapabilityModifier.TABLEAU_PROJECT,
+        SourceCapabilityModifier.TABLEAU_SITE,
+        SourceCapabilityModifier.TABLEAU_WORKBOOK,
+    ],
+)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.DOMAINS, "Requires transformer", supported=False)
 @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
 @capability(
     SourceCapability.USAGE_STATS,
     "Dashboard/Chart view counts, enabled using extract_usage_stats config",
+    subtype_modifier=[
+        SourceCapabilityModifier.DASHBOARD,
+        SourceCapabilityModifier.CHART,
+    ],
 )
 @capability(
     SourceCapability.DELETION_DETECTION,
-    "Enabled by default when stateful ingestion is turned on.",
+    "Enabled by default via stateful ingestion.",
 )
 @capability(SourceCapability.OWNERSHIP, "Requires recipe configuration")
 @capability(SourceCapability.TAGS, "Requires recipe configuration")
@@ -874,6 +927,7 @@ def report_user_role(report: TableauSourceReport, server: Server) -> None:
     SourceCapability.LINEAGE_FINE,
     "Enabled by default, configure using `extract_column_level_lineage`",
 )
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class TableauSource(StatefulIngestionSourceBase, TestableSource):
     platform = "tableau"
 
@@ -1157,7 +1211,7 @@ class TableauSiteSource:
                     self.report.warning(
                         title="Incomplete project hierarchy",
                         message="Project details missing. Child projects will be ingested without reference to their parent project. We generally need Site Administrator Explorer permissions to extract the complete project hierarchy.",
-                        context=f"Missing {project.parent_id}, referenced by {project.id} {project.project_name}",
+                        context=f"Missing {project.parent_id}, referenced by {project.id} {project.name}",
                     )
                     project.parent_id = None
 
@@ -1348,6 +1402,26 @@ class TableauSiteSource:
         # More info here: https://help.tableau.com/current/api/metadata_api/en-us/reference/view.doc.html
         return not dashboard_or_view.get(c.LUID)
 
+    def _should_ingest_worksheet(self, worksheet: Dict) -> bool:
+        return (
+            self.config.ingest_hidden_assets is True
+            or (
+                isinstance(self.config.ingest_hidden_assets, list)
+                and "worksheet" in self.config.ingest_hidden_assets
+            )
+            or not self._is_hidden_view(worksheet)
+        )
+
+    def _should_ingest_dashboard(self, dashboard: Dict) -> bool:
+        return (
+            self.config.ingest_hidden_assets is True
+            or (
+                isinstance(self.config.ingest_hidden_assets, list)
+                and "dashboard" in self.config.ingest_hidden_assets
+            )
+            or not self._is_hidden_view(dashboard)
+        )
+
     def get_connection_object_page(
         self,
         query: str,
@@ -1369,7 +1443,9 @@ class TableauSiteSource:
         `fetch_size:` The number of records to retrieve from Tableau
             Server in a single API call, starting from the current cursor position on Tableau Server.
         """
-        retries_remaining = retries_remaining or self.config.max_retries
+        retries_remaining = (
+            self.config.max_retries if retries_remaining is None else retries_remaining
+        )
 
         logger.debug(
             f"Query {connection_type} to get {fetch_size} objects with cursor {current_cursor}"
@@ -1512,12 +1588,15 @@ class TableauSiteSource:
                         }}""",
                     )
             else:
-                # As of Tableau Server 2024.2, the metadata API sporadically returns a 30-second
-                # timeout error.
-                # It doesn't reliably happen, so retrying a couple of times makes sense.
                 if all(
+                    # As of Tableau Server 2024.2, the metadata API sporadically returns a 30-second
+                    # timeout error.
+                    # It doesn't reliably happen, so retrying a couple of times makes sense.
                     error.get("message")
                     == "Execution canceled because timeout of 30000 millis was reached"
+                    # The Metadata API sometimes returns an 'unexpected error' message when querying
+                    # embeddedDatasourcesConnection. Try retrying a couple of times.
+                    or error.get("message") == "Unexpected error occurred"
                     for error in errors
                 ):
                     # If it was only a timeout error, we can retry.
@@ -1529,8 +1608,8 @@ class TableauSiteSource:
                         (self.config.max_retries - retries_remaining + 1) ** 2, 60
                     )
                     logger.info(
-                        f"Query {connection_type} received a 30 second timeout error - will retry in {backoff_time} seconds. "
-                        f"Retries remaining: {retries_remaining}"
+                        f"Query {connection_type} received a retryable error with {retries_remaining} retries remaining, "
+                        f"will retry in {backoff_time} seconds: {errors}"
                     )
                     time.sleep(backoff_time)
                     return self.get_connection_object_page(
@@ -1540,7 +1619,7 @@ class TableauSiteSource:
                         fetch_size=fetch_size,
                         current_cursor=current_cursor,
                         retry_on_auth_error=True,
-                        retries_remaining=retries_remaining,
+                        retries_remaining=retries_remaining - 1,
                     )
                 raise RuntimeError(f"Query {connection_type} error: {errors}")
 
@@ -1623,7 +1702,7 @@ class TableauSiteSource:
                 # if multiple project has name C. Ideal solution is to use projectLuidWithin to avoid duplicate project,
                 # however Tableau supports projectLuidWithin in Tableau Cloud June 2022 / Server 2022.3 and later.
                 project_luid: Optional[str] = self._get_workbook_project_luid(workbook)
-                if project_luid not in self.tableau_project_registry.keys():
+                if project_luid not in self.tableau_project_registry:
                     wrk_name: Optional[str] = workbook.get(c.NAME)
                     wrk_id: Optional[str] = workbook.get(c.ID)
                     prj_name: Optional[str] = workbook.get(c.PROJECT_NAME)
@@ -2147,32 +2226,32 @@ class TableauSiteSource:
                     else []
                 )
 
-                # The Tableau SQL parser much worse than our sqlglot based parser,
-                # so relying on metadata parsed by Tableau from SQL queries can be
-                # less accurate. This option allows us to ignore Tableau's parser and
-                # only use our own.
-                if self.config.force_extraction_of_lineage_from_custom_sql_queries:
-                    logger.debug("Extracting TLL & CLL from custom sql (forced)")
+                tableau_table_list = csql.get(c.TABLES, [])
+                if self.config.force_extraction_of_lineage_from_custom_sql_queries or (
+                    not tableau_table_list
+                    and self.config.extract_lineage_from_unsupported_custom_sql_queries
+                ):
+                    if not tableau_table_list:
+                        # custom sql tables may contain unsupported sql, causing incomplete lineage
+                        # we extract the lineage from the raw queries
+                        logger.debug(
+                            "Parsing TLL & CLL from custom sql (tableau metadata incomplete)"
+                        )
+                    else:
+                        # The Tableau SQL parser is much worse than our sqlglot based parser,
+                        # so relying on metadata parsed by Tableau from SQL queries can be
+                        # less accurate. This option allows us to ignore Tableau's parser and
+                        # only use our own.
+                        logger.debug("Parsing TLL & CLL from custom sql (forced)")
+
                     yield from self._create_lineage_from_unsupported_csql(
                         csql_urn, csql, columns
                     )
                 else:
-                    tables = csql.get(c.TABLES, [])
-
-                    if tables:
-                        # lineage from custom sql -> datasets/tables #
-                        yield from self._create_lineage_to_upstream_tables(
-                            csql_urn, tables, datasource
-                        )
-                    elif (
-                        self.config.extract_lineage_from_unsupported_custom_sql_queries
-                    ):
-                        logger.debug("Extracting TLL & CLL from custom sql")
-                        # custom sql tables may contain unsupported sql, causing incomplete lineage
-                        # we extract the lineage from the raw queries
-                        yield from self._create_lineage_from_unsupported_csql(
-                            csql_urn, csql, columns
-                        )
+                    # lineage from custom sql -> datasets/tables #
+                    yield from self._create_lineage_to_upstream_tables(
+                        csql_urn, tableau_table_list, datasource
+                    )
 
             #  Schema Metadata
             schema_metadata = self.get_schema_metadata_for_custom_sql(columns)
@@ -2210,7 +2289,6 @@ class TableauSiteSource:
             yield self.get_metadata_change_event(dataset_snapshot)
             yield self.get_metadata_change_proposal(
                 dataset_snapshot.urn,
-                aspect_name=c.SUB_TYPES,
                 aspect=SubTypesClass(typeNames=[DatasetSubTypes.VIEW, c.CUSTOM_SQL]),
             )
 
@@ -2253,7 +2331,7 @@ class TableauSiteSource:
         # It is possible due to https://github.com/tableau/server-client-python/issues/1210
         if (
             ds.get(c.LUID)
-            and ds[c.LUID] not in self.datasource_project_map.keys()
+            and ds[c.LUID] not in self.datasource_project_map
             and self.report.get_all_datasources_query_failed
         ):
             logger.debug(
@@ -2265,7 +2343,7 @@ class TableauSiteSource:
 
         if (
             ds.get(c.LUID)
-            and ds[c.LUID] in self.datasource_project_map.keys()
+            and ds[c.LUID] in self.datasource_project_map
             and self.datasource_project_map[ds[c.LUID]] in self.tableau_project_registry
         ):
             return self.datasource_project_map[ds[c.LUID]]
@@ -2375,7 +2453,6 @@ class TableauSiteSource:
             upstream_lineage = UpstreamLineage(upstreams=upstream_tables)
             yield self.get_metadata_change_proposal(
                 csql_urn,
-                aspect_name=c.UPSTREAM_LINEAGE,
                 aspect=upstream_lineage,
             )
             self.report.num_tables_with_upstream_lineage += 1
@@ -2561,7 +2638,6 @@ class TableauSiteSource:
         )
         yield self.get_metadata_change_proposal(
             csql_urn,
-            aspect_name=c.UPSTREAM_LINEAGE,
             aspect=upstream_lineage,
         )
         self.report.num_tables_with_upstream_lineage += 1
@@ -2607,14 +2683,10 @@ class TableauSiteSource:
     def get_metadata_change_proposal(
         self,
         urn: str,
-        aspect_name: str,
         aspect: Union["UpstreamLineage", "SubTypesClass"],
     ) -> MetadataWorkUnit:
         return MetadataChangeProposalWrapper(
-            entityType=c.DATASET,
-            changeType=ChangeTypeClass.UPSERT,
             entityUrn=urn,
-            aspectName=aspect_name,
             aspect=aspect,
         ).as_workunit()
 
@@ -2671,13 +2743,12 @@ class TableauSiteSource:
             dataset_snapshot.aspects.append(browse_paths)
 
         # Ownership
-        owner = (
-            self._get_ownership(datasource_info[c.OWNER][c.USERNAME])
-            if datasource_info
-            and datasource_info.get(c.OWNER)
-            and datasource_info[c.OWNER].get(c.USERNAME)
+        owner_identifier = (
+            self._get_owner_identifier(datasource_info[c.OWNER])
+            if datasource_info and datasource_info.get(c.OWNER)
             else None
         )
+        owner = self._get_ownership(owner_identifier) if owner_identifier else None
         if owner is not None:
             dataset_snapshot.aspects.append(owner)
 
@@ -2722,7 +2793,6 @@ class TableauSiteSource:
                 )
                 yield self.get_metadata_change_proposal(
                     datasource_urn,
-                    aspect_name=c.UPSTREAM_LINEAGE,
                     aspect=upstream_lineage,
                 )
                 self.report.num_tables_with_upstream_lineage += 1
@@ -2741,7 +2811,6 @@ class TableauSiteSource:
         yield self.get_metadata_change_event(dataset_snapshot)
         yield self.get_metadata_change_proposal(
             dataset_snapshot.urn,
-            aspect_name=c.SUB_TYPES,
             aspect=SubTypesClass(
                 typeNames=(
                     ["Embedded Data Source"]
@@ -2827,7 +2896,11 @@ class TableauSiteSource:
         return datasource
 
     def emit_published_datasources(self) -> Iterable[MetadataWorkUnit]:
-        datasource_filter = {c.ID_WITH_IN: self.datasource_ids_being_used}
+        datasource_filter = (
+            {}
+            if self.config.emit_all_published_datasources
+            else {c.ID_WITH_IN: self.datasource_ids_being_used}
+        )
 
         for datasource in self.get_connection_objects(
             query=published_datasource_graphql_query,
@@ -3059,7 +3132,7 @@ class TableauSiteSource:
             query_filter=sheets_filter,
             page_size=self.config.effective_sheet_page_size,
         ):
-            if self.config.ingest_hidden_assets or not self._is_hidden_view(sheet):
+            if self._should_ingest_worksheet(sheet):
                 yield from self.emit_sheets_as_charts(sheet, sheet.get(c.WORKBOOK))
             else:
                 self.report.num_hidden_assets_skipped += 1
@@ -3080,7 +3153,7 @@ class TableauSiteSource:
 
         creator: Optional[str] = None
         if workbook is not None and workbook.get(c.OWNER) is not None:
-            creator = workbook[c.OWNER].get(c.USERNAME)
+            creator = self._get_owner_identifier(workbook[c.OWNER])
         created_at = sheet.get(c.CREATED_AT, datetime.now())
         updated_at = sheet.get(c.UPDATED_AT, datetime.now())
         last_modified = self.get_last_modified(creator, created_at, updated_at)
@@ -3229,7 +3302,7 @@ class TableauSiteSource:
 
     def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUnit]:
         workbook_container_key = self.gen_workbook_key(workbook[c.ID])
-        creator = workbook.get(c.OWNER, {}).get(c.USERNAME)
+        creator = self._get_owner_identifier(workbook.get(c.OWNER, {}))
 
         owner_urn = (
             builder.make_user_urn(creator)
@@ -3252,7 +3325,7 @@ class TableauSiteSource:
 
         parent_key = None
         project_luid: Optional[str] = self._get_workbook_project_luid(workbook)
-        if project_luid and project_luid in self.tableau_project_registry.keys():
+        if project_luid and project_luid in self.tableau_project_registry:
             parent_key = self.gen_project_key(project_luid)
         else:
             workbook_id: Optional[str] = workbook.get(c.ID)
@@ -3380,7 +3453,7 @@ class TableauSiteSource:
             query_filter=dashboards_filter,
             page_size=self.config.effective_dashboard_page_size,
         ):
-            if self.config.ingest_hidden_assets or not self._is_hidden_view(dashboard):
+            if self._should_ingest_dashboard(dashboard):
                 yield from self.emit_dashboard(dashboard, dashboard.get(c.WORKBOOK))
             else:
                 self.report.num_hidden_assets_skipped += 1
@@ -3411,7 +3484,7 @@ class TableauSiteSource:
 
         creator: Optional[str] = None
         if workbook is not None and workbook.get(c.OWNER) is not None:
-            creator = workbook[c.OWNER].get(c.USERNAME)
+            creator = self._get_owner_identifier(workbook[c.OWNER])
         created_at = dashboard.get(c.CREATED_AT, datetime.now())
         updated_at = dashboard.get(c.UPDATED_AT, datetime.now())
         last_modified = self.get_last_modified(creator, created_at, updated_at)
@@ -3520,7 +3593,11 @@ class TableauSiteSource:
         return browse_paths
 
     def emit_embedded_datasources(self) -> Iterable[MetadataWorkUnit]:
-        datasource_filter = {c.ID_WITH_IN: self.embedded_datasource_ids_being_used}
+        datasource_filter = (
+            {}
+            if self.config.emit_all_embedded_datasources
+            else {c.ID_WITH_IN: self.embedded_datasource_ids_being_used}
+        )
 
         for datasource in self.get_connection_objects(
             query=embedded_datasource_graphql_query,
@@ -3554,6 +3631,20 @@ class TableauSiteSource:
             )
         return last_modified
 
+    def _get_owner_identifier(self, owner_dict: dict) -> Optional[str]:
+        """Extract owner identifier (email or username) based on configuration."""
+        if not owner_dict:
+            return None
+
+        if self.config.use_email_as_username:
+            email = owner_dict.get(c.EMAIL)
+            if email:
+                return email
+            # Fall back to username if email is not available
+            self.report.num_email_fallback_to_username += 1
+
+        return owner_dict.get(c.USERNAME)
+
     @lru_cache(maxsize=None)
     def _get_ownership(self, user: str) -> Optional[OwnershipClass]:
         if self.config.ingest_owner and user:
@@ -3632,7 +3723,7 @@ class TableauSiteSource:
                 container_key=project_key,
                 name=project_.name,
                 description=project_.description,
-                sub_types=[c.PROJECT],
+                sub_types=[BIContainerSubTypes.TABLEAU_PROJECT],
                 parent_container_key=parent_project_key,
             )
 
@@ -3650,7 +3741,7 @@ class TableauSiteSource:
         yield from gen_containers(
             container_key=self.gen_site_key(self.site_id),
             name=self.site.name or "Default",
-            sub_types=[c.SITE],
+            sub_types=[BIContainerSubTypes.TABLEAU_SITE],
         )
 
     def _fetch_groups(self):
@@ -3777,3 +3868,15 @@ class TableauSiteSource:
                     self.report.emit_upstream_tables_timer[self.site_content_url] = (
                         timer.elapsed_seconds(digits=2)
                     )
+
+            # Log owner extraction statistics if there were fallbacks
+            if (
+                self.config.use_email_as_username
+                and self.config.ingest_owner
+                and self.report.num_email_fallback_to_username > 0
+            ):
+                logger.info(
+                    f"Owner extraction summary for site '{self.site_content_url}': "
+                    f"{self.report.num_email_fallback_to_username} entities fell back from email to username "
+                    f"(email was not available)"
+                )

datahub/ingestion/source/tableau/tableau_common.py

@@ -65,6 +65,7 @@ workbook_graphql_query = """
       projectName
       owner {
         username
+        email
       }
       description
       uri
@@ -107,6 +108,7 @@ sheet_graphql_query = """
         luid
         owner {
           username
+          email
         }
     }
     datasourceFields {
@@ -185,6 +187,7 @@ dashboard_graphql_query = """
         luid
         owner {
           username
+          email
         }
     }
 }
@@ -268,6 +271,7 @@ embedded_datasource_graphql_query = """
         luid
         owner {
           username
+          email
         }
     }
 }
@@ -424,6 +428,7 @@ published_datasource_graphql_query = """
     }
     owner {
       username
+      email
     }
     description
     uri
@@ -579,10 +584,12 @@ def get_platform(connection_type: str) -> str:
         platform = "oracle"
     elif connection_type in ("tbio", "teradata"):
         platform = "teradata"
-    elif connection_type in ("sqlserver"):
+    elif connection_type in ("sqlserver",):
         platform = "mssql"
-    elif connection_type in ("athena"):
+    elif connection_type in ("athena",):
         platform = "athena"
+    elif connection_type in ("googlebigquery",):
+        platform = "bigquery"
     elif connection_type.endswith("_jdbc"):
         # e.g. convert trino_jdbc -> trino
         platform = connection_type[: -len("_jdbc")]
@@ -774,7 +781,7 @@ def get_overridden_info(
     if (
         lineage_overrides is not None
         and lineage_overrides.platform_override_map is not None
-        and original_platform in lineage_overrides.platform_override_map.keys()
+        and original_platform in lineage_overrides.platform_override_map
     ):
         platform = lineage_overrides.platform_override_map[original_platform]
 
@@ -782,7 +789,7 @@ def get_overridden_info(
         lineage_overrides is not None
         and lineage_overrides.database_override_map is not None
         and upstream_db is not None
-        and upstream_db in lineage_overrides.database_override_map.keys()
+        and upstream_db in lineage_overrides.database_override_map
     ):
         upstream_db = lineage_overrides.database_override_map[upstream_db]
 

datahub/ingestion/source/tableau/tableau_constant.py

@@ -50,7 +50,6 @@ TABLES = "tables"
 DESCRIPTION = "description"
 SQL = "SQL"
 QUERY = "query"
-SUB_TYPES = "subTypes"
 VIEW = "view"
 CUSTOM_SQL = "Custom SQL"
 REMOTE_TYPE = "remoteType"
@@ -58,9 +57,9 @@ UNKNOWN = "UNKNOWN"
 PUBLISHED_DATA_SOURCE = "PublishedDatasource"
 LUID = "luid"
 EMBEDDED_DATA_SOURCE = "EmbeddedDatasource"
-UPSTREAM_LINEAGE = "upstreamLineage"
 OWNER = "owner"
 USERNAME = "username"
+EMAIL = "email"
 HAS_EXTRACTS = "hasExtracts"
 EXTRACT_LAST_REFRESH_TIME = "extractLastRefreshTime"
 EXTRACT_LAST_INCREMENTAL_UPDATE_TIME = "extractLastIncrementalUpdateTime"
@@ -78,8 +77,6 @@ CHART = "chart"
 DASHBOARD = "dashboard"
 DASHBOARDS_CONNECTION = "dashboardsConnection"
 EMBEDDED_DATA_SOURCES_CONNECTION = "embeddedDatasourcesConnection"
-PROJECT = "Project"
-SITE = "Site"
 IS_UNSUPPORTED_CUSTOM_SQL = "isUnsupportedCustomSql"
 SITE_PERMISSION = "sitePermission"
 ROLE_SITE_ADMIN_EXPLORER = "SiteAdministratorExplorer"