openmetadata-ingestion 1.6.2.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/source/dashboard/tableau/client.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -13,20 +13,23 @@ Wrapper module of TableauServerConnection client
 """
 import math
 import traceback
-from typing import Any, Callable, Dict, List, Optional
+from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 import validators
 from cached_property import cached_property
-from packaging import version
-from tableau_api_lib import TableauServerConnection
-from tableau_api_lib.utils import extract_pages
-
-from metadata.ingestion.source.dashboard.tableau import (
-    TABLEAU_GET_VIEWS_PARAM_DICT,
-    TABLEAU_GET_WORKBOOKS_PARAM_DICT,
+from tableauserverclient import (
+    Pager,
+    PersonalAccessTokenAuth,
+    ProjectItem,
+    Server,
+    TableauAuth,
+    ViewItem,
 )
+
 from metadata.ingestion.source.dashboard.tableau.models import (
+    CustomSQLTablesResponse,
     DataSource,
+    TableauBaseModel,
     TableauChart,
     TableauDashboard,
     TableauDatasources,
@@ -35,12 +38,26 @@ from metadata.ingestion.source.dashboard.tableau.models import (
 )
 from metadata.ingestion.source.dashboard.tableau.queries import (
     TABLEAU_DATASOURCES_QUERY,
+    TALEAU_GET_CUSTOM_SQL_QUERY,
 )
 from metadata.utils.logger import ometa_logger
+from metadata.utils.ssl_manager import SSLManager
 
 logger = ometa_logger()
 
 
+class TableauWorkBookException(Exception):
+    """
+    Raise when Workbooks information is not retrieved from the Tableau APIs
+    """
+
+
+class TableauChartsException(Exception):
+    """
+    Raise when Charts information is not retrieved from the Tableau APIs
+    """
+
+
 class TableauOwnersNotFound(Exception):
     """
     Raise when Owner information is not retrieved from the Tableau APIs
@@ -58,129 +75,219 @@ class TableauClient:
     Wrapper to TableauServerConnection
     """
 
-    _client: TableauServerConnection
-
     def __init__(
         self,
-        tableau_server_config: Dict[str, Dict[str, Any]],
+        tableau_server_auth: Union[PersonalAccessTokenAuth, TableauAuth],
         config,
-        env: str,
-        ssl_verify: bool,
+        verify_ssl: Union[bool, str],
         pagination_limit: int,
+        ssl_manager: Optional[SSLManager] = None,
     ):
-        # ssl_verify is typed as a `bool` in TableauServerConnection
-        # However, it is passed as `verify=self.ssl_verify` in each `requests` call.
-        # In requests (https://requests.readthedocs.io/en/latest/user/advanced.html?highlight=ssl#ssl-cert-verification)
-        # the param can be None, False to ignore HTTPS certs or a string with the path to the cert.
-        self._client = TableauServerConnection(
-            config_json=tableau_server_config,
-            env=env,
-            ssl_verify=ssl_verify,
-        )
+        self.tableau_server = Server(str(config.hostPort), use_server_version=True)
+        if config.apiVersion:
+            self.tableau_server.version = config.apiVersion
+        self.tableau_server.add_http_options({"verify": verify_ssl})
+        self.tableau_server.auth.sign_in(tableau_server_auth)
         self.config = config
-        self._client.sign_in().json()
         self.pagination_limit = pagination_limit
+        self.custom_sql_table_queries: Dict[str, List[str]] = {}
+        self.owner_cache: Dict[str, TableauOwner] = {}
+        self.all_projects: List[ProjectItem] = []
+        self.ssl_manager = ssl_manager
 
     @cached_property
     def server_info(self) -> Callable:
-        return self._client.server_info
+        return self.tableau_server.server_info.get
 
-    @property
     def server_api_version(self) -> str:
-        return self.server_info().json()["serverInfo"]["restApiVersion"]
+        return self.tableau_server.version
 
     @property
     def site_id(self) -> str:
-        return self._client.site_id
+        return self.tableau_server.site_id
 
-    @property
-    def query_workbooks_for_site(self) -> Callable:
-        return self._client.query_workbooks_for_site
+    def get_tableau_owner(
+        self, owner_id: str, include_owners: bool = True
+    ) -> Optional[TableauOwner]:
+        """
+        Get tableau owner with optional include_owners flag
+        """
+        try:
+            if not include_owners:
+                return None
+            if owner_id in self.owner_cache:
+                return self.owner_cache[owner_id]
+            owner = self.tableau_server.users.get_by_id(owner_id) if owner_id else None
+            if owner and owner.email:
+                owner_obj = TableauOwner(
+                    id=str(owner.id), name=owner.name, email=owner.email
+                )
+                self.owner_cache[owner_id] = owner_obj
+                return owner_obj
+        except Exception as err:
+            logger.debug(f"Failed to fetch owner details for ID {owner_id}: {str(err)}")
+        return None
 
-    @property
-    def query_views_for_site(self) -> Callable:
-        return self._client.query_views_for_site
+    def get_workbook_charts_and_user_count(
+        self, views: List[ViewItem], include_owners: bool = True
+    ) -> Optional[Tuple[Optional[int], Optional[List[TableauChart]]]]:
+        """
+        Fetches workbook charts and dashboard user view count
+        """
+        view_count = 0
+        charts: Optional[List[TableauChart]] = []
+        for view in views or []:
+            try:
+                charts.append(
+                    TableauChart(
+                        id=str(view.id),
+                        name=view.name,
+                        tags=view.tags,
+                        owner=self.get_tableau_owner(view.owner_id, include_owners),
+                        contentUrl=view.content_url,
+                        sheetType=view.sheet_type,
+                    )
+                )
+                view_count += view.total_views
+            except AttributeError as e:
+                logger.debug(
+                    f"Failed to process view due to missing attribute: {str(e)}"
+                )
+                continue
+            except Exception as e:
+                logger.debug(f"Failed to process view: {str(e)}")
+                continue
 
-    def get_owners(self) -> Optional[List[TableauOwner]]:
-        owners = [workbook.owner for workbook in self.get_workbooks()]
-        if len(owners) > 0:
-            return owners
-        raise TableauOwnersNotFound(
-            "Unable to fetch Dashboard Owners from tableau\n"
-            "Please check if the user has permissions to access the Owner information"
-        )
+        return charts, view_count
 
-    def get_workbooks(self) -> List[TableauDashboard]:
-        return [
-            TableauDashboard(**workbook)
-            for workbook in extract_pages(
-                self.query_workbooks_for_site,
-                parameter_dict=TABLEAU_GET_WORKBOOKS_PARAM_DICT,
-            )
-        ]
-
-    def get_charts(self) -> List[TableauChart]:
-        # For charts, we can also pick up usage as a field
-        return [
-            TableauChart(**chart)
-            for chart in extract_pages(
-                self.query_views_for_site,
-                content_id=self.site_id,
-                parameter_dict=TABLEAU_GET_VIEWS_PARAM_DICT,
-            )
-        ]
+    def get_all_projects(self) -> None:
+        """
+        Get all projects from the tableau server
+        """
+        try:
+            logger.debug("Getting all projects from the tableau server")
+            all_projects: List[ProjectItem] = []
+            for project in Pager(self.tableau_server.projects):
+                all_projects.append(project)
+            self.all_projects = all_projects
+        except Exception as e:
+            logger.debug(f"Failed to get all projects: {str(e)}")
 
-    def get_workbook_charts(self, dashboard_id: str) -> Optional[List[TableauChart]]:
+    def get_project_parents_by_id(self, project_id: str) -> Optional[str]:
         """
-        Get the charts for a workbook
+        Get the parents of a project by id
         """
         try:
-            return [
-                TableauChart(**chart)
-                for chart in self._client.query_views_for_workbook(
-                    workbook_id=dashboard_id,
-                    parameter_dict=TABLEAU_GET_VIEWS_PARAM_DICT,
-                ).json()["views"]["view"]
-            ]
-        except Exception as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(
-                f"Error processing charts for dashboard [{dashboard_id}]: {exc}"
-            )
+            parent_projects = []
+            current_project_id = project_id
+
+            while current_project_id:
+                # Find project with current ID
+                project = next(
+                    (
+                        proj
+                        for proj in self.all_projects
+                        if str(proj.id) == str(current_project_id)
+                    ),
+                    None,
+                )
+
+                if not project:
+                    break
+
+                parent_projects.append(project.name)
+
+                # Get parent ID and continue loop if exists
+                current_project_id = (
+                    project.parent_id if hasattr(project, "parent_id") else None
+                )
+
+            if parent_projects:
+                parent_projects = ".".join(reversed(parent_projects))
+                return parent_projects
+        except Exception as e:
+            logger.debug(f"Failed to get project parents by id: {str(e)}")
         return None
 
-    def test_api_version(self):
+    def get_workbooks(self, include_owners: bool = True) -> Iterable[TableauDashboard]:
         """
-        Method to validate the declared v/s server tableau rest api version
+        Fetch all tableau workbooks
         """
-        server_api_version = version.parse(self.server_api_version)
-        declared_api_version = version.parse(self.config.apiVersion)
-        if declared_api_version > server_api_version:
-            raise ValueError(
-                f"""
-            Your API version of '{declared_api_version}' is too damn high!
-            The server you are establishing a connection with is using REST API version '{server_api_version}'.
-            """
-            )
-        if declared_api_version < server_api_version:
-            raise ValueError(
-                f"""
-            The Tableau Server REST API version you specified is lower than the version your server uses.
-            Your Tableau Server is on REST API version {server_api_version}.
-            The REST API version you specified is {declared_api_version}.
-            For optimal results, please change the 'api_version' config variable to {server_api_version}.
-            """
-            )
+        self.get_all_projects()
+        self.cache_custom_sql_tables()
+        for workbook in Pager(self.tableau_server.workbooks):
+            try:
+                self.tableau_server.workbooks.populate_views(workbook, usage=True)
+                charts, user_views = self.get_workbook_charts_and_user_count(
+                    workbook.views, include_owners
+                )
+                workbook = TableauDashboard(
+                    id=str(workbook.id),
+                    name=workbook.name,
+                    project=TableauBaseModel(
+                        id=str(workbook.project_id), name=workbook.project_name
+                    ),
+                    owner=self.get_tableau_owner(workbook.owner_id, include_owners),
+                    description=workbook.description,
+                    tags=workbook.tags,
+                    webpageUrl=workbook.webpage_url,
+                    charts=charts,
+                    user_views=user_views,
+                )
+                yield workbook
+            except AttributeError as err:
+                logger.warning(
+                    f"Failed to process workbook due to missing attribute: {str(err)}"
+                )
+                continue
+            except Exception as err:
+                logger.warning(f"Failed to process workbook: {str(err)}")
+                continue
+
+    def test_get_workbooks(self):
+        for workbook in Pager(self.tableau_server.workbooks):
+            if workbook.id is not None:
+                self.tableau_server.workbooks.populate_views(workbook, usage=True)
+                return workbook
+            break
+        raise TableauWorkBookException(
+            "Unable to fetch Dashboards from tableau\n"
+            "Please check if the user has permissions to access the Dashboards information"
+        )
+
+    def test_get_workbook_views(self, include_owners: bool = True):
+        workbook = self.test_get_workbooks()
+        charts, _ = self.get_workbook_charts_and_user_count(
+            workbook.views, include_owners
+        )
+        if charts:
+            return True
+        raise TableauChartsException(
+            "Unable to fetch Charts from tableau\n"
+            "Please check if the user has permissions to access the Charts information"
+        )
+
+    def test_get_owners(
+        self, include_owners: bool = True
+    ) -> Optional[List[TableauOwner]]:
+        workbook = self.test_get_workbooks()
+        owners = self.get_tableau_owner(workbook.owner_id, include_owners)
+        if owners is not None:
+            return owners
+        raise TableauOwnersNotFound(
+            "Unable to fetch Dashboard Owners from tableau\n"
+            "Please check if the user has permissions to access the Owner information"
+        )
 
     def test_site_url(self):
         """
         Method to test the site url and site name fields
         """
-        validation = validators.url(self.config.siteUrl)
+        validation = validators.url(self.config.siteName)
         if validation:
             raise ValueError(
                 f"""
-            The site url "{self.config.siteUrl}" is in incorrect format.
+            The site url "{self.config.siteName}" is in incorrect format.
             If "https://xxx.tableau.com/#/site/MarketingTeam/home" represents the homepage url for your tableau site,
             the "MarketingTeam" from the url should be entered in the Site Name and Site Url fields.
             """
@@ -191,17 +298,16 @@ class TableauClient:
         """
         Method to test the datamodels
         """
-        workbooks = self.get_workbooks()
+        workbook = self.test_get_workbooks()
 
-        if len(workbooks) == 0:
+        if workbook.id is None:
             raise TableauDataModelsException(
                 "Unable to get any workbooks to fetch tableau data sources"
             )
 
         # Take the 1st workbook's id and pass to the graphql query
-        test_workbook = workbooks[0]
         data = self._query_datasources(
-            dashboard_id=test_workbook.id, entities_per_page=1, offset=0
+            dashboard_id=workbook.id, entities_per_page=1, offset=0
         )
         if data:
             return data
@@ -220,18 +326,23 @@ class TableauClient:
         Method to query the graphql endpoint to get data sources
         """
         try:
-            datasources_graphql_result = self._client.metadata_graphql_query(
+            datasources_graphql_result = self.tableau_server.metadata.query(
                 query=TABLEAU_DATASOURCES_QUERY.format(
                     workbook_id=dashboard_id, first=entities_per_page, offset=offset
                 )
             )
-            if datasources_graphql_result:
-                resp = datasources_graphql_result.json()
-                if resp and resp.get("data"):
+            if datasources_graphql_result and datasources_graphql_result.get("data"):
+                if datasources_graphql_result["data"].get("workbooks"):
                     tableau_datasource_connection = TableauDatasourcesConnection(
-                        **resp["data"]["workbooks"][0]
+                        **datasources_graphql_result["data"]["workbooks"][0]
                     )
                     return tableau_datasource_connection.embeddedDatasourcesConnection
+                else:
+                    logger.warning(
+                        f"No Datasources found in GraphQL datasources query result for the workbook {dashboard_id}. "
+                        "If this is a recently created or updated workbook, it may take some time "
+                        f"to become available for querying via the GraphQL API. : \n graphql = {datasources_graphql_result}\n"
+                    )
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning(
@@ -270,7 +381,62 @@ class TableauClient:
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning("Unable to fetch Data Sources")
+        return []
+
+    def get_custom_sql_table_queries(self, datasource_id: str) -> Optional[List[str]]:
+        """
+        Get custom SQL table queries for a specific dashboard/workbook ID
+        """
+        logger.debug(f"Getting custom SQL table queries for datasource {datasource_id}")
+
+        if datasource_id in self.custom_sql_table_queries:
+            logger.debug(f"Found cached queries for datasource {datasource_id}")
+            return self.custom_sql_table_queries[datasource_id]
+        logger.debug(f"No cached queries for datasource {datasource_id}")
         return None
 
+    def cache_custom_sql_tables(self) -> None:
+        """
+        Fetch all custom SQL tables and cache their queries by workbook ID
+        """
+        try:
+            result = self.tableau_server.metadata.query(
+                query=TALEAU_GET_CUSTOM_SQL_QUERY
+            )
+            if not result:
+                logger.debug("No result returned from GraphQL query")
+                return
+
+            response = CustomSQLTablesResponse(**result)
+            if not response.data:
+                logger.debug("No data found in GraphQL response")
+                return
+
+            for tables in response.data.values():
+                for table in tables:
+                    if not (table.query and table.downstreamDatasources):
+                        logger.debug(
+                            f"Skipping table {table} - missing query or workbooks"
+                        )
+                        continue
+
+                    query = table.query
+                    for datasource in table.downstreamDatasources:
+                        self.custom_sql_table_queries.setdefault(
+                            datasource.id, []
+                        ).append(query)
+
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Unable to fetch Custom SQL Tables")
+
     def sign_out(self) -> None:
-        self._client.sign_out()
+        self.tableau_server.auth.sign_out()
+        self.cleanup()
+
+    def cleanup(self) -> None:
+        """
+        Clean up temporary SSL files if they exist
+        """
+        if self.ssl_manager:
+            self.ssl_manager.cleanup_temp_files()

metadata/ingestion/source/dashboard/tableau/connection.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -13,10 +13,9 @@
 Source connection handler
 """
 import traceback
-from functools import partial
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Union
 
-from tableau_api_lib.utils import extract_pages
+import tableauserverclient as TSC
 
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
@@ -36,14 +35,10 @@ from metadata.ingestion.connections.test_connections import (
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.dashboard.tableau import (
-    TABLEAU_GET_VIEWS_PARAM_DICT,
-    TABLEAU_GET_WORKBOOKS_PARAM_DICT,
-)
 from metadata.ingestion.source.dashboard.tableau.client import TableauClient
 from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
-from metadata.utils.ssl_registry import get_verify_ssl_fn
+from metadata.utils.ssl_manager import SSLManager
 
 logger = ingestion_logger()
 
@@ -52,15 +47,15 @@ def get_connection(connection: TableauConnection) -> TableauClient:
     """
     Create connection
     """
-    tableau_server_config = build_server_config(connection)
-    get_verify_ssl = get_verify_ssl_fn(connection.verifySSL)
+    tableau_server_auth = build_server_config(connection)
+    verify_ssl, ssl_manager = set_verify_ssl(connection)
     try:
         return TableauClient(
-            tableau_server_config=tableau_server_config,
+            tableau_server_auth=tableau_server_auth,
             config=connection,
-            env=connection.env,
-            ssl_verify=get_verify_ssl(connection.sslConfig),
+            verify_ssl=verify_ssl,
             pagination_limit=connection.paginationLimit,
+            ssl_manager=ssl_manager,
         )
     except Exception as exc:
         logger.debug(traceback.format_exc())
@@ -69,6 +64,48 @@ def get_connection(connection: TableauConnection) -> TableauClient:
         )
 
 
+def set_verify_ssl(
+    connection: TableauConnection,
+) -> tuple[Union[bool, str], Optional[SSLManager]]:
+    """
+    Set verify ssl based on connection configuration
+    ref: https://tableau.github.io/server-client-python/docs/sign-in-out#handling-ssl-certificates-for-tableau-server
+    """
+    if connection.verifySSL.value == "no-ssl":
+        return None, None
+
+    if connection.verifySSL.value == "ignore":
+        return False, None
+
+    if connection.verifySSL.value == "validate":
+        # Use SSLManager to create temporary certificate files
+        if not connection.sslConfig:
+            raise ValueError(
+                "SSL Config is required when verifySSL is set to 'validate'. "
+                "Please provide CA certificate, SSL certificate, or SSL key."
+            )
+
+        # Create SSLManager to handle certificate files
+        ssl_manager = SSLManager(
+            ca=connection.sslConfig.root.caCertificate,
+            cert=connection.sslConfig.root.sslCertificate,
+            key=connection.sslConfig.root.sslKey,
+        )
+
+        # Return the CA certificate file path for verification
+        # If no CA certificate is provided, use default verification
+        if ssl_manager.ca_file_path:
+            return ssl_manager.ca_file_path, ssl_manager
+        else:
+            # If no CA certificate is provided but SSL is enabled, use default verification
+            return True, ssl_manager
+
+    raise ValueError(
+        f"Unsupported verifySSL value: {connection.verifySSL.value}. "
+        "Expected one of ['no-ssl', 'ignore', 'validate']."
+    )
+
+
 def test_connection(
     metadata: OpenMetadata,
     client: TableauClient,
@@ -83,26 +120,11 @@ def test_connection(
 
     test_fn = {
         "ServerInfo": client.server_info,
-        # The Tableau server_info API doesn't provide direct access to the API version.
-        # This is due to the "api_version" being a mandatory field for the tableau library's connection class.
-        # Without this information, requests to the Tableau server cannot be made,
-        # including fetching the server info containing the "api_version".
-        # Consequently, we'll compare the declared api_version with the server's api_version during the test connection
-        # once the tableau library's connection class is initialized.
-        "ValidateApiVersion": client.test_api_version,
+        "ValidateApiVersion": client.server_api_version,
         "ValidateSiteUrl": client.test_site_url,
-        "GetWorkbooks": partial(
-            extract_pages,
-            query_func=client.query_workbooks_for_site,
-            parameter_dict=TABLEAU_GET_WORKBOOKS_PARAM_DICT,
-        ),
-        "GetViews": partial(
-            extract_pages,
-            query_func=client.query_views_for_site,
-            content_id=client.site_id,
-            parameter_dict=TABLEAU_GET_VIEWS_PARAM_DICT,
-        ),
-        "GetOwners": client.get_owners,
+        "GetWorkbooks": client.test_get_workbooks,
+        "GetViews": client.test_get_workbook_views,
+        "GetOwners": client.test_get_owners,
         "GetDataModels": client.test_get_datamodels,
     }
 
@@ -123,24 +145,20 @@ def build_server_config(connection: TableauConnection) -> Dict[str, Dict[str, An
     Returns:
         Client configuration
     """
-    tableau_server_config = {
-        f"{connection.env}": {
-            "server": str(connection.hostPort),
-            "api_version": str(connection.apiVersion),
-            "site_name": connection.siteName if connection.siteName else "",
-            "site_url": connection.siteUrl if connection.siteUrl else "",
-        }
-    }
+
     if isinstance(connection.authType, BasicAuth):
-        tableau_server_config[connection.env]["username"] = connection.authType.username
-        tableau_server_config[connection.env][
-            "password"
-        ] = connection.authType.password.get_secret_value()
+        tableau_auth = TSC.TableauAuth(
+            username=connection.authType.username,
+            password=connection.authType.password.get_secret_value(),
+            site_id=connection.siteName if connection.siteName else "",
+        )
     elif isinstance(connection.authType, AccessTokenAuth):
-        tableau_server_config[connection.env][
-            "personal_access_token_name"
-        ] = connection.authType.personalAccessTokenName
-        tableau_server_config[connection.env][
-            "personal_access_token_secret"
-        ] = connection.authType.personalAccessTokenSecret.get_secret_value()
-    return tableau_server_config
+        tableau_auth = TSC.PersonalAccessTokenAuth(
+            token_name=connection.authType.personalAccessTokenName,
+            personal_access_token=connection.authType.personalAccessTokenSecret.get_secret_value(),
+            site_id=connection.siteName if connection.siteName else "",
+        )
+    else:
+        raise ValueError("Unsupported authentication type")
+
+    return tableau_auth