openmetadata-ingestion 1.2.5.1__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/ingestion/source/dashboard/tableau/client.py

@@ -15,7 +15,9 @@ import math
 import traceback
 from typing import Any, Callable, Dict, List, Optional
 
+import validators
 from cached_property import cached_property
+from packaging import version
 from tableau_api_lib import TableauServerConnection
 from tableau_api_lib.utils import extract_pages
 
@@ -60,20 +62,22 @@ class TableauClient:
 
     def __init__(
         self,
-        config: Dict[str, Dict[str, Any]],
+        tableau_server_config: Dict[str, Dict[str, Any]],
+        config,
         env: str,
         ssl_verify: bool,
         pagination_limit: int,
     ):
         # ssl_verify is typed as a `bool` in TableauServerConnection
         # However, it is passed as `verify=self.ssl_verify` in each `requests` call.
-        # In requests (https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification)
+        # In requests (https://requests.readthedocs.io/en/latest/user/advanced.html?highlight=ssl#ssl-cert-verification)
         # the param can be None, False to ignore HTTPS certs or a string with the path to the cert.
         self._client = TableauServerConnection(
-            config_json=config,
+            config_json=tableau_server_config,
             env=env,
             ssl_verify=ssl_verify,
         )
+        self.config = config
         self._client.sign_in().json()
         self.pagination_limit = pagination_limit
 
@@ -81,6 +85,10 @@ class TableauClient:
     def server_info(self) -> Callable:
         return self._client.server_info
 
+    @property
+    def server_api_version(self) -> str:
+        return self.server_info().json()["serverInfo"]["restApiVersion"]
+
     @property
     def site_id(self) -> str:
         return self._client.site_id
@@ -122,11 +130,79 @@ class TableauClient:
             )
         ]
 
+    def get_workbook_charts(self, dashboard_id: str) -> Optional[List[TableauChart]]:
+        """
+        Get the charts for a workbook
+        """
+        try:
+            return [
+                TableauChart(**chart)
+                for chart in self._client.query_views_for_workbook(
+                    workbook_id=dashboard_id,
+                    parameter_dict=TABLEAU_GET_VIEWS_PARAM_DICT,
+                ).json()["views"]["view"]
+            ]
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Error processing charts for dashboard [{dashboard_id}]: {exc}"
+            )
+        return None
+
+    def test_api_version(self):
+        """
+        Method to validate the declared v/s server tableau rest api version
+        """
+        server_api_version = version.parse(self.server_api_version)
+        declared_api_version = version.parse(self.config.apiVersion)
+        if declared_api_version > server_api_version:
+            raise ValueError(
+                f"""
+            Your API version of '{declared_api_version}' is too damn high!
+            The server you are establishing a connection with is using REST API version '{server_api_version}'.
+            """
+            )
+        if declared_api_version < server_api_version:
+            raise ValueError(
+                f"""
+            The Tableau Server REST API version you specified is lower than the version your server uses.
+            Your Tableau Server is on REST API version {server_api_version}.
+            The REST API version you specified is {declared_api_version}.
+            For optimal results, please change the 'api_version' config variable to {server_api_version}.
+            """
+            )
+
+    def test_site_url(self):
+        """
+        Method to test the site url and site name fields
+        """
+        validation = validators.url(self.config.siteUrl)
+        if validation:
+            raise ValueError(
+                f"""
+            The site url "{self.config.siteUrl}" is in incorrect format.
+            If "https://xxx.tableau.com/#/site/MarketingTeam/home" represents the homepage url for your tableau site,
+            the "MarketingTeam" from the url should be entered in the Site Name and Site Url fields.
+            """
+            )
+        return True
+
     def test_get_datamodels(self):
         """
         Method to test the datamodels
         """
-        data = self._query_datasources(entities_per_page=1, offset=0)
+        workbooks = self.get_workbooks()
+
+        if len(workbooks) == 0:
+            raise TableauDataModelsException(
+                "Unable to get any workbooks to fetch tableau data sources"
+            )
+
+        # Take the 1st workbook's id and pass to the graphql query
+        test_workbook = workbooks[0]
+        data = self._query_datasources(
+            dashboard_id=test_workbook.id, entities_per_page=1, offset=0
+        )
         if data:
             return data
         raise TableauDataModelsException(
@@ -138,7 +214,7 @@ class TableauClient:
         )
 
     def _query_datasources(
-        self, entities_per_page: int, offset: int
+        self, dashboard_id: str, entities_per_page: int, offset: int
     ) -> Optional[TableauDatasources]:
         """
         Method to query the graphql endpoint to get data sources
@@ -146,14 +222,14 @@ class TableauClient:
         try:
             datasources_graphql_result = self._client.metadata_graphql_query(
                 query=TABLEAU_DATASOURCES_QUERY.format(
-                    first=entities_per_page, offset=offset
+                    workbook_id=dashboard_id, first=entities_per_page, offset=offset
                 )
             )
             if datasources_graphql_result:
                 resp = datasources_graphql_result.json()
                 if resp and resp.get("data"):
                     tableau_datasource_connection = TableauDatasourcesConnection(
-                        **resp.get("data")
+                        **resp["data"]["workbooks"][0]
                     )
                     return tableau_datasource_connection.embeddedDatasourcesConnection
         except Exception:
@@ -167,13 +243,15 @@ class TableauClient:
             )
         return None
 
-    def get_datasources(self) -> Optional[List[DataSource]]:
+    def get_datasources(self, dashboard_id: str) -> Optional[List[DataSource]]:
         """
-        Paginate and get the list of all data sources
+        Paginate and get the list of all data sources of the workbook
         """
         try:
             # Query the graphql endpoint once to get total count of data sources
-            tableau_datasource = self._query_datasources(entities_per_page=1, offset=1)
+            tableau_datasource = self._query_datasources(
+                dashboard_id=dashboard_id, entities_per_page=1, offset=1
+            )
             entities_per_page = min(50, self.pagination_limit)
             indexes = math.ceil(tableau_datasource.totalCount / entities_per_page)
 
@@ -182,7 +260,9 @@ class TableauClient:
             for index in range(indexes):
                 offset = index * entities_per_page
                 tableau_datasource = self._query_datasources(
-                    entities_per_page=entities_per_page, offset=offset
+                    dashboard_id=dashboard_id,
+                    entities_per_page=entities_per_page,
+                    offset=offset,
                 )
                 if tableau_datasource:
                     data_sources.extend(tableau_datasource.nodes)

metadata/ingestion/source/dashboard/tableau/connection.py

@@ -52,7 +52,8 @@ def get_connection(connection: TableauConnection) -> TableauClient:
     get_verify_ssl = get_verify_ssl_fn(connection.verifySSL)
     try:
         return TableauClient(
-            config=tableau_server_config,
+            tableau_server_config=tableau_server_config,
+            config=connection,
             env=connection.env,
             ssl_verify=get_verify_ssl(connection.sslConfig),
             pagination_limit=connection.paginationLimit,
@@ -77,6 +78,14 @@ def test_connection(
 
     test_fn = {
         "ServerInfo": client.server_info,
+        # The Tableau server_info API doesn't provide direct access to the API version.
+        # This is due to the "api_version" being a mandatory field for the tableau library's connection class.
+        # Without this information, requests to the Tableau server cannot be made,
+        # including fetching the server info containing the "api_version".
+        # Consequently, we'll compare the declared api_version with the server's api_version during the test connection
+        # once the tableau library's connection class is initialized.
+        "ValidateApiVersion": client.test_api_version,
+        "ValidateSiteUrl": client.test_site_url,
         "GetWorkbooks": partial(
             extract_pages,
             query_func=client.query_workbooks_for_site,

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -14,6 +14,8 @@ Tableau source module
 import traceback
 from typing import Any, Iterable, List, Optional, Set
 
+from requests.utils import urlparse
+
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
 from metadata.generated.schema.api.data.createDashboardDataModel import (
@@ -39,11 +41,14 @@ from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -82,51 +87,6 @@ class TableauSource(DashboardServiceSource):
     metadata_config: OpenMetadataConnection
     client: TableauClient
 
-    def __init__(
-        self,
-        config: WorkflowSource,
-        metadata: OpenMetadata,
-    ):
-        super().__init__(config, metadata)
-        self.workbooks: List[
-            TableauDashboard
-        ] = []  # We will populate this in `prepare`
-        self.tags: Set[TableauTag] = set()
-
-    def prepare(self):
-        """Restructure the API response"""
-        try:
-            # get workbooks which are considered Dashboards in OM
-            self.workbooks = self.client.get_workbooks()
-
-            # get views which are considered charts in OM
-            charts = self.client.get_charts()
-
-            # get datasources which are considered as datamodels in OM
-            data_models = self.client.get_datasources()
-
-            # add all the charts (views) and datasources from the API to each workbook
-            for workbook in self.workbooks:
-                workbook.charts = [
-                    chart for chart in charts if chart.workbook.id == workbook.id
-                ]
-
-                for data_model in data_models or []:
-                    if data_model.workbook and data_model.workbook.luid == workbook.id:
-                        workbook.dataModels.append(data_model)
-
-            # collect all the tags from charts and workbooks before yielding final entities
-            if self.source_config.includeTags:
-                for container in [self.workbooks, charts]:
-                    for elem in container:
-                        self.tags.update(elem.tags)
-
-        except Exception:
-            logger.debug(traceback.format_exc())
-            logger.error("Error in fetching the Tableau Workbook metadata")
-
-        return super().prepare()
-
     @classmethod
     def create(cls, config_dict: dict, metadata: OpenMetadata):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
@@ -138,36 +98,59 @@ class TableauSource(DashboardServiceSource):
         return cls(config, metadata)
 
     def get_dashboards_list(self) -> Optional[List[TableauDashboard]]:
-        return self.workbooks
+        return self.client.get_workbooks()
 
     def get_dashboard_name(self, dashboard: TableauDashboard) -> str:
         return dashboard.name
 
     def get_dashboard_details(self, dashboard: TableauDashboard) -> TableauDashboard:
         """
-        Get Dashboard Details. Returning the identity here as we prepare everything
-        during the `prepare` stage
+        Get Dashboard Details including the dashboard charts and datamodels
         """
+
+        # Get the tableau views/sheets
+        dashboard.charts = self.client.get_workbook_charts(dashboard_id=dashboard.id)
+
+        # Get the tableau data sources
+        dashboard.dataModels = self.client.get_datasources(dashboard_id=dashboard.id)
+
         return dashboard
 
-    def get_owner_details(
+    def get_owner_ref(
         self, dashboard_details: TableauDashboard
     ) -> Optional[EntityReference]:
-        """Get dashboard owner from email"""
-        if dashboard_details.owner and dashboard_details.owner.email:
-            user = self.metadata.get_user_by_email(dashboard_details.owner.email)
-            if user:
-                return EntityReference(id=user.id.__root__, type="user")
+        """
+        Get dashboard owner from email
+        """
+        try:
+            if dashboard_details.owner and dashboard_details.owner.email:
+                return self.metadata.get_reference_by_email(
+                    dashboard_details.owner.email
+                )
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not fetch owner data due to {err}")
         return None
 
-    def yield_tag(self, *_, **__) -> Iterable[Either[OMetaTagAndClassification]]:
-        yield from get_ometa_tag_and_classification(
-            tags=[tag.label for tag in self.tags],
-            classification_name=TABLEAU_TAG_CATEGORY,
-            tag_description="Tableau Tag",
-            classification_description="Tags associated with tableau entities",
-            include_tags=self.source_config.includeTags,
-        )
+    def yield_tags(
+        self, dashboard_details: TableauDashboard
+    ) -> Iterable[Either[OMetaTagAndClassification]]:
+        """
+        Method to yield tags related to specific dashboards
+        """
+        if self.source_config.includeTags:
+            tags: Set[TableauTag] = set()
+            for container in [[dashboard_details], dashboard_details.charts]:
+                for elem in container:
+                    tags.update(elem.tags)
+
+            yield from get_ometa_tag_and_classification(
+                tags=[tag.label for tag in tags],
+                classification_name=TABLEAU_TAG_CATEGORY,
+                tag_description="Tableau Tag",
+                classification_description="Tags associated with tableau entities",
+                include_tags=self.source_config.includeTags,
+            )
 
     def yield_datamodel(
         self, dashboard_details: TableauDashboard
@@ -190,14 +173,14 @@ class TableauSource(DashboardServiceSource):
                         columns=self.get_column_info(data_model),
                     )
                     yield Either(right=data_model_request)
-                    self.register_record_datamodel(datamodel_requst=data_model_request)
+                    self.register_record_datamodel(datamodel_request=data_model_request)
 
                 except Exception as exc:
                     yield Either(
                         left=StackTraceError(
                             name=data_model_name,
                             error=f"Error yielding Data Model [{data_model_name}]: {exc}",
-                            stack_trace=traceback.format_exc(),
+                            stackTrace=traceback.format_exc(),
                         )
                     )
 
@@ -214,6 +197,10 @@ class TableauSource(DashboardServiceSource):
         topology. And they are cleared after processing each Dashboard because of the 'clear_cache' option.
         """
         try:
+            dashboard_url = (
+                f"{clean_uri(str(self.config.serviceConnection.__root__.config.hostPort))}"
+                f"/#{urlparse(dashboard_details.webpageUrl).fragment}/views"
+            )
             dashboard_request = CreateDashboardRequest(
                 name=dashboard_details.id,
                 displayName=dashboard_details.name,
@@ -226,7 +213,7 @@ class TableauSource(DashboardServiceSource):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 dataModels=[
                     fqn.build(
@@ -243,8 +230,9 @@ class TableauSource(DashboardServiceSource):
                     classification_name=TABLEAU_TAG_CATEGORY,
                     include_tags=self.source_config.includeTags,
                 ),
-                sourceUrl=dashboard_details.webpageUrl,
+                sourceUrl=dashboard_url,
                 service=self.context.dashboard_service,
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -253,7 +241,7 @@ class TableauSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=dashboard_details.id,
                     error=f"Error to yield dashboard for {dashboard_details}: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -294,7 +282,7 @@ class TableauSource(DashboardServiceSource):
                             "Error to yield dashboard lineage details for DB "
                             f"service name [{db_service_name}]: {err}"
                         ),
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -342,7 +330,7 @@ class TableauSource(DashboardServiceSource):
                     left=StackTraceError(
                         name="Chart",
                         error=f"Error to yield dashboard chart [{chart}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 

metadata/ingestion/source/dashboard/tableau/models.py

@@ -99,12 +99,6 @@ class DatasourceField(BaseModel):
     description: Optional[str]
 
 
-class Workbook(BaseModel):
-    id: str
-    luid: str
-    name: Optional[str]
-
-
 class UpstreamTableColumn(BaseModel):
     id: str
     name: Optional[str]
@@ -130,7 +124,6 @@ class DataSource(BaseModel):
     id: str
     name: Optional[str]
     fields: Optional[List[DatasourceField]]
-    workbook: Optional[Workbook]
     upstreamTables: Optional[List[UpstreamTable]]
 
 
@@ -148,7 +141,6 @@ class TableauChart(TableauBaseModel):
     Aux class for Chart object of the tableau_api_lib response
     """
 
-    workbook: TableauBaseModel
     owner: Optional[TableauOwner]
     tags: Optional[List[TableauTag]] = []
     _extract_tags = validator("tags", pre=True, allow_reuse=True)(transform_tags)

metadata/ingestion/source/dashboard/tableau/queries.py

@@ -15,6 +15,10 @@ GraphQL queries used during ingestion
 
 TABLEAU_DATASOURCES_QUERY = """
 {{
+workbooks(filter:{{luid: "{workbook_id}"}}){{
+  id
+  luid
+  name
   embeddedDatasourcesConnection(first: {first}, offset: {offset} ) {{
     nodes {{
       id
@@ -29,11 +33,6 @@ TABLEAU_DATASOURCES_QUERY = """
         }}
         description
       }}
-      workbook {{
-        id
-        luid
-        name
-      }}
       upstreamTables {{
         id
         luid
@@ -57,5 +56,6 @@ TABLEAU_DATASOURCES_QUERY = """
     }}
     totalCount
   }}
+  }}
 }}
 """

metadata/ingestion/source/database/athena/client.py

@@ -0,0 +1,80 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Wrapper module of Athena client
+"""
+import traceback
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.athenaConnection import (
+    AthenaConnection,
+)
+from metadata.ingestion.models.lf_tags_model import LFTags
+from metadata.ingestion.source.database.athena.connection import (
+    get_lake_formation_client,
+)
+from metadata.utils.logger import ometa_logger
+
+logger = ometa_logger()
+
+
+class AthenaLakeFormationClient:
+    """
+    Athena Lake Formation Client
+    """
+
+    def __init__(
+        self,
+        connection: AthenaConnection,
+    ):
+        self.lake_formation_client = get_lake_formation_client(connection=connection)
+
+    def get_database_tags(self, name: str) -> Optional[LFTags]:
+        """
+        Method to call the API and get the database tags
+        """
+        try:
+            response = self.lake_formation_client.get_resource_lf_tags(
+                Resource={"Database": {"Name": name}}
+            )
+            lf_tags = LFTags(**response)
+            return lf_tags.LFTagOnDatabase
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Unable to get lf tags for database resource [{name}] due to: {exc}"
+            )
+        return None
+
+    def get_table_and_column_tags(self, schema_name: str, table_name: str) -> LFTags:
+        """
+        Method to call the API and get the table and column tags
+        """
+        try:
+            response = self.lake_formation_client.get_resource_lf_tags(
+                Resource={
+                    "Table": {
+                        "DatabaseName": schema_name,
+                        "Name": table_name,
+                    },
+                    "TableWithColumns": {
+                        "DatabaseName": schema_name,
+                        "Name": table_name,
+                    },
+                }
+            )
+            return LFTags(**response)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Unable to get lf tags for table resource [{table_name}] due to: {exc}"
+            )
+        return LFTags()

metadata/ingestion/source/database/athena/connection.py

@@ -75,6 +75,13 @@ def get_connection(connection: AthenaConnection) -> Engine:
     )
 
 
+def get_lake_formation_client(connection: AthenaConnection):
+    """
+    Get the lake formation client
+    """
+    return AWSClient(connection.awsConfig).get_lake_formation_client()
+
+
 def test_connection(
     metadata: OpenMetadata,
     engine: Engine,