openmetadata-ingestion 1.8.10.0__py3-none-any.whl → 1.9.1.0__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -436,21 +436,21 @@ class PowerbiSource(DashboardServiceSource):
         measures = []
         for measure in table.measures or []:
             try:
-                measure_type = (
-                    DataType.MEASURE_HIDDEN
-                    if measure.isHidden
-                    else DataType.MEASURE_VISIBLE
-                )
+                measure_type = DataType.MEASURE_VISIBLE
+                if measure.isHidden:
+                    measure_type = DataType.MEASURE_HIDDEN
+                expression_text = f"Expression : {measure.expression}"
                 description_text = (
-                    f"{measure.description}\n\nExpression : {measure.expression}"
+                    f"Description : {measure.description}"
                     if measure.description
-                    else f"Expression : {measure.expression}"
+                    else ""
                 )
+                description_field_text = f"{expression_text}\n\n{description_text}"
                 parsed_measure = PowerBiMeasureModel(
                     dataType=measure_type,
                     dataTypeDisplay=measure_type,
                     name=measure.name,
-                    description=description_text,
+                    description=description_field_text,
                 )
                 measures.append(Column(**parsed_measure.model_dump()))
             except Exception as err:
@@ -866,7 +866,7 @@ class PowerbiSource(DashboardServiceSource):
 
         try:
             table_info = self._parse_table_info_from_source_exp(table, datamodel_entity)
-            table_name = table.name or table_info.get("table")
+            table_name = table_info.get("table") or table.name
             schema_name = table_info.get("schema")
             database_name = table_info.get("database")
             if (

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -135,7 +135,7 @@ class PowerBiMeasures(BaseModel):
     name: str
     expression: str
     description: Optional[str] = None
-    isHidden: bool
+    isHidden: Optional[bool] = False
 
 
 class PowerBITableSource(BaseModel):

metadata/ingestion/source/dashboard/superset/connection.py

@@ -27,7 +27,7 @@ from metadata.generated.schema.entity.services.connections.database.mysqlConnect
     MysqlConnection as MysqlConnectionConfig,
 )
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
-    PostgresConnection,
+    PostgresConnection as PostgresConnectionConfig,
 )
 from metadata.generated.schema.entity.services.connections.testConnectionResult import (
     TestConnectionResult,
@@ -47,9 +47,7 @@ from metadata.ingestion.source.dashboard.superset.queries import (
     FETCH_DASHBOARDS_TEST,
 )
 from metadata.ingestion.source.database.mysql.connection import MySQLConnection
-from metadata.ingestion.source.database.postgres.connection import (
-    get_connection as pg_get_connection,
-)
+from metadata.ingestion.source.database.postgres.connection import PostgresConnection
 from metadata.utils.constants import THREE_MIN
 
 
@@ -61,10 +59,10 @@ def get_connection(
     """
     if isinstance(connection.connection, SupersetApiConnection):
         return SupersetAPIClient(connection)
-    if isinstance(connection.connection, PostgresConnection):
-        return pg_get_connection(connection=connection.connection)
+    if isinstance(connection.connection, PostgresConnectionConfig):
+        return PostgresConnection(connection.connection).client
     if isinstance(connection.connection, MysqlConnectionConfig):
-        return MySQLConnection(connection.connection).get_client()
+        return MySQLConnection(connection.connection).client
     return None
 
 

metadata/ingestion/source/dashboard/tableau/client.py

@@ -113,7 +113,7 @@ class TableauClient:
             owner = self.tableau_server.users.get_by_id(owner_id) if owner_id else None
             if owner and owner.email:
                 owner_obj = TableauOwner(
-                    id=owner.id, name=owner.name, email=owner.email
+                    id=str(owner.id), name=owner.name, email=owner.email
                 )
                 self.owner_cache[owner_id] = owner_obj
                 return owner_obj
@@ -133,7 +133,7 @@ class TableauClient:
             try:
                 charts.append(
                     TableauChart(
-                        id=view.id,
+                        id=str(view.id),
                         name=view.name,
                         tags=view.tags,
                         owner=self.get_tableau_owner(view.owner_id),
@@ -215,10 +215,10 @@ class TableauClient:
                     workbook.views
                 )
                 workbook = TableauDashboard(
-                    id=workbook.id,
+                    id=str(workbook.id),
                     name=workbook.name,
                     project=TableauBaseModel(
-                        id=workbook.project_id, name=workbook.project_name
+                        id=str(workbook.project_id), name=workbook.project_name
                     ),
                     owner=self.get_tableau_owner(workbook.owner_id),
                     description=workbook.description,

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -257,6 +257,7 @@ class TableauSource(DashboardServiceSource):
                 ),
                 sql=self._get_datamodel_sql_query(data_model=data_model),
                 owners=self.get_owner_ref(dashboard_details=dashboard_details),
+                project=self.get_project_name(dashboard_details=dashboard_details),
             )
             yield Either(right=data_model_request)
             self.register_record_datamodel(datamodel_request=data_model_request)

metadata/ingestion/source/database/datalake/connection.py

@@ -12,8 +12,6 @@
 """
 Source connection handler
 """
-from dataclasses import dataclass
-from functools import singledispatch
 from typing import Optional
 
 from metadata.generated.schema.entity.automations.workflow import (
@@ -29,88 +27,68 @@ from metadata.generated.schema.entity.services.connections.database.datalake.s3C
     S3Config,
 )
 from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
-    DatalakeConnection,
+    DatalakeConnection as DatalakeConnectionConfig,
 )
 from metadata.generated.schema.entity.services.connections.testConnectionResult import (
     TestConnectionResult,
 )
+from metadata.ingestion.connections.connection import BaseConnection
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.datalake.clients.azure_blob import (
     DatalakeAzureBlobClient,
 )
+from metadata.ingestion.source.database.datalake.clients.base import DatalakeBaseClient
 from metadata.ingestion.source.database.datalake.clients.gcs import DatalakeGcsClient
 from metadata.ingestion.source.database.datalake.clients.s3 import DatalakeS3Client
 from metadata.utils.constants import THREE_MIN
 
 
-# Only import specific datalake dependencies if necessary
-# pylint: disable=import-outside-toplevel
-@dataclass
-class DatalakeClient:
-    def __init__(self, client, config) -> None:
-        self.client = client
-        self.config = config
-
-
-@singledispatch
-def get_datalake_client(config):
-    """
-    Method to retrieve datalake client from the config
-    """
-    if config:
-        msg = f"Config not implemented for type {type(config)}: {config}"
-        raise NotImplementedError(msg)
-
-
-@get_datalake_client.register
-def _(config: S3Config):
-    return DatalakeS3Client.from_config(config)
-
-
-@get_datalake_client.register
-def _(config: GCSConfig):
-    return DatalakeGcsClient.from_config(config)
-
-
-@get_datalake_client.register
-def _(config: AzureConfig):
-    return DatalakeAzureBlobClient.from_config(config)
-
-
-def get_connection(connection: DatalakeConnection) -> DatalakeClient:
-    """
-    Create connection.
-
-    Returns an AWS, Azure or GCS Clients.
-    """
-    return DatalakeClient(
-        client=get_datalake_client(connection.configSource),
-        config=connection,
-    )
-
-
-def test_connection(
-    metadata: OpenMetadata,
-    connection: DatalakeClient,
-    service_connection: DatalakeConnection,
-    automation_workflow: Optional[AutomationWorkflow] = None,
-    timeout_seconds: Optional[int] = THREE_MIN,
-) -> TestConnectionResult:
-    """
-    Test connection. This can be executed either as part
-    of a metadata workflow or during an Automation Workflow
-    """
-    test_fn = {
-        "ListBuckets": connection.client.get_test_list_buckets_fn(
-            connection.config.bucketName
-        ),
-    }
-
-    return test_connection_steps(
-        metadata=metadata,
-        test_fn=test_fn,
-        service_type=service_connection.type.value,
-        automation_workflow=automation_workflow,
-        timeout_seconds=timeout_seconds,
-    )
+class DatalakeConnection(BaseConnection[DatalakeConnectionConfig, DatalakeBaseClient]):
+    def _get_client(self) -> DatalakeBaseClient:
+        """
+        Return the appropriate Datalake client based on configSource.
+        """
+        connection = self.service_connection
+
+        if isinstance(connection.configSource, S3Config):
+            return DatalakeS3Client.from_config(connection.configSource)
+        elif isinstance(connection.configSource, GCSConfig):
+            return DatalakeGcsClient.from_config(connection.configSource)
+        elif isinstance(connection.configSource, AzureConfig):
+            return DatalakeAzureBlobClient.from_config(connection.configSource)
+        else:
+            msg = f"Config not implemented for type {type(connection.configSource)}: {connection.configSource}"
+            raise NotImplementedError(msg)
+
+    def get_connection_dict(self) -> dict:
+        """
+        Return the connection dictionary for this service.
+        """
+        raise NotImplementedError("get_connection_dict is not implemented for Datalake")
+
+    def test_connection(
+        self,
+        metadata: OpenMetadata,
+        automation_workflow: Optional[AutomationWorkflow] = None,
+        timeout_seconds: Optional[int] = THREE_MIN,
+    ) -> TestConnectionResult:
+        """
+        Test connection. This can be executed either as part
+        of a metadata workflow or during an Automation Workflow
+        """
+        test_fn = {
+            "ListBuckets": self.client.get_test_list_buckets_fn(
+                self.service_connection.bucketName
+            ),
+        }
+
+        return test_connection_steps(
+            metadata=metadata,
+            test_fn=test_fn,
+            service_type=self.service_connection.type.value
+            if self.service_connection.type
+            else "Datalake",
+            automation_workflow=automation_workflow,
+            timeout_seconds=timeout_seconds,
+        )

metadata/ingestion/source/database/datalake/metadata.py

@@ -88,15 +88,14 @@ class DatalakeSource(DatabaseServiceSource):
         )
         self.metadata = metadata
         self.service_connection = self.config.serviceConnection.root.config
-        self.connection = get_connection(self.service_connection)
-        self.client = self.connection.client
+        self.client = get_connection(self.service_connection)
         self.table_constraints = None
         self.database_source_state = set()
         self.config_source = self.service_connection.configSource
-        self.connection_obj = self.connection
+        self.connection_obj = self.client
         self.test_connection()
         self.reader = get_reader(
-            config_source=self.config_source, client=self.client._client
+            config_source=self.config_source, client=self.client.client
         )
 
     @classmethod

metadata/ingestion/source/database/datalake/service_spec.py

@@ -1,6 +1,7 @@
 from metadata.data_quality.interface.pandas.pandas_test_suite_interface import (
     PandasTestSuiteInterface,
 )
+from metadata.ingestion.source.database.datalake.connection import DatalakeConnection
 from metadata.ingestion.source.database.datalake.metadata import DatalakeSource
 from metadata.profiler.interface.pandas.profiler_interface import (
     PandasProfilerInterface,
@@ -13,4 +14,5 @@ ServiceSpec = DefaultDatabaseSpec(
     profiler_class=PandasProfilerInterface,
     test_suite_class=PandasTestSuiteInterface,
     sampler_class=DatalakeSampler,
+    connection_class=DatalakeConnection,
 )

metadata/ingestion/source/database/dbt/constants.py

@@ -14,11 +14,17 @@ Constants required for dbt
 
 from enum import Enum
 
+from metadata.generated.schema.entity.data.apiEndpoint import APIEndpoint
+from metadata.generated.schema.entity.data.dashboard import Dashboard
+from metadata.generated.schema.entity.data.mlmodel import MlModel
+
 DBT_RUN_RESULT_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
 
 # Based on https://schemas.getdbt.com/dbt/manifest/v7/index.html
 REQUIRED_MANIFEST_KEYS = ["name", "schema", "resource_type"]
 
+REQUIRED_EXPOSURE_KEYS = ["name", "meta", "type", "resource_type", "sources"]
+
 # Based on https://schemas.getdbt.com/dbt/catalog/v1.json
 REQUIRED_CATALOG_KEYS = ["name", "type", "index"]
 
@@ -84,7 +90,6 @@ REQUIRED_NODE_KEYS = {
     "language",
 }
 
-
 NONE_KEYWORDS_LIST = ["none", "null"]
 
 DBT_CATALOG_FILE_NAME = "catalog.json"
@@ -148,6 +153,8 @@ class DbtCommonEnum(Enum):
     OWNER = "owner"
     NODES = "nodes"
     SOURCES = "sources"
+    EXPOSURE = "exposure"
+    EXPOSURES = "exposures"
     SOURCES_FILE = "sources_file"
     SOURCE = "source"
     RESOURCETYPE = "resource_type"
@@ -156,3 +163,12 @@ class DbtCommonEnum(Enum):
     RESULTS = "results"
     TEST_SUITE_NAME = "test_suite_name"
     DBT_TEST_SUITE = "DBT_TEST_SUITE"
+
+
+# DBT Supports more types of exposures but only these map nicely
+# https://docs.getdbt.com/docs/build/exposures#available-properties
+ExposureTypeMap = {
+    "dashboard": {"entity_type": Dashboard, "entity_type_name": "dashboard"},
+    "ml": {"entity_type": MlModel, "entity_type_name": "mlmodel"},
+    "application": {"entity_type": APIEndpoint, "entity_type_name": "apiEndpoint"},
+}

metadata/ingestion/source/database/dbt/dbt_service.py

@@ -81,6 +81,7 @@ class DbtServiceTopology(ServiceTopology):
             "process_dbt_data_model",
             "process_dbt_entities",
             "process_dbt_tests",
+            "process_dbt_exposures",
         ],
     )
     process_dbt_data_model: Annotated[
@@ -151,6 +152,20 @@ class DbtServiceTopology(ServiceTopology):
         ],
     )
 
+    process_dbt_exposures: Annotated[
+        TopologyNode, Field(description="Process dbt exposures")
+    ] = TopologyNode(
+        producer="get_dbt_exposures",
+        stages=[
+            NodeStage(
+                type_=AddLineageRequest,
+                processor="create_dbt_exposures_lineage",
+                consumer=["yield_data_models"],
+                nullable=True,
+            ),
+        ],
+    )
+
 
 class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
     """
@@ -175,7 +190,7 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
         # This step is necessary as the manifest file may not always adhere to the schema definition
         # and the presence of other nodes can hinder the ingestion process from progressing any further.
         # Therefore, we are only retaining the essential data for further processing.
-        required_manifest_keys = {"nodes", "sources", "metadata"}
+        required_manifest_keys = {"nodes", "sources", "metadata", "exposures"}
         manifest_dict.update(
             {
                 key: {}
@@ -284,6 +299,12 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
         Method to process DBT lineage from upstream nodes
         """
 
+    @abstractmethod
+    def create_dbt_exposures_lineage(self, exposure_spec: dict) -> AddLineageRequest:
+        """
+        Method to process DBT lineage from exposures
+        """
+
     @abstractmethod
     def create_dbt_query_lineage(
         self, data_model_link: DataModelLink
@@ -311,6 +332,14 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
         for _, dbt_test in self.context.get().dbt_tests.items():
             yield dbt_test
 
+    def get_dbt_exposures(self) -> dict:
+        """
+        Prepare the DBT exposures
+        """
+
+        for _, exposure in self.context.get().exposures.items():
+            yield exposure
+
     @abstractmethod
     def create_dbt_tests_definition(
         self, dbt_test: dict

metadata/ingestion/source/database/dbt/metadata.py

@@ -70,10 +70,12 @@ from metadata.ingestion.source.database.database_service import DataModelLink
 from metadata.ingestion.source.database.dbt.constants import (
     DBT_RUN_RESULT_DATE_FORMAT,
     REQUIRED_CATALOG_KEYS,
+    REQUIRED_EXPOSURE_KEYS,
     REQUIRED_MANIFEST_KEYS,
     DbtCommonEnum,
     DbtTestFailureEnum,
     DbtTestSuccessEnum,
+    ExposureTypeMap,
     SkipResourceTypeEnum,
 )
 from metadata.ingestion.source.database.dbt.dbt_service import (
@@ -236,6 +238,7 @@ class DbtSource(DbtServiceSource):
             manifest_entities = {
                 **dbt_files.dbt_manifest[DbtCommonEnum.NODES.value],
                 **dbt_files.dbt_manifest[DbtCommonEnum.SOURCES.value],
+                **dbt_files.dbt_manifest.get(DbtCommonEnum.EXPOSURES.value, {}),
             }
             catalog_entities = None
             if dbt_files.dbt_catalog:
@@ -249,6 +252,23 @@ class DbtSource(DbtServiceSource):
                 ]:
                     continue
 
+                if (
+                    manifest_node[DbtCommonEnum.RESOURCETYPE.value]
+                    == DbtCommonEnum.EXPOSURE.value
+                ):
+                    if all(
+                        required_key in manifest_node
+                        for required_key in REQUIRED_EXPOSURE_KEYS
+                    ):
+                        logger.debug(f"Successfully Validated DBT Node: {key}")
+                    else:
+                        logger.warning(
+                            f"Error validating DBT Node: {key}\n"
+                            f"Please check if following keys exist for the node: {REQUIRED_EXPOSURE_KEYS}"
+                        )
+
+                    continue
+
                 # Validate if all the required keys are present in the manifest nodes
                 if all(
                     required_key in manifest_node
@@ -373,6 +393,19 @@ class DbtSource(DbtServiceSource):
             None,
         )
 
+    def add_dbt_exposure(self, key: str, manifest_node, manifest_entities):
+        exposure_entity = self.parse_exposure_node(manifest_node)
+
+        if exposure_entity:
+            self.context.get().exposures[key] = {
+                DbtCommonEnum.EXPOSURE: exposure_entity,
+                DbtCommonEnum.MANIFEST_NODE: manifest_node,
+            }
+
+            self.context.get().exposures[key][
+                DbtCommonEnum.UPSTREAM
+            ] = self.parse_upstream_nodes(manifest_entities, manifest_node)
+
     def add_dbt_sources(
         self, key: str, manifest_node, manifest_entities, dbt_objects: DbtObjects
     ) -> None:
@@ -466,6 +499,7 @@ class DbtSource(DbtServiceSource):
             manifest_entities = {
                 **dbt_objects.dbt_manifest.sources,
                 **dbt_objects.dbt_manifest.nodes,
+                **dbt_objects.dbt_manifest.exposures,
             }
             catalog_entities = None
             if dbt_objects.dbt_catalog:
@@ -474,6 +508,7 @@ class DbtSource(DbtServiceSource):
                     **dbt_objects.dbt_catalog.nodes,
                 }
             self.context.get().data_model_links = []
+            self.context.get().exposures = {}
             self.context.get().dbt_tests = {}
             self.context.get().run_results_generate_time = None
             # Since we'll be processing multiple run_results for a single project
@@ -517,6 +552,10 @@ class DbtSource(DbtServiceSource):
                             dbt_objects=dbt_objects,
                         )
 
+                    if resource_type == DbtCommonEnum.EXPOSURE.value:
+                        self.add_dbt_exposure(key, manifest_node, manifest_entities)
+                        continue
+
                     # Skip the ephemeral nodes since it is not materialized
                     if check_ephemeral_node(manifest_node):
                         logger.debug(f"Skipping ephemeral DBT node: {key}.")
@@ -762,6 +801,57 @@ class DbtSource(DbtServiceSource):
 
         return columns
 
+    def parse_exposure_node(self, exposure_spec) -> Optional[Any]:
+        """
+        Parses the exposure node verifying if it's type is supported and if provided label matches FQN of
+        Open Metadata entity. Returns entity object if both conditions are met.
+
+        The implementation assumes that value of meta.open_metadata_fqn provided in DBT exposures object matches
+        to FQN of OpenMetadata entity.
+
+        ```yaml
+        exposures:
+          - name: orders_dashboard
+            label: orders
+            meta:
+              open_metadata_fqn: sample_looker.orders  # OpenMetadata entity FullyQualifiedName
+            type: dashboard
+            maturity: high
+            url: http://localhost:808/looker/dashboard/8/
+            description: >
+              Exemplary OM Looker Dashboard.
+
+            depends_on:
+              - ref('fact_sales')
+        ```
+        """
+        exposure_type = exposure_spec.type.value
+        entity_type = ExposureTypeMap.get(exposure_type, {}).get("entity_type")
+
+        if not entity_type:
+            logger.warning(f"Exposure type [{exposure_spec.type.value}] not supported.")
+
+            return None
+
+        try:
+            entity_fqn = exposure_spec.meta["open_metadata_fqn"]
+        except KeyError:
+            logger.warning(
+                f"meta.open_metadata_fqn not found in [{exposure_spec.name}] exposure spec."
+            )
+            return None
+
+        entity = self.metadata.get_by_name(fqn=entity_fqn, entity=entity_type)
+
+        if not entity:
+            logger.warning(
+                f"Entity [{entity_fqn}] of [{exposure_type}] type not found in Open Metadata."
+            )
+
+            return None
+
+        return entity
+
     def create_dbt_lineage(
         self, data_model_link: DataModelLink
     ) -> Iterable[Either[AddLineageRequest]]:
@@ -834,7 +924,7 @@ class DbtSource(DbtServiceSource):
                 lineages = get_lineage_by_query(
                     self.metadata,
                     query=query,
-                    service_name=source_elements[0],
+                    service_names=source_elements[0],
                     database_name=source_elements[1],
                     schema_name=source_elements[2],
                     dialect=dialect,
@@ -855,6 +945,51 @@ class DbtSource(DbtServiceSource):
                     )
                 )
 
+    def create_dbt_exposures_lineage(
+        self, exposure_spec: dict
+    ) -> Iterable[Either[AddLineageRequest]]:
+        to_entity = exposure_spec[DbtCommonEnum.EXPOSURE]
+        upstream = exposure_spec[DbtCommonEnum.UPSTREAM]
+        manifest_node = exposure_spec[DbtCommonEnum.MANIFEST_NODE]
+
+        for upstream_node in upstream:
+            try:
+                from_es_result = self.metadata.es_search_from_fqn(
+                    entity_type=Table,
+                    fqn_search_string=upstream_node,
+                )
+                from_entity: Optional[
+                    Union[Table, List[Table]]
+                ] = get_entity_from_es_result(
+                    entity_list=from_es_result, fetch_multiple_entities=False
+                )
+                if from_entity and to_entity:
+                    yield Either(
+                        right=AddLineageRequest(
+                            edge=EntitiesEdge(
+                                fromEntity=EntityReference(
+                                    id=Uuid(from_entity.id.root),
+                                    type="table",
+                                ),
+                                toEntity=EntityReference(
+                                    id=Uuid(to_entity.id.root),
+                                    type=ExposureTypeMap[manifest_node.type.value][
+                                        "entity_type_name"
+                                    ],
+                                ),
+                                lineageDetails=LineageDetails(
+                                    source=LineageSource.DbtLineage
+                                ),
+                            )
+                        )
+                    )
+
+            except Exception as exc:  # pylint: disable=broad-except
+                logger.debug(traceback.format_exc())
+                logger.warning(
+                    f"Failed to parse the node {upstream_node} to capture lineage: {exc}"
+                )
+
     def process_dbt_meta(self, manifest_meta):
         """
         Method to process DBT meta for Tags and GlossaryTerms

metadata/ingestion/source/database/deltalake/clients/pyspark.py

@@ -79,7 +79,7 @@ class DeltalakePySparkClient(DeltalakeBaseClient):
                 "org.apache.spark.sql.delta.catalog.DeltaCatalog",
             )
             # Download delta-core jars when creating the SparkSession
-            .config("spark.jars.packages", "io.delta:delta-core_2.12:2.0.0")
+            .config("spark.jars.packages", "io.delta:delta-spark_2.12:3.2.0")
         )
 
         # Check that the attribute exists and is properly informed