openmetadata-ingestion 1.4.7.2__py3-none-any.whl → 1.5.0.0__py3-none-any.whl

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -41,7 +41,6 @@ from metadata.generated.schema.metadataIngestion.dbtconfig.dbtS3Config import (
 )
 from metadata.ingestion.source.database.dbt.constants import (
     DBT_CATALOG_FILE_NAME,
-    DBT_FILE_NAMES_LIST,
     DBT_MANIFEST_FILE_NAME,
     DBT_RUN_RESULTS_FILE_NAME,
 )
@@ -129,7 +128,7 @@ def _(config: DbtHttpConfig):
         yield DbtFiles(
             dbt_catalog=dbt_catalog.json() if dbt_catalog else None,
             dbt_manifest=dbt_manifest.json(),
-            dbt_run_results=dbt_run_results.json() if dbt_run_results else None,
+            dbt_run_results=[dbt_run_results.json()] if dbt_run_results else None,
         )
     except DBTConfigException as exc:
         raise exc
@@ -152,7 +151,7 @@ def _(config: DbtCloudConfig):  # pylint: disable=too-many-locals
         expiry = 0
         auth_token = config.dbtCloudAuthToken.get_secret_value(), expiry
         client_config = ClientConfig(
-            base_url=config.dbtCloudUrl,
+            base_url=str(config.dbtCloudUrl),
             api_version="api/v2",
             auth_token=lambda: auth_token,
             auth_header="Authorization",
@@ -183,7 +182,8 @@ def _(config: DbtCloudConfig):  # pylint: disable=too-many-locals
             last_run = runs_data[0]
             run_id = last_run["id"]
             logger.info(
-                f"Retrieved last successful run [{run_id}] finished {last_run['finished_at_humanized']} (duration: {last_run['duration_humanized']})"
+                f"Retrieved last successful run [{str(run_id)}]: "
+                f"Finished {str(last_run['finished_at_humanized'])} (duration: {str(last_run['duration_humanized'])})"
             )
             try:
                 logger.debug("Requesting [dbt_catalog]")
@@ -215,7 +215,7 @@ def _(config: DbtCloudConfig):  # pylint: disable=too-many-locals
         yield DbtFiles(
             dbt_catalog=dbt_catalog,
             dbt_manifest=dbt_manifest,
-            dbt_run_results=dbt_run_results,
+            dbt_run_results=[dbt_run_results] if dbt_run_results else None,
         )
     except DBTConfigException as exc:
         raise exc
@@ -232,13 +232,12 @@ def get_blobs_grouped_by_dir(blobs: List[str]) -> Dict[str, List[str]]:
     for blob in blobs:
         subdirectory = blob.rsplit("/", 1)[0] if "/" in blob else ""
         blob_file_name = blob.rsplit("/", 1)[1] if "/" in blob else blob
-        if next(
-            (
-                file_name
-                for file_name in DBT_FILE_NAMES_LIST
-                if file_name.lower() == blob_file_name.lower()
-            ),
-            None,
+        # We'll be processing multiple run_result files from a single dir
+        # Grouping them together to process them in a single go
+        if (
+            DBT_MANIFEST_FILE_NAME == blob_file_name.lower()
+            or DBT_CATALOG_FILE_NAME == blob_file_name.lower()
+            or DBT_RUN_RESULTS_FILE_NAME in blob_file_name.lower()
         ):
             blob_grouped_by_directory[subdirectory].append(blob)
     return blob_grouped_by_directory
@@ -256,7 +255,7 @@ def download_dbt_files(
     ) in blob_grouped_by_directory.items():
         dbt_catalog = None
         dbt_manifest = None
-        dbt_run_results = None
+        dbt_run_results = []
         kwargs = {}
         if bucket_name:
             kwargs = {"bucket_name": bucket_name}
@@ -264,10 +263,11 @@ def download_dbt_files(
             for blob in blobs:
                 if blob:
                     reader = get_reader(config_source=config, client=client)
-                    if DBT_MANIFEST_FILE_NAME in blob:
+                    blob_file_name = blob.rsplit("/", 1)[1] if "/" in blob else blob
+                    if DBT_MANIFEST_FILE_NAME == blob_file_name.lower():
                         logger.debug(f"{DBT_MANIFEST_FILE_NAME} found in {key}")
                         dbt_manifest = reader.read(path=blob, **kwargs)
-                    if DBT_CATALOG_FILE_NAME in blob:
+                    if DBT_CATALOG_FILE_NAME == blob_file_name.lower():
                         try:
                             logger.debug(f"{DBT_CATALOG_FILE_NAME} found in {key}")
                             dbt_catalog = reader.read(path=blob, **kwargs)
@@ -275,10 +275,12 @@ def download_dbt_files(
                             logger.warning(
                                 f"{DBT_CATALOG_FILE_NAME} not found in {key}: {exc}"
                             )
-                    if DBT_RUN_RESULTS_FILE_NAME in blob:
+                    if DBT_RUN_RESULTS_FILE_NAME in blob_file_name.lower():
                         try:
-                            logger.debug(f"{DBT_RUN_RESULTS_FILE_NAME} found in {key}")
-                            dbt_run_results = reader.read(path=blob, **kwargs)
+                            logger.debug(f"{blob_file_name} found in {key}")
+                            dbt_run_result = reader.read(path=blob, **kwargs)
+                            if dbt_run_result:
+                                dbt_run_results.append(json.loads(dbt_run_result))
                         except Exception as exc:
                             logger.warning(
                                 f"{DBT_RUN_RESULTS_FILE_NAME} not found in {key}: {exc}"
@@ -288,9 +290,7 @@ def download_dbt_files(
             yield DbtFiles(
                 dbt_catalog=json.loads(dbt_catalog) if dbt_catalog else None,
                 dbt_manifest=json.loads(dbt_manifest),
-                dbt_run_results=json.loads(dbt_run_results)
-                if dbt_run_results
-                else None,
+                dbt_run_results=dbt_run_results if dbt_run_results else None,
             )
         except DBTConfigException as exc:
             logger.warning(exc)

metadata/ingestion/source/database/dbt/dbt_service.py

@@ -16,6 +16,8 @@ from abc import ABC, abstractmethod
 from typing import Iterable
 
 from dbt_artifacts_parser.parser import parse_catalog, parse_manifest, parse_run_results
+from pydantic import Field
+from typing_extensions import Annotated
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
@@ -54,7 +56,9 @@ class DbtServiceTopology(ServiceTopology):
     dbt files -> dbt tags -> data models -> descriptions -> lineage -> tests.
     """
 
-    root = TopologyNode(
+    root: Annotated[
+        TopologyNode, Field(description="Root node for the topology")
+    ] = TopologyNode(
         producer="get_dbt_files",
         stages=[
             NodeStage(
@@ -69,7 +73,9 @@ class DbtServiceTopology(ServiceTopology):
             "process_dbt_tests",
         ],
     )
-    process_dbt_data_model = TopologyNode(
+    process_dbt_data_model: Annotated[
+        TopologyNode, Field(description="Process dbt data models")
+    ] = TopologyNode(
         producer="get_dbt_objects",
         stages=[
             NodeStage(
@@ -87,7 +93,9 @@ class DbtServiceTopology(ServiceTopology):
             ),
         ],
     )
-    process_dbt_entities = TopologyNode(
+    process_dbt_entities: Annotated[
+        TopologyNode, Field(description="Process dbt entities")
+    ] = TopologyNode(
         producer="get_data_model",
         stages=[
             NodeStage(
@@ -106,7 +114,9 @@ class DbtServiceTopology(ServiceTopology):
             ),
         ],
     )
-    process_dbt_tests = TopologyNode(
+    process_dbt_tests: Annotated[
+        TopologyNode, Field(description="Process dbt tests")
+    ] = TopologyNode(
         producer="get_dbt_tests",
         stages=[
             NodeStage(
@@ -174,9 +184,10 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
             if self.context.get().dbt_file.dbt_catalog
             else None,
             dbt_manifest=parse_manifest(self.context.get().dbt_file.dbt_manifest),
-            dbt_run_results=parse_run_results(
-                self.context.get().dbt_file.dbt_run_results
-            )
+            dbt_run_results=[
+                parse_run_results(run_result_file)
+                for run_result_file in self.context.get().dbt_file.dbt_run_results
+            ]
             if self.context.get().dbt_file.dbt_run_results
             else None,
         )
@@ -206,8 +217,7 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
         """
         Prepare the data models
         """
-        for data_model_link in self.context.get().data_model_links:
-            yield data_model_link
+        yield from self.context.get().data_model_links
 
     @abstractmethod
     def create_dbt_lineage(self, data_model_link: DataModelLink) -> AddLineageRequest:

metadata/ingestion/source/database/dbt/metadata.py

@@ -1,3 +1,4 @@
+#  pylint: disable=too-many-lines
 #  Copyright 2021 Collate
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,7 +14,7 @@ DBT source methods.
 """
 import traceback
 from datetime import datetime
-from typing import Iterable, List, Optional, Union
+from typing import Any, Iterable, List, Optional, Union
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
@@ -45,10 +46,16 @@ from metadata.generated.schema.tests.testDefinition import (
     TestDefinition,
     TestPlatform,
 )
-from metadata.generated.schema.type.basic import FullyQualifiedEntityName
+from metadata.generated.schema.type.basic import (
+    FullyQualifiedEntityName,
+    SqlQuery,
+    Timestamp,
+    Uuid,
+)
 from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query
@@ -86,6 +93,7 @@ from metadata.ingestion.source.database.dbt.dbt_utils import (
 from metadata.ingestion.source.database.dbt.models import DbtMeta
 from metadata.utils import fqn
 from metadata.utils.elasticsearch import get_entity_from_es_result
+from metadata.utils.entity_link import get_table_fqn
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_labels
 from metadata.utils.time_utils import convert_timestamp_to_milliseconds
@@ -119,7 +127,7 @@ class DbtSource(DbtServiceSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
         return cls(config, metadata)
 
     def test_connection(self) -> None:
@@ -131,11 +139,10 @@ class DbtSource(DbtServiceSource):
         """
         By default for DBT nothing is required to be prepared
         """
-        pass
 
     def get_dbt_owner(
-        self, manifest_node: dict, catalog_node: Optional[dict]
-    ) -> Optional[EntityReference]:
+        self, manifest_node: Any, catalog_node: Optional[Any]
+    ) -> Optional[EntityReferenceList]:
         """
         Returns dbt owner
         """
@@ -147,7 +154,16 @@ class DbtSource(DbtServiceSource):
             if manifest_node:
                 dbt_owner = manifest_node.meta.get(DbtCommonEnum.OWNER.value)
             if dbt_owner:
-                owner = self.metadata.get_reference_by_name(name=dbt_owner)
+                owner = self.metadata.get_reference_by_name(
+                    name=dbt_owner, is_owner=True
+                )
+
+                if owner:
+                    return owner
+
+                # If owner is not found, try to find the owner in OMD using email
+                owner = self.metadata.get_reference_by_email(name=dbt_owner)
+
                 if not owner:
                     logger.warning(
                         "Unable to ingest owner from DBT since no user or"
@@ -184,6 +200,7 @@ class DbtSource(DbtServiceSource):
                 **dbt_files.dbt_manifest[DbtCommonEnum.NODES.value],
                 **dbt_files.dbt_manifest[DbtCommonEnum.SOURCES.value],
             }
+            catalog_entities = None
             if dbt_files.dbt_catalog:
                 catalog_entities = {
                     **dbt_files.dbt_catalog[DbtCommonEnum.NODES.value],
@@ -208,7 +225,7 @@ class DbtSource(DbtServiceSource):
                     )
 
                 # Validate the catalog file if it is passed
-                if dbt_files.dbt_catalog:
+                if catalog_entities:
                     catalog_node = catalog_entities.get(key)
                     if catalog_node and "columns" in catalog_node:
                         self.check_columns(catalog_node=catalog_node)
@@ -303,7 +320,8 @@ class DbtSource(DbtServiceSource):
         self.context.get().dbt_tests[key][DbtCommonEnum.RESULTS.value] = next(
             (
                 item
-                for item in dbt_objects.dbt_run_results.results
+                for run_result in dbt_objects.dbt_run_results
+                for item in run_result.results
                 if item.unique_id == key
             ),
             None,
@@ -322,6 +340,7 @@ class DbtSource(DbtServiceSource):
                 **dbt_objects.dbt_manifest.sources,
                 **dbt_objects.dbt_manifest.nodes,
             }
+            catalog_entities = None
             if dbt_objects.dbt_catalog:
                 catalog_entities = {
                     **dbt_objects.dbt_catalog.sources,
@@ -330,12 +349,14 @@ class DbtSource(DbtServiceSource):
             self.context.get().data_model_links = []
             self.context.get().dbt_tests = {}
             self.context.get().run_results_generate_time = None
+            # Since we'll be processing multiple run_results for a single project
+            # we'll only consider the first run_results generated_at time
             if (
                 dbt_objects.dbt_run_results
-                and dbt_objects.dbt_run_results.metadata.generated_at
+                and dbt_objects.dbt_run_results[0].metadata.generated_at
             ):
                 self.context.get().run_results_generate_time = (
-                    dbt_objects.dbt_run_results.metadata.generated_at
+                    dbt_objects.dbt_run_results[0].metadata.generated_at
                 )
             for key, manifest_node in manifest_entities.items():
                 try:
@@ -383,7 +404,7 @@ class DbtSource(DbtServiceSource):
                     logger.debug(f"Processing DBT node: {model_name}")
 
                     catalog_node = None
-                    if dbt_objects.dbt_catalog:
+                    if catalog_entities:
                         catalog_node = catalog_entities.get(key)
 
                     dbt_table_tags_list = []
@@ -407,7 +428,6 @@ class DbtSource(DbtServiceSource):
                     dbt_raw_query = get_dbt_raw_query(manifest_node)
 
                     # Get the table entity from ES
-                    # TODO: Change to get_by_name once the postgres case sensitive calls is fixed
                     table_fqn = fqn.build(
                         self.metadata,
                         entity_type=Table,
@@ -438,15 +458,19 @@ class DbtSource(DbtServiceSource):
                                 if manifest_node.description
                                 else None,
                                 path=get_data_model_path(manifest_node=manifest_node),
-                                rawSql=dbt_raw_query if dbt_raw_query else "",
-                                sql=dbt_compiled_query if dbt_compiled_query else "",
+                                rawSql=SqlQuery(dbt_raw_query)
+                                if dbt_raw_query
+                                else None,
+                                sql=SqlQuery(dbt_compiled_query)
+                                if dbt_compiled_query
+                                else None,
                                 columns=self.parse_data_model_columns(
                                     manifest_node, catalog_node
                                 ),
                                 upstream=self.parse_upstream_nodes(
                                     manifest_entities, manifest_node
                                 ),
-                                owner=self.get_dbt_owner(
+                                owners=self.get_dbt_owner(
                                     manifest_node=manifest_node,
                                     catalog_node=catalog_node,
                                 ),
@@ -512,7 +536,6 @@ class DbtSource(DbtServiceSource):
                         )
 
                         # check if the parent table exists in OM before adding it to the upstream list
-                        # TODO: Change to get_by_name once the postgres case sensitive calls is fixed
                         parent_table_entity: Optional[
                             Union[Table, List[Table]]
                         ] = get_entity_from_es_result(
@@ -532,7 +555,7 @@ class DbtSource(DbtServiceSource):
         return upstream_nodes
 
     def parse_data_model_columns(
-        self, manifest_node: dict, catalog_node: dict
+        self, manifest_node: Any, catalog_node: Any
     ) -> List[Column]:
         """
         Method to parse the DBT columns
@@ -542,7 +565,7 @@ class DbtSource(DbtServiceSource):
         for key, manifest_column in manifest_columns.items():
             try:
                 logger.debug(f"Processing DBT column: {key}")
-                # If catalog file is passed pass the column information from catalog file
+                # If catalog file is passed, pass the column information from catalog file
                 catalog_column = None
                 if catalog_node and catalog_node.columns:
                     catalog_column = catalog_node.columns.get(key)
@@ -614,9 +637,7 @@ class DbtSource(DbtServiceSource):
         Method to process DBT lineage from upstream nodes
         """
         to_entity: Table = data_model_link.table_entity
-        logger.debug(
-            f"Processing DBT lineage for: {to_entity.fullyQualifiedName.__root__}"
-        )
+        logger.debug(f"Processing DBT lineage for: {to_entity.fullyQualifiedName.root}")
 
         for upstream_node in data_model_link.datamodel.upstream:
             try:
@@ -634,11 +655,11 @@ class DbtSource(DbtServiceSource):
                         right=AddLineageRequest(
                             edge=EntitiesEdge(
                                 fromEntity=EntityReference(
-                                    id=from_entity.id.__root__,
+                                    id=Uuid(from_entity.id.root),
                                     type="table",
                                 ),
                                 toEntity=EntityReference(
-                                    id=to_entity.id.__root__,
+                                    id=Uuid(to_entity.id.root),
                                     type="table",
                                 ),
                                 lineageDetails=LineageDetails(
@@ -660,48 +681,48 @@ class DbtSource(DbtServiceSource):
         """
         Method to process DBT lineage from queries
         """
-        to_entity: Table = data_model_link.table_entity
-        logger.debug(
-            f"Processing DBT Query lineage for: {to_entity.fullyQualifiedName.__root__}"
-        )
-
-        try:
-            source_elements = fqn.split(to_entity.fullyQualifiedName.__root__)
-            # remove service name from fqn to make it parseable in format db.schema.table
-            query_fqn = fqn._build(  # pylint: disable=protected-access
-                *source_elements[-3:]
-            )
-            query = (
-                f"create table {query_fqn} as {data_model_link.datamodel.sql.__root__}"
-            )
-            connection_type = str(
-                self.config.serviceConnection.__root__.config.type.value
+        if data_model_link.datamodel.sql:
+            to_entity: Table = data_model_link.table_entity
+            logger.debug(
+                f"Processing DBT Query lineage for: {to_entity.fullyQualifiedName.root}"
             )
-            dialect = ConnectionTypeDialectMapper.dialect_of(connection_type)
-            lineages = get_lineage_by_query(
-                self.metadata,
-                query=query,
-                service_name=source_elements[0],
-                database_name=source_elements[1],
-                schema_name=source_elements[2],
-                dialect=dialect,
-                timeout_seconds=self.source_config.parsingTimeoutLimit,
-                lineage_source=LineageSource.DbtLineage,
-            )
-            for lineage_request in lineages or []:
-                yield lineage_request
 
-        except Exception as exc:  # pylint: disable=broad-except
-            yield Either(
-                left=StackTraceError(
-                    name=data_model_link.datamodel.sql.__root__,
-                    error=(
-                        f"Failed to parse the query {data_model_link.datamodel.sql.__root__}"
-                        f" to capture lineage: {exc}"
-                    ),
-                    stackTrace=traceback.format_exc(),
+            try:
+                source_elements = fqn.split(to_entity.fullyQualifiedName.root)
+                # remove service name from fqn to make it parseable in format db.schema.table
+                query_fqn = fqn._build(  # pylint: disable=protected-access
+                    *source_elements[-3:]
+                )
+                query = (
+                    f"create table {query_fqn} as {data_model_link.datamodel.sql.root}"
+                )
+                connection_type = str(
+                    self.config.serviceConnection.root.config.type.value
+                )
+                dialect = ConnectionTypeDialectMapper.dialect_of(connection_type)
+                lineages = get_lineage_by_query(
+                    self.metadata,
+                    query=query,
+                    service_name=source_elements[0],
+                    database_name=source_elements[1],
+                    schema_name=source_elements[2],
+                    dialect=dialect,
+                    timeout_seconds=self.source_config.parsingTimeoutLimit,
+                    lineage_source=LineageSource.DbtLineage,
+                )
+                yield from lineages or []
+
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name=data_model_link.datamodel.sql.root,
+                        error=(
+                            f"Failed to parse the query {data_model_link.datamodel.sql.root}"
+                            f" to capture lineage: {exc}"
+                        ),
+                        stackTrace=traceback.format_exc(),
+                    )
                 )
-            )
 
     def process_dbt_meta(self, manifest_meta):
         """
@@ -715,7 +736,7 @@ class DbtSource(DbtServiceSource):
                     get_tag_labels(
                         metadata=self.metadata,
                         tags=dbt_meta_info.openmetadata.glossary,
-                        include_tags=self.source_config.includeTags,
+                        include_tags=True,
                         tag_type=GlossaryTerm,
                     )
                     or []
@@ -728,7 +749,7 @@ class DbtSource(DbtServiceSource):
                         metadata=self.metadata,
                         tags=[tier_fqn.split(fqn.FQN_SEPARATOR)[-1]],
                         classification_name=tier_fqn.split(fqn.FQN_SEPARATOR)[0],
-                        include_tags=self.source_config.includeTags,
+                        include_tags=True,
                     )
                     or []
                 )
@@ -745,12 +766,12 @@ class DbtSource(DbtServiceSource):
         """
         table_entity: Table = data_model_link.table_entity
         logger.debug(
-            f"Processing DBT Descriptions for: {table_entity.fullyQualifiedName.__root__}"
+            f"Processing DBT Descriptions for: {table_entity.fullyQualifiedName.root}"
         )
         if table_entity:
             try:
                 service_name, database_name, schema_name, table_name = fqn.split(
-                    table_entity.fullyQualifiedName.__root__
+                    table_entity.fullyQualifiedName.root
                 )
                 data_model = data_model_link.datamodel
                 force_override = False
@@ -765,7 +786,7 @@ class DbtSource(DbtServiceSource):
                     self.metadata.patch_description(
                         entity=Table,
                         source=table_entity,
-                        description=data_model.description.__root__,
+                        description=data_model.description.root,
                         force=force_override,
                     )
 
@@ -782,7 +803,7 @@ class DbtSource(DbtServiceSource):
                                     database_name=database_name,
                                     schema_name=schema_name,
                                     table_name=table_name,
-                                    column_name=column.name.__root__,
+                                    column_name=column.name.root,
                                 ),
                                 description=column.description,
                             )
@@ -795,7 +816,7 @@ class DbtSource(DbtServiceSource):
             except Exception as exc:  # pylint: disable=broad-except
                 logger.debug(traceback.format_exc())
                 logger.warning(
-                    f"Failed to parse the node {table_entity.fullyQualifiedName.__root__} "
+                    f"Failed to parse the node {table_entity.fullyQualifiedName.root} "
                     f"to update dbt description: {exc}"
                 )
 
@@ -832,7 +853,7 @@ class DbtSource(DbtServiceSource):
                                 manifest_node
                             ),
                             displayName=None,
-                            owner=None,
+                            owners=None,
                         )
                     )
         except Exception as err:  # pylint: disable=broad-except
@@ -859,20 +880,42 @@ class DbtSource(DbtServiceSource):
                     test_suite = check_or_create_test_suite(
                         self.metadata, entity_link_str
                     )
-                    yield Either(
-                        right=CreateTestCaseRequest(
-                            name=manifest_node.name,
-                            description=manifest_node.description,
-                            testDefinition=FullyQualifiedEntityName(
-                                __root__=manifest_node.name
-                            ),
-                            entityLink=entity_link_str,
-                            testSuite=test_suite.fullyQualifiedName,
-                            parameterValues=create_test_case_parameter_values(dbt_test),
-                            displayName=None,
-                            owner=None,
-                        )
+                    table_fqn = get_table_fqn(entity_link_str)
+                    source_elements = table_fqn.split(fqn.FQN_SEPARATOR)
+                    test_case_fqn = fqn.build(
+                        self.metadata,
+                        entity_type=TestCase,
+                        service_name=source_elements[0],
+                        database_name=source_elements[1],
+                        schema_name=source_elements[2],
+                        table_name=source_elements[3],
+                        column_name=manifest_node.column_name
+                        if hasattr(manifest_node, "column_name")
+                        else None,
+                        test_case_name=manifest_node.name,
+                    )
+
+                    test_case = self.metadata.get_by_name(
+                        TestCase, test_case_fqn, fields=["testDefinition,testSuite"]
                     )
+                    if test_case is None:
+                        # Create the test case only if it does not exist
+                        yield Either(
+                            right=CreateTestCaseRequest(
+                                name=manifest_node.name,
+                                description=manifest_node.description,
+                                testDefinition=FullyQualifiedEntityName(
+                                    manifest_node.name
+                                ),
+                                entityLink=entity_link_str,
+                                testSuite=test_suite.fullyQualifiedName,
+                                parameterValues=create_test_case_parameter_values(
+                                    dbt_test
+                                ),
+                                displayName=None,
+                                owners=None,
+                            )
+                        )
         except Exception as err:  # pylint: disable=broad-except
             yield Either(
                 left=StackTraceError(
@@ -894,6 +937,12 @@ class DbtSource(DbtServiceSource):
                     f"Adding DBT Test Case Results for node: {manifest_node.name}"
                 )
                 dbt_test_result = dbt_test.get(DbtCommonEnum.RESULTS.value)
+                if not dbt_test_result:
+                    logger.warning(
+                        f"DBT Test Case Results not found for node: {manifest_node.name}"
+                    )
+                    return
+
                 test_case_status = TestCaseStatus.Aborted
                 test_result_value = 0
                 if dbt_test_result.status.value in [
@@ -927,8 +976,8 @@ class DbtSource(DbtServiceSource):
 
                 # Create the test case result object
                 test_case_result = TestCaseResult(
-                    timestamp=convert_timestamp_to_milliseconds(
-                        dbt_timestamp.timestamp()
+                    timestamp=Timestamp(
+                        convert_timestamp_to_milliseconds(dbt_timestamp.timestamp())
                     ),
                     testCaseStatus=test_case_status,
                     testResultValue=[