openmetadata-ingestion 1.3.1.3__py3-none-any.whl → 1.3.2.0__py3-none-any.whl

metadata/generated/schema/tests/testCase.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testCase.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/testCaseResolutionStatus.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testCaseResolutionStatus.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/testDefinition.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testDefinition.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/testSuite.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testSuite.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00

metadata/generated/schema/type/auditLog.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/auditLog.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/basic.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/basic.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/bulkOperationResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/bulkOperationResult.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/changeEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/changeEvent.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/changeEventType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/changeEventType.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/collectionDescriptor.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/collectionDescriptor.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvDocumentation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvDocumentation.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvErrorType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvErrorType.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvFile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvFile.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvImportResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvImportResult.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperties/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00

metadata/generated/schema/type/customProperties/enumConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/customProperties/enumConfig.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperty.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/customProperty.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/dailyCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/dailyCount.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-03-19T09:19:01+00:00
+#   timestamp: 2024-04-08T09:19:52+00:00
 
 from __future__ import annotations
 

metadata/great_expectations/action.py

@@ -15,6 +15,7 @@ Open Metadata table quality.
 This subpackage needs to be used in Great Expectations
 checkpoints actions.
 """
+import logging
 import traceback
 from datetime import datetime, timezone
 from typing import Dict, List, Optional, Union, cast
@@ -72,9 +73,10 @@ from metadata.great_expectations.utils.ometa_config_handler import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
 from metadata.utils.entity_link import get_entity_link
-from metadata.utils.logger import great_expectations_logger
 
-logger = great_expectations_logger()
+logger = logging.getLogger(
+    "great_expectations.validation_operators.validation_operators.openmetadata"
+)
 
 
 class OpenMetadataValidationAction(ValidationAction):
@@ -429,7 +431,7 @@ class OpenMetadataValidationAction(ValidationAction):
                 test_case_fqn=test_case.fullyQualifiedName.__root__,
             )
 
-            logger.info(
+            logger.debug(
                 f"Test case result for {test_case.fullyQualifiedName.__root__} successfully ingested"
             )
 

metadata/ingestion/api/parser.py

@@ -69,6 +69,28 @@ from metadata.generated.schema.metadataIngestion.databaseServiceQueryUsagePipeli
     DatabaseServiceQueryUsagePipeline,
     DatabaseUsageConfigType,
 )
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtAzureConfig import (
+    DbtAzureConfig,
+)
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtCloudConfig import (
+    DbtCloudConfig,
+)
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtGCSConfig import (
+    DbtGcsConfig,
+)
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtHttpConfig import (
+    DbtHttpConfig,
+)
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtLocalConfig import (
+    DbtLocalConfig,
+)
+from metadata.generated.schema.metadataIngestion.dbtconfig.dbtS3Config import (
+    DbtS3Config,
+)
+from metadata.generated.schema.metadataIngestion.dbtPipeline import (
+    DbtConfigType,
+    DbtPipeline,
+)
 from metadata.generated.schema.metadataIngestion.messagingServiceMetadataPipeline import (
     MessagingMetadataConfigType,
     MessagingServiceMetadataPipeline,
@@ -125,6 +147,16 @@ SOURCE_CONFIG_CLASS_MAP = {
     DatabaseMetadataConfigType.DatabaseMetadata.value: DatabaseServiceMetadataPipeline,
     StorageMetadataConfigType.StorageMetadata.value: StorageServiceMetadataPipeline,
     SearchMetadataConfigType.SearchMetadata.value: SearchServiceMetadataPipeline,
+    DbtConfigType.DBT.value: DbtPipeline,
+}
+
+DBT_CONFIG_TYPE_MAP = {
+    "cloud": DbtCloudConfig,
+    "local": DbtLocalConfig,
+    "http": DbtHttpConfig,
+    "s3": DbtS3Config,
+    "gcs": DbtGcsConfig,
+    "azure": DbtAzureConfig,
 }
 
 
@@ -171,6 +203,7 @@ def get_source_config_class(
     Type[PipelineServiceMetadataPipeline],
     Type[MlModelServiceMetadataPipeline],
     Type[DatabaseServiceMetadataPipeline],
+    Type[DbtPipeline],
 ]:
     """
     Return the source config type for a source string
@@ -179,7 +212,7 @@ def get_source_config_class(
     """
     source_config_class = SOURCE_CONFIG_CLASS_MAP.get(source_config_type)
 
-    if source_config_type:
+    if source_config_class:
         return source_config_class
 
     raise ValueError(f"Cannot find the service type of {source_config_type}")
@@ -266,6 +299,27 @@ def _unsafe_parse_config(config: dict, cls: Type[T], message: str) -> None:
         raise err
 
 
+def _unsafe_parse_dbt_config(config: dict, cls: Type[T], message: str) -> None:
+    """
+    Given a config dictionary and the class it should match,
+    try to parse it or log the given message
+    """
+    logger.debug(f"Parsing message: [{message}]")
+    try:
+        # Parse the oneOf config types of dbt to check
+        dbt_config_type = config["dbtConfigSource"]["dbtConfigType"]
+        dbt_config_class = DBT_CONFIG_TYPE_MAP.get(dbt_config_type)
+        dbt_config_class.parse_obj(config["dbtConfigSource"])
+
+        # Parse the entire dbtPipeline object
+        cls.parse_obj(config)
+    except ValidationError as err:
+        logger.debug(
+            f"The supported properties for {cls.__name__} are {list(cls.__fields__.keys())}"
+        )
+        raise err
+
+
 def _parse_inner_connection(config_dict: dict, source_type: str) -> None:
     """
     Parse the inner connection of the flagged connectors
@@ -291,32 +345,35 @@ def parse_service_connection(config_dict: dict) -> None:
     :param config_dict: JSON configuration
     """
     # Unsafe access to the keys. Allow a KeyError if the config is not well formatted
-    source_type = config_dict["source"]["serviceConnection"]["config"].get("type")
-    if source_type is None:
-        raise InvalidWorkflowException("Missing type in the serviceConnection config")
+    if config_dict["source"].get("serviceConnection"):
+        source_type = config_dict["source"]["serviceConnection"]["config"].get("type")
+        if source_type is None:
+            raise InvalidWorkflowException(
+                "Missing type in the serviceConnection config"
+            )
 
-    logger.debug(
-        f"Error parsing the Workflow Configuration for {source_type} ingestion"
-    )
+        logger.debug(
+            f"Error parsing the Workflow Configuration for {source_type} ingestion"
+        )
 
-    service_type = get_service_type(source_type)
-    connection_class = get_connection_class(source_type, service_type)
+        service_type = get_service_type(source_type)
+        connection_class = get_connection_class(source_type, service_type)
 
-    if source_type in HAS_INNER_CONNECTION:
-        # We will first parse the inner `connection` configuration
-        _parse_inner_connection(
-            config_dict["source"]["serviceConnection"]["config"]["connection"][
-                "config"
-            ]["connection"],
-            source_type,
-        )
+        if source_type in HAS_INNER_CONNECTION:
+            # We will first parse the inner `connection` configuration
+            _parse_inner_connection(
+                config_dict["source"]["serviceConnection"]["config"]["connection"][
+                    "config"
+                ]["connection"],
+                source_type,
+            )
 
-    # Parse the service connection dictionary with the scoped class
-    _unsafe_parse_config(
-        config=config_dict["source"]["serviceConnection"]["config"],
-        cls=connection_class,
-        message="Error parsing the service connection",
-    )
+        # Parse the service connection dictionary with the scoped class
+        _unsafe_parse_config(
+            config=config_dict["source"]["serviceConnection"]["config"],
+            cls=connection_class,
+            message="Error parsing the service connection",
+        )
 
 
 def parse_source_config(config_dict: dict) -> None:
@@ -334,6 +391,13 @@ def parse_source_config(config_dict: dict) -> None:
 
     source_config_class = get_source_config_class(source_config_type)
 
+    if source_config_class == DbtPipeline:
+        _unsafe_parse_dbt_config(
+            config=config_dict["source"]["sourceConfig"]["config"],
+            cls=source_config_class,
+            message="Error parsing the dbt source config",
+        )
+
     _unsafe_parse_config(
         config=config_dict["source"]["sourceConfig"]["config"],
         cls=source_config_class,

metadata/ingestion/api/topology_runner.py

@@ -265,11 +265,12 @@ class TopologyRunnerMixin(Generic[C]):
             if entity:
                 same_fingerprint = True
 
-        create_entity_request_hash = generate_source_hash(
-            create_request=entity_request.right,
-        )
+        create_entity_request_hash = None
 
         if hasattr(entity_request.right, "sourceHash"):
+            create_entity_request_hash = generate_source_hash(
+                create_request=entity_request.right,
+            )
             entity_request.right.sourceHash = create_entity_request_hash
 
         if entity is None and stage.use_cache: