openmetadata-ingestion 1.4.0.1__py3-none-any.whl → 1.4.2.0__py3-none-any.whl

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-05-23T06:46:22+00:00
+#   timestamp: 2024-06-12T13:05:57+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-05-23T06:46:22+00:00
+#   timestamp: 2024-06-12T13:05:57+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-05-23T06:46:22+00:00
+#   timestamp: 2024-06-12T13:05:57+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-05-23T06:46:22+00:00
+#   timestamp: 2024-06-12T13:05:57+00:00
 
 from __future__ import annotations
 

metadata/ingestion/source/dashboard/quicksight/metadata.py

@@ -129,11 +129,11 @@ class QuicksightSource(DashboardServiceSource):
         """
         return dashboard.Name
 
-    def get_dashboard_details(self, dashboard: dict) -> DashboardDetail:
+    def get_dashboard_details(self, dashboard: DashboardDetail) -> DashboardDetail:
         """
         Get Dashboard Details
         """
-        return DashboardDetail(**dashboard)
+        return dashboard
 
     def yield_dashboard(
         self, dashboard_details: DashboardDetail

metadata/ingestion/source/database/common_db_source.py

@@ -510,7 +510,7 @@ class CommonDbSourceService(
             self.register_record(table_request=table_request)
 
             # Flag view as visited
-            if table_type == TableType.View or schema_definition:
+            if table_type == TableType.View and schema_definition:
                 table_view = TableView.parse_obj(
                     {
                         "table_name": table_name,

metadata/ingestion/source/database/databricks/metadata.py

@@ -136,7 +136,11 @@ def get_columns(self, connection, table_name, schema=None, **kw):
     result = []
     for col_name, col_type, _comment in rows:
         # Handle both oss hive and Databricks' hive partition header, respectively
-        if col_name in ("# Partition Information", "# Partitioning"):
+        if col_name in (
+            "# Partition Information",
+            "# Partitioning",
+            "# Clustering Information",
+        ):
             break
         # Take out the more detailed type information
         # e.g. 'map<ixnt,int>' -> 'map'

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -180,7 +180,11 @@ def _(config: DbtCloudConfig):  # pylint: disable=too-many-locals
             )
         runs_data = response.get("data")
         if runs_data:
-            run_id = runs_data[0]["id"]
+            last_run = runs_data[0]
+            run_id = last_run["id"]
+            logger.info(
+                f"Retrieved last successful run [{run_id}] finished {last_run['finished_at_humanized']} (duration: {last_run['duration_humanized']})"
+            )
             try:
                 logger.debug("Requesting [dbt_catalog]")
                 dbt_catalog = client.get(

metadata/ingestion/source/database/dbt/metadata.py

@@ -339,11 +339,15 @@ class DbtSource(DbtServiceSource):
                 )
             for key, manifest_node in manifest_entities.items():
                 try:
+                    resource_type = getattr(
+                        manifest_node.resource_type,
+                        "value",
+                        manifest_node.resource_type,
+                    )
                     # If the run_results file is passed then only DBT tests will be processed
                     if (
                         dbt_objects.dbt_run_results
-                        and manifest_node.resource_type.value
-                        == SkipResourceTypeEnum.TEST.value
+                        and resource_type == SkipResourceTypeEnum.TEST.value
                     ):
                         # Test nodes will be processed further in the topology
                         self.add_dbt_tests(
@@ -360,9 +364,7 @@ class DbtSource(DbtServiceSource):
                         continue
 
                     # Skip the analysis and test nodes
-                    if manifest_node.resource_type.value in [
-                        item.value for item in SkipResourceTypeEnum
-                    ]:
+                    if resource_type in [item.value for item in SkipResourceTypeEnum]:
                         logger.debug(f"Skipping DBT node: {key}.")
                         continue
 
@@ -428,7 +430,7 @@ class DbtSource(DbtServiceSource):
                             table_entity=table_entity,
                             datamodel=DataModel(
                                 modelType=ModelType.DBT,
-                                resourceType=manifest_node.resource_type.value,
+                                resourceType=resource_type,
                                 description=manifest_node.description
                                 if manifest_node.description
                                 else None,

metadata/ingestion/source/database/query_parser_source.py

@@ -21,9 +21,10 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
+from metadata.ingestion.source.connections import get_test_connection_fn
 from metadata.utils.helpers import get_start_and_end
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.ssl_manager import get_ssl_connection
 
 logger = ingestion_logger()
 
@@ -55,7 +56,9 @@ class QueryParserSource(Source, ABC):
         self.service_connection = self.config.serviceConnection.__root__.config
         self.source_config = self.config.sourceConfig.config
         self.start, self.end = get_start_and_end(self.source_config.queryLogDuration)
-        self.engine = get_connection(self.service_connection) if get_engine else None
+        self.engine = (
+            get_ssl_connection(self.service_connection) if get_engine else None
+        )
 
     @property
     def name(self) -> str:

metadata/ingestion/source/messaging/kafka/metadata.py

@@ -11,7 +11,7 @@
 """
 Kafka source ingestion
 """
-from typing import Optional
+from typing import Optional, cast
 
 from metadata.generated.schema.entity.services.connections.messaging.kafkaConnection import (
     KafkaConnection,
@@ -28,7 +28,9 @@ from metadata.utils.ssl_manager import SSLManager
 class KafkaSource(CommonBrokerSource):
     def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
         self.ssl_manager = None
-        service_connection = config.serviceConnection.__root__.config
+        service_connection = cast(
+            KafkaConnection, config.serviceConnection.__root__.config
+        )
         if service_connection.schemaRegistrySSL:
 
             self.ssl_manager = SSLManager(
@@ -37,7 +39,7 @@ class KafkaSource(CommonBrokerSource):
                 cert=service_connection.schemaRegistrySSL.__root__.sslCertificate,
             )
             service_connection = self.ssl_manager.setup_ssl(
-                config.serviceConnection.__root__.config.sslConfig
+                config.serviceConnection.__root__.config
             )
         super().__init__(config, metadata)
 

metadata/ingestion/source/pipeline/openlineage/metadata.py

@@ -77,7 +77,9 @@ class OpenlineageSource(PipelineServiceSource):
     """
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         """Create class instance"""
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: OpenLineageConnection = config.serviceConnection.__root__.config
@@ -379,7 +381,7 @@ class OpenlineageSource(PipelineServiceSource):
             {json.dumps(pipeline_details.run_facet, indent=4).strip()}```"""
             request = CreatePipelineRequest(
                 name=pipeline_name,
-                service=self.context.pipeline_service,
+                service=self.context.get().pipeline_service,
                 description=description,
             )
 
@@ -433,8 +435,8 @@ class OpenlineageSource(PipelineServiceSource):
         pipeline_fqn = fqn.build(
             metadata=self.metadata,
             entity_type=Pipeline,
-            service_name=self.context.pipeline_service,
-            pipeline_name=self.context.pipeline,
+            service_name=self.context.get().pipeline_service,
+            pipeline_name=self.context.get().pipeline,
         )
 
         pipeline_entity = self.metadata.get_by_name(entity=Pipeline, fqn=pipeline_fqn)

metadata/profiler/interface/profiler_interface.py

@@ -51,7 +51,6 @@ from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline
 from metadata.generated.schema.tests.customMetric import CustomMetric
 from metadata.ingestion.api.status import Status
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.connections import get_connection
 from metadata.profiler.api.models import (
     DatabaseAndSchemaConfig,
     ProfilerProcessorConfig,
@@ -63,6 +62,7 @@ from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.processor.runner import QueryRunner
 from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT
 from metadata.utils.partition import get_partition_details
+from metadata.utils.ssl_manager import get_ssl_connection
 
 
 class ProfilerProcessorStatus(Status):
@@ -110,7 +110,7 @@ class ProfilerInterface(ABC):
         self.ometa_client = ometa_client
         self.source_config = source_config
         self.service_connection_config = service_connection_config
-        self.connection = get_connection(self.service_connection_config)
+        self.connection = get_ssl_connection(self.service_connection_config)
         self.status = ProfilerProcessorStatus()
         try:
             fqn = self.table_entity.fullyQualifiedName

metadata/profiler/processor/metric_filter.py

@@ -229,7 +229,7 @@ class MetricFilter:
         metric_names = next(
             (
                 include_columns.metrics
-                for include_columns in columns_config
+                for include_columns in columns_config or []
                 if include_columns.columnName == column.name
             ),
             None,

metadata/profiler/source/metadata_ext.py

@@ -41,7 +41,6 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 )
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.connections import get_connection
 from metadata.profiler.source.metadata import (
     OpenMetadataSource,
     ProfilerSourceAndEntity,
@@ -52,6 +51,7 @@ from metadata.utils.class_helper import get_service_type_from_source_type
 from metadata.utils.filters import filter_by_database, filter_by_schema, filter_by_table
 from metadata.utils.importer import import_source_class
 from metadata.utils.logger import profiler_logger
+from metadata.utils.ssl_manager import get_ssl_connection
 
 logger = profiler_logger()
 
@@ -112,7 +112,7 @@ class OpenMetadataSourceExt(OpenMetadataSource):
         if database_name:
             logger.info(f"Ingesting from database: {database_name}")
             new_service_connection.database = database_name
-        self.engine = get_connection(new_service_connection)
+        self.engine = get_ssl_connection(new_service_connection)
         self.inspector = inspect(self.engine)
         self._connection = None  # Lazy init as well
 

metadata/utils/ssl_manager.py

@@ -15,6 +15,7 @@ Module to manage SSL certificates
 """
 import os
 import tempfile
+import traceback
 from functools import singledispatch, singledispatchmethod
 from typing import Optional, Union, cast
 
@@ -43,6 +44,10 @@ from metadata.generated.schema.entity.services.connections.messaging.kafkaConnec
 )
 from metadata.generated.schema.security.ssl import verifySSLConfig
 from metadata.ingestion.connections.builders import init_empty_connection_arguments
+from metadata.ingestion.source.connections import get_connection
+from metadata.utils.logger import utils_logger
+
+logger = utils_logger()
 
 
 class SSLManager:
@@ -179,3 +184,14 @@ def _(connection):
             else None
         )
     return None
+
+
+def get_ssl_connection(service_config):
+    try:
+        ssl_manager: SSLManager = check_ssl_and_init(service_config)
+        if ssl_manager:
+            service_config = ssl_manager.setup_ssl(service_config)
+    except Exception:
+        logger.debug("Failed to setup SSL for the connection")
+        logger.debug(traceback.format_exc())
+    return get_connection(service_config)

metadata/utils/ssl_registry.py

@@ -39,7 +39,7 @@ def ignore_ssl_init(_: Optional[SslConfig]) -> bool:
 
 @ssl_verification_registry.add(VerifySSL.validate.value)
 def validate_ssl_init(ssl_config: Optional[SslConfig]) -> str:
-    return ssl_config.__root__.certificatePath
+    return ssl_config.__root__.caCertificate.get_secret_value()
 
 
 def get_verify_ssl_fn(verify_ssl: VerifySSL) -> Callable:

metadata/workflow/profiler.py

@@ -15,13 +15,14 @@ from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
 )
 from metadata.ingestion.api.steps import Processor, Sink
-from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
+from metadata.ingestion.source.connections import get_test_connection_fn
 from metadata.pii.processor import PIIProcessor
 from metadata.profiler.processor.processor import ProfilerProcessor
 from metadata.profiler.source.metadata import OpenMetadataSource
 from metadata.profiler.source.metadata_ext import OpenMetadataSourceExt
 from metadata.utils.importer import import_sink_class
 from metadata.utils.logger import profiler_logger
+from metadata.utils.ssl_manager import get_ssl_connection
 from metadata.workflow.ingestion import IngestionWorkflow
 
 logger = profiler_logger()
@@ -62,7 +63,7 @@ class ProfilerWorkflow(IngestionWorkflow):
 
     def test_connection(self):
         service_config = self.config.source.serviceConnection.__root__.config
-        conn = get_connection(service_config)
+        conn = get_ssl_connection(service_config)
 
         test_connection_fn = get_test_connection_fn(service_config)
         test_connection_fn(self.metadata, conn, service_config)