openmetadata-ingestion 1.6.2.2__py3-none-any.whl → 1.6.3.0.dev0__py3-none-any.whl

metadata/utils/db_utils.py

@@ -69,6 +69,10 @@ def get_view_lineage(
         fqn=table_fqn,
     )
 
+    if not view_definition:
+        logger.warning(f"View definition for view {table_fqn} not available")
+        return
+
     try:
         connection_type = str(connection_type)
         dialect = ConnectionTypeDialectMapper.dialect_of(connection_type)

metadata/utils/logger.py

@@ -28,7 +28,6 @@ from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.type.queryParserData import QueryParserData
 from metadata.generated.schema.type.tableQuery import TableQueries
 from metadata.ingestion.api.models import Entity
-from metadata.ingestion.lineage.masker import mask_query
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -284,19 +283,13 @@ def _(record: PatchRequest) -> str:
 @get_log_name.register
 def _(record: TableQueries) -> str:
     """Get the log of the TableQuery"""
-    queries = "\n------\n".join(
-        mask_query(query.query, query.dialect) for query in record.queries
-    )
-    return f"Table Queries [{queries}]"
+    return f"Table Queries [{len(record.queries)}]"
 
 
 @get_log_name.register
 def _(record: QueryParserData) -> str:
     """Get the log of the ParsedData"""
-    queries = "\n------\n".join(
-        mask_query(query.sql, query.dialect) for query in record.parsedData
-    )
-    return f"Usage ParsedData [{queries}]"
+    return f"Usage ParsedData [{len(record.parsedData)}]"
 
 
 def redacted_config(config: Dict[str, Union[str, dict]]) -> Dict[str, Union[str, dict]]:

metadata/utils/ssl_manager.py

@@ -17,7 +17,7 @@ import os
 import tempfile
 import traceback
 from functools import singledispatch, singledispatchmethod
-from typing import Optional, Union, cast
+from typing import List, Optional, Union, cast
 
 from pydantic import SecretStr
 
@@ -66,7 +66,9 @@ logger = utils_logger()
 class SSLManager:
     "SSL Manager to manage SSL certificates for service connections"
 
-    def __init__(self, ca=None, key=None, cert=None):
+    def __init__(
+        self, ca=None, key=None, cert=None, *args, **kwargs
+    ):  # pylint: disable=keyword-arg-before-vararg
         self.temp_files = []
         self.ca_file_path = None
         self.cert_file_path = None
@@ -77,6 +79,14 @@ class SSLManager:
             self.cert_file_path = self.create_temp_file(cert)
         if key:
             self.key_file_path = self.create_temp_file(key)
+        if args:
+            for arg in args:
+                if arg:
+                    setattr(self, f"{arg}", self.create_temp_file(arg))
+        if kwargs:
+            for dict_key, value in kwargs.items():
+                if value:
+                    setattr(self, f"{dict_key}", self.create_temp_file(value))
 
     def create_temp_file(self, content: SecretStr):
         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@@ -197,8 +207,15 @@ class SSLManager:
         return connection
 
     @setup_ssl.register(KafkaConnection)
-    def _(self, connection):
+    def _(self, connection) -> KafkaConnection:
         connection = cast(KafkaConnection, connection)
+        if connection.consumerConfigSSL:
+            connection.consumerConfig = {
+                **connection.consumerConfig,
+                "ssl.ca.location": getattr(self, "ca_consumer_config", None),
+                "ssl.key.location": getattr(self, "key_consumer_config", None),
+                "ssl.certificate.location": getattr(self, "cert_consumer_config", None),
+            }
         connection.schemaRegistryConfig["ssl.ca.location"] = self.ca_file_path
         connection.schemaRegistryConfig["ssl.key.location"] = self.key_file_path
         connection.schemaRegistryConfig[
@@ -208,7 +225,9 @@ class SSLManager:
 
 
 @singledispatch
-def check_ssl_and_init(_) -> Optional[SSLManager]:
+def check_ssl_and_init(
+    _, *args, **kwargs  # pylint: disable=unused-argument
+) -> Optional[Union[SSLManager, List[SSLManager]]]:
     return None
 
 
@@ -274,6 +293,38 @@ def _(connection):
     return None
 
 
+@check_ssl_and_init.register(KafkaConnection)
+def _(connection, *args, **kwargs):
+
+    service_connection: KafkaConnection = cast(KafkaConnection, connection)
+    ssl_consumer_config: Optional[
+        verifySSLConfig.SslConfig
+    ] = service_connection.consumerConfigSSL
+    ssl_schema_registry: Optional[
+        verifySSLConfig.SslConfig
+    ] = service_connection.schemaRegistrySSL
+
+    ssl_consumer_config_dict = {}
+
+    if ssl_consumer_config:
+        ssl_consumer_config_dict = {
+            "ca_consumer_config": ssl_consumer_config.root.caCertificate,
+            "cert_consumer_config": ssl_consumer_config.root.sslCertificate,
+            "key_consumer_config": ssl_consumer_config.root.sslKey,
+        }
+    ssl_schema_registry_dict = {}
+
+    if ssl_schema_registry:
+        ssl_schema_registry_dict = {
+            "ca_schema_registry": ssl_schema_registry.root.caCertificate,
+            "cert_schema_registry": ssl_schema_registry.root.sslCertificate,
+            "key_schema_registry": ssl_schema_registry.root.sslKey,
+        }
+    if ssl_consumer_config_dict or ssl_schema_registry_dict:
+        return SSLManager(**ssl_consumer_config_dict, **ssl_schema_registry_dict)
+    return None
+
+
 @check_ssl_and_init.register(PostgresConnection)
 @check_ssl_and_init.register(RedshiftConnection)
 @check_ssl_and_init.register(GreenplumConnection)