openmetadata-ingestion 1.9.4.0__py3-none-any.whl → 1.9.6.0__py3-none-any.whl

metadata/ingestion/source/database/bigquery/lineage.py

@@ -11,8 +11,6 @@
 """
 Handle big query lineage extraction
 """
-from typing import Dict, List
-
 from metadata.ingestion.source.database.bigquery.queries import (
     BIGQUERY_GET_STORED_PROCEDURE_QUERIES,
     BIGQUERY_STATEMENT,
@@ -22,7 +20,6 @@ from metadata.ingestion.source.database.bigquery.query_parser import (
 )
 from metadata.ingestion.source.database.lineage_source import LineageSource
 from metadata.ingestion.source.database.stored_procedures_mixin import (
-    QueryByProcedure,
     StoredProcedureLineageMixin,
 )
 from metadata.utils.helpers import get_start_and_end
@@ -42,20 +39,18 @@ class BigqueryLineageSource(
         AND (
             statement_type IN ("MERGE", "CREATE_TABLE_AS_SELECT", "UPDATE") 
             OR (statement_type = "INSERT" and UPPER(query) like '%%INSERT%%INTO%%SELECT%%')
+            OR (statement_type = "CREATE_TABLE" and UPPER(query) like '%%CLONE%%')
         )
     """
 
-    def get_stored_procedure_queries_dict(self) -> Dict[str, List[QueryByProcedure]]:
+    def get_stored_procedure_sql_statement(self) -> str:
         """
-        Pick the stored procedure name from the context
-        and return the list of associated queries
+        Return the SQL statement to get the stored procedure queries
         """
         start, _ = get_start_and_end(self.source_config.queryLogDuration)
         query = BIGQUERY_GET_STORED_PROCEDURE_QUERIES.format(
             start_date=start,
             region=self.service_connection.usageLocation,
         )
-        queries_dict = self.procedure_queries_dict(
-            query=query,
-        )
-        return queries_dict
+
+        return query

metadata/ingestion/source/database/dbt/constants.py

@@ -105,7 +105,6 @@ class SkipResourceTypeEnum(Enum):
 
     ANALYSIS = "analysis"
     TEST = "test"
-    SOURCE = "source"
 
 
 class CompiledQueriesEnum(Enum):

metadata/ingestion/source/database/dbt/metadata.py

@@ -543,7 +543,7 @@ class DbtSource(DbtServiceSource):
 
                     if (
                         dbt_objects.dbt_sources
-                        and resource_type == SkipResourceTypeEnum.SOURCE.value
+                        and resource_type == DbtCommonEnum.SOURCE.value
                     ):
                         self.add_dbt_sources(
                             key,
@@ -708,20 +708,6 @@ class DbtSource(DbtServiceSource):
                     )
                     continue
 
-        if dbt_node.resource_type == SkipResourceTypeEnum.SOURCE.value:
-            parent_fqn = fqn.build(
-                self.metadata,
-                entity_type=Table,
-                service_name=self.config.serviceName,
-                database_name=get_corrected_name(dbt_node.database),
-                schema_name=get_corrected_name(dbt_node.schema_),
-                table_name=dbt_node.name,
-            )
-
-            # check if the parent table exists in OM before adding it to the upstream list
-            if self._get_table_entity(table_fqn=parent_fqn):
-                upstream_nodes.append(parent_fqn)
-
         return upstream_nodes
 
     def parse_data_model_columns(

metadata/ingestion/source/database/hive/connection.py

@@ -48,11 +48,20 @@ from metadata.ingestion.connections.test_connections import (
     test_connection_db_schema_sources,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.hive.custom_hive_connection import (
+    CustomHiveConnection,
+)
 from metadata.utils.constants import THREE_MIN
+from metadata.utils.ssl_manager import check_ssl_and_init
 
 HIVE_POSTGRES_SCHEME = "hive+postgres"
 HIVE_MYSQL_SCHEME = "hive+mysql"
 
+# Monkey-patch the pyhive.hive module to use our custom connection
+import pyhive.hive
+
+pyhive.hive.Connection = CustomHiveConnection
+
 
 def get_connection_url(connection: HiveConnection) -> str:
     """
@@ -113,6 +122,34 @@ def get_connection(connection: HiveConnection) -> Engine:
             "kerberos_service_name"
         ] = connection.kerberosServiceName
 
+    # Handle SSL using SSL manager (following established patterns)
+    ssl_manager = check_ssl_and_init(connection)
+    if ssl_manager:
+        connection = ssl_manager.setup_ssl(connection)
+        # Store SSL manager for cleanup
+        connection._ssl_manager = ssl_manager
+
+    # Add SSL configuration to connection arguments if SSL is enabled
+    if hasattr(connection, "useSSL") and connection.useSSL:
+        if not connection.connectionArguments:
+            connection.connectionArguments = init_empty_connection_arguments()
+        connection.connectionArguments.root["use_ssl"] = True
+
+        # Add SSL certificate configuration if available
+        if hasattr(connection, "sslConfig") and connection.sslConfig:
+            if connection.sslConfig.root.sslCertificate:
+                connection.connectionArguments.root[
+                    "ssl_certfile"
+                ] = connection.sslConfig.root.sslCertificate
+            if connection.sslConfig.root.sslKey:
+                connection.connectionArguments.root[
+                    "ssl_keyfile"
+                ] = connection.sslConfig.root.sslKey
+            if connection.sslConfig.root.caCertificate:
+                connection.connectionArguments.root[
+                    "ssl_ca_certs"
+                ] = connection.sslConfig.root.caCertificate
+
     return create_generic_db_connection(
         connection=connection,
         get_connection_url_fn=get_connection_url,

metadata/ingestion/source/database/hive/custom_hive_connection.py

@@ -0,0 +1,195 @@
+import contextlib
+import getpass
+import ssl
+
+import thrift.protocol.TBinaryProtocol
+import thrift.transport.TSocket
+import thrift.transport.TTransport
+from pyhive.hive import Connection as BaseConnection
+from pyhive.hive import _check_status, get_installed_sasl
+from TCLIService import TCLIService, ttypes
+
+
+class CustomHiveConnection(BaseConnection):
+    """Custom Hive connection that integrates puretransport and SSL certificate support"""
+
+    def __init__(
+        self,
+        host=None,
+        port=None,
+        scheme=None,
+        username=None,
+        database="default",
+        auth=None,
+        configuration=None,
+        kerberos_service_name=None,
+        password=None,
+        check_hostname=None,
+        ssl_cert=None,
+        thrift_transport=None,
+        use_ssl=False,
+        ssl_certfile=None,
+        ssl_keyfile=None,
+        ssl_ca_certs=None,
+        ssl_cert_reqs=None,
+        ssl_check_hostname=None,
+    ):
+        """Connect to HiveServer2 with integrated puretransport and SSL support"""
+
+        # Handle HTTPS scheme with SSL context
+        if scheme in ("https", "http") and thrift_transport is None:
+            port = port or 1000
+            ssl_context = None
+            if scheme == "https":
+                from ssl import create_default_context
+
+                ssl_context = create_default_context()
+                ssl_context.check_hostname = check_hostname == "true"
+                ssl_cert = ssl_cert or "none"
+                ssl_cert_parameter_map = {
+                    "none": 0,  # CERT_NONE
+                    "optional": 1,  # CERT_OPTIONAL
+                    "required": 2,  # CERT_REQUIRED
+                }
+                ssl_context.verify_mode = ssl_cert_parameter_map.get(ssl_cert, 0)
+            thrift_transport = thrift.transport.THttpClient.THttpClient(
+                uri_or_host="{scheme}://{host}:{port}/cliservice/".format(
+                    scheme=scheme, host=host, port=port
+                ),
+                ssl_context=ssl_context,
+            )
+
+            if auth in ("BASIC", "NOSASL", "NONE", None):
+                # Always needs the Authorization header
+                self._set_authorization_header(thrift_transport, username, password)
+            elif auth == "KERBEROS" and kerberos_service_name:
+                self._set_kerberos_header(thrift_transport, kerberos_service_name, host)
+            else:
+                raise ValueError(
+                    "Authentication is not valid use one of:"
+                    "BASIC, NOSASL, KERBEROS, NONE"
+                )
+            host, port, auth, kerberos_service_name, password = (
+                None,
+                None,
+                None,
+                None,
+                None,
+            )
+
+        username = username or getpass.getuser()
+        configuration = configuration or {}
+
+        if (password is not None) != (auth in ("LDAP", "CUSTOM")):
+            raise ValueError(
+                "Password should be set if and only if in LDAP or CUSTOM mode; "
+                "Remove password or use one of those modes"
+            )
+        if (kerberos_service_name is not None) != (auth == "KERBEROS"):
+            raise ValueError(
+                "kerberos_service_name should be set if and only if in KERBEROS mode"
+            )
+
+        # Use puretransport if SSL is enabled or if thrift_transport is provided
+        if use_ssl or thrift_transport is not None:
+            if thrift_transport is not None:
+                # Use provided thrift_transport
+                self._transport = thrift_transport
+            else:
+                # Create puretransport with SSL
+                import puretransport
+
+                # Prepare socket_kwargs for SSL
+                socket_kwargs = {}
+                if ssl_certfile:
+                    socket_kwargs["certfile"] = ssl_certfile
+                if ssl_keyfile:
+                    socket_kwargs["keyfile"] = ssl_keyfile
+                if ssl_ca_certs:
+                    socket_kwargs["ca_certs"] = ssl_ca_certs
+                if ssl_cert_reqs is not None:
+                    socket_kwargs["cert_reqs"] = ssl_cert_reqs
+                elif use_ssl:
+                    socket_kwargs["cert_reqs"] = ssl.CERT_NONE
+
+                # Create puretransport
+                self._transport = puretransport.transport_factory(
+                    host=host or "localhost",
+                    port=port or 10000,
+                    username=username,
+                    password=password or username,
+                    use_ssl=use_ssl,
+                    socket_kwargs=socket_kwargs if socket_kwargs else None,
+                )
+        else:
+            # Use standard connection logic
+            if port is None:
+                port = 10000
+            if auth is None:
+                auth = "NONE"
+            socket = thrift.transport.TSocket.TSocket(host, port)
+            if auth == "NOSASL":
+                # NOSASL corresponds to hive.server2.authentication=NOSASL in hive-site.xml
+                self._transport = thrift.transport.TTransport.TBufferedTransport(socket)
+            elif auth in ("LDAP", "KERBEROS", "NONE", "CUSTOM"):
+                # Defer import so package dependency is optional
+                import thrift_sasl
+
+                if auth == "KERBEROS":
+                    # KERBEROS mode in hive.server2.authentication is GSSAPI in sasl library
+                    sasl_auth = "GSSAPI"
+                else:
+                    sasl_auth = "PLAIN"
+                    if password is None:
+                        # Password doesn't matter in NONE mode, just needs to be nonempty.
+                        password = "x"
+
+                self._transport = thrift_sasl.TSaslClientTransport(
+                    lambda: get_installed_sasl(
+                        host=host,
+                        sasl_auth=sasl_auth,
+                        service=kerberos_service_name,
+                        username=username,
+                        password=password,
+                    ),
+                    sasl_auth,
+                    socket,
+                )
+            else:
+                # All HS2 config options:
+                # https://cwiki.apache.org/confluence/display/Hive/Setting+Up+HiveServer2#SettingUpHiveServer2-Configuration
+                # PAM currently left to end user via thrift_transport option.
+                raise NotImplementedError(
+                    "Only NONE, NOSASL, LDAP, KERBEROS, CUSTOM "
+                    "authentication are supported, got {}".format(auth)
+                )
+
+        protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(self._transport)
+        self._client = TCLIService.Client(protocol)
+        # oldest version that still contains features we care about
+        # "V6 uses binary type for binary payload (was string) and uses columnar result set"
+        protocol_version = ttypes.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6
+
+        try:
+            self._transport.open()
+            open_session_req = ttypes.TOpenSessionReq(
+                client_protocol=protocol_version,
+                configuration=configuration,
+                username=username,
+            )
+            response = self._client.OpenSession(open_session_req)
+            _check_status(response)
+            assert (
+                response.sessionHandle is not None
+            ), "Expected a session from OpenSession"
+            self._sessionHandle = response.sessionHandle
+            assert (
+                response.serverProtocolVersion == protocol_version
+            ), "Unable to handle protocol version {}".format(
+                response.serverProtocolVersion
+            )
+            with contextlib.closing(self.cursor()) as cursor:
+                cursor.execute("USE `{}`".format(database))
+        except:
+            self._transport.close()
+            raise