openmetadata-ingestion 1.7.4.2__py3-none-any.whl → 1.8.0.0__py3-none-any.whl

metadata/ingestion/source/database/bigquery/helper.py

@@ -72,7 +72,9 @@ def get_inspector_details(
                 "lifetime"
             ] = new_service_connection.credentials.gcpImpersonateServiceAccount.lifetime
 
-    client = get_bigquery_client(project_id=database_name, **kwargs)
+    client = get_bigquery_client(
+        project_id=new_service_connection.billingProjectId or database_name, **kwargs
+    )
     engine = get_connection(new_service_connection)
     inspector = inspect(engine)
 
@@ -86,15 +88,15 @@ def get_pk_constraint(
     This function overrides to get primary key constraint
     """
     try:
-        constraints = PK_CACHE.get(f"{connection.engine.url.host}.{schema}")
+        project, schema = schema.split(".")
+        constraints = PK_CACHE.get(f"{project}.{schema}")
         if constraints is None:
             constraints = connection.engine.execute(
                 BIGQUERY_TABLE_CONSTRAINTS.format(
-                    project_id=connection.engine.url.host,
-                    schema_name=schema,
+                    project_id=project, schema_name=schema
                 )
             )
-            PK_CACHE[f"{connection.engine.url.host}.{schema}"] = constraints.fetchall()
+            PK_CACHE[f"{project}.{schema}"] = constraints.fetchall()
 
         col_name = []
         table_constraints = [row for row in constraints if row.table_name == table_name]
@@ -116,15 +118,15 @@ def get_foreign_keys(
     This function overrides to get foreign key constraint
     """
     try:
-        constraints = FK_CACHE.get(f"{connection.engine.url.host}.{schema}")
+        project, schema = schema.split(".")
+        constraints = FK_CACHE.get(f"{project}.{schema}")
         if constraints is None:
             constraints = connection.engine.execute(
                 BIGQUERY_FOREIGN_CONSTRAINTS.format(
-                    project_id=connection.engine.url.host,
-                    schema_name=schema,
+                    project_id=project, schema_name=schema
                 )
             )
-            FK_CACHE[f"{connection.engine.url.host}.{schema}"] = constraints.fetchall()
+            FK_CACHE[f"{project}.{schema}"] = constraints.fetchall()
 
         col_name = []
         table_constraints = [row for row in constraints if row.table_name == table_name]

metadata/ingestion/source/database/bigquery/metadata.py

@@ -83,7 +83,10 @@ from metadata.ingestion.source.database.bigquery.models import (
     BigQueryStoredProcedure,
 )
 from metadata.ingestion.source.database.bigquery.queries import (
+    BIGQUERY_GET_MATERIALIZED_VIEW_NAMES,
+    BIGQUERY_GET_SCHEMA_NAMES,
     BIGQUERY_GET_STORED_PROCEDURES,
+    BIGQUERY_GET_VIEW_NAMES,
     BIGQUERY_LIFE_CYCLE_QUERY,
     BIGQUERY_SCHEMA_DESCRIPTION,
     BIGQUERY_TABLE_AND_TYPE,
@@ -238,6 +241,7 @@ Inspector.get_all_table_ddls = get_all_table_ddls
 Inspector.get_table_ddl = get_table_ddl
 
 
+# pylint: disable=too-many-public-methods
 class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
     """
     Implements the necessary methods to extract
@@ -347,6 +351,33 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
             logger.debug(traceback.format_exc())
             raise InvalidSourceException(f"Error setting BigQuery project IDs: {exc}")
 
+    # pylint: disable=arguments-differ
+    def _get_columns_with_constraints(
+        self, schema_name: str, table_name: str, inspector: Inspector
+    ) -> Tuple[List, List, List]:
+        database_name = self.context.get().database
+        schema_name = f"{database_name}.{schema_name}"
+        return super()._get_columns_with_constraints(schema_name, table_name, inspector)
+
+    def _get_columns_internal(
+        self,
+        schema_name: str,
+        table_name: str,
+        db_name: str,
+        inspector: Inspector,
+        table_type: TableType = None,
+    ):
+        """
+        Get columns list
+        """
+
+        return inspector.get_columns(
+            table_name,
+            f"{db_name}.{schema_name}",
+            table_type=table_type,
+            db_name=db_name,
+        )
+
     def _test_connection(self) -> None:
         for project_id in self.project_ids:
             inspector_details = get_inspector_details(
@@ -375,7 +406,7 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
         table_names_and_types = (
             self.engine.execute(
                 BIGQUERY_TABLE_AND_TYPE.format(
-                    project_id=self.client.project, schema_name=schema_name
+                    project_id=self.context.get().database, schema_name=schema_name
                 )
             )
             or []
@@ -409,8 +440,6 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
         logic on how to handle table types, e.g., material views,...
         """
 
-        view_names = self.inspector.get_view_names(schema_name) or []
-
         if self.incremental.enabled:
             view_names = [
                 view_name
@@ -419,10 +448,50 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
                 in self.incremental_table_processor.get_not_deleted(schema_name)
             ]
 
-        return [
-            TableNameAndType(name=view_name, type_=TableType.View)
-            for view_name in view_names
-        ]
+        table_name_and_types = []
+        for table_type, query in {
+            TableType.View: BIGQUERY_GET_VIEW_NAMES,
+            TableType.MaterializedView: BIGQUERY_GET_MATERIALIZED_VIEW_NAMES,
+        }.items():
+            view_names = list(
+                map(
+                    lambda x: x[0],
+                    (
+                        self.engine.execute(
+                            query.format(
+                                project=self.context.get().database, dataset=schema_name
+                            )
+                        )
+                        or []
+                    ),
+                )
+            )
+            if self.incremental.enabled:
+                view_names = [
+                    view_name
+                    for view_name in view_names
+                    if view_name
+                    in self.incremental_table_processor.get_not_deleted(schema_name)
+                ]
+
+            table_name_and_types.extend(
+                [
+                    TableNameAndType(name=view_name, type_=table_type)
+                    for view_name in view_names
+                ]
+            )
+
+        return table_name_and_types
+
+    # pylint: disable=arguments-differ
+    @calculate_execution_time()
+    def get_table_description(
+        self, schema_name: str, table_name: str, inspector: Inspector
+    ) -> str:
+        schema_name = f"{self.context.get().database}.{schema_name}"
+        return super().get_table_description(
+            schema_name=schema_name, table_name=table_name, inspector=inspector
+        )
 
     def yield_tag(
         self, schema_name: str
@@ -430,7 +499,9 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
         """Build tag context"""
         try:
             # Fetching labels on the databaseSchema ( dataset ) level
-            dataset_obj = self.client.get_dataset(schema_name)
+            dataset_obj = self.client.get_dataset(
+                f"{self.context.get().database}.{schema_name}"
+            )
             if dataset_obj.labels:
                 for key, value in dataset_obj.labels.items():
                     yield from get_ometa_tag_and_classification(
@@ -477,7 +548,7 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
         try:
             query_resp = self.client.query(
                 BIGQUERY_SCHEMA_DESCRIPTION.format(
-                    project_id=self.client.project,
+                    project_id=self.context.get().database,
                     region=self.service_connection.usageLocation,
                     schema_name=schema_name,
                 )
@@ -505,7 +576,7 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
         3. Adds the Deleted Tables to the context
         """
         self.incremental_table_processor.set_changed_tables_map(
-            project=self.client.project,
+            project=self.context.get().database,
             dataset=schema_name,
             start_date=self.incremental.start_datetime_utc,
         )
@@ -526,6 +597,18 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
             ]
         )
 
+    def get_raw_database_schema_names(self) -> Iterable[str]:
+        if self.service_connection.__dict__.get("databaseSchema"):
+            yield self.service_connection.databaseSchema
+        else:
+            for schema in self.engine.execute(
+                BIGQUERY_GET_SCHEMA_NAMES.format(
+                    project=self.context.get().database,
+                    region=self.service_connection.usageLocation,
+                )
+            ):
+                yield schema[0]
+
     def _get_filtered_schema_names(
         self, return_fqn: bool = False, add_to_status: bool = True
     ) -> Iterable[str]:
@@ -575,7 +658,9 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
             ),
         )
         if self.source_config.includeTags:
-            dataset_obj = self.client.get_dataset(schema_name)
+            dataset_obj = self.client.get_dataset(
+                f"{self.context.get().database}.{schema_name}"
+            )
             if dataset_obj.labels:
                 database_schema_request_obj.tags = []
                 for label_classification, label_tag_name in dataset_obj.labels.items():
@@ -707,7 +792,9 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
 
             if self.source_config.includeDDL:
                 schema_definition = inspector.get_table_ddl(
-                    self.connection, table_name, schema_name
+                    self.connection,
+                    table_name,
+                    f"{self.context.get().database}.{schema_name}",
                 )
                 schema_definition = (
                     str(schema_definition).strip()
@@ -784,7 +871,9 @@ class BigquerySource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource):
         try:
             database = self.context.get().database
             table = self.client.get_table(fqn._build(database, schema_name, table_name))
-            columns = inspector.get_columns(table_name, schema_name, db_name=database)
+            columns = inspector.get_columns(
+                table_name, f"{database}.{schema_name}", db_name=database
+            )
             if (
                 hasattr(table, "external_data_configuration")
                 and hasattr(table.external_data_configuration, "hive_partitioning")

metadata/ingestion/source/database/bigquery/queries.py

@@ -63,7 +63,7 @@ BIGQUERY_SCHEMA_DESCRIPTION = textwrap.dedent(
 
 BIGQUERY_TABLE_AND_TYPE = textwrap.dedent(
     """
-    select table_name, table_type from `{project_id}`.{schema_name}.INFORMATION_SCHEMA.TABLES where table_type != 'VIEW'
+    select table_name, table_type from `{project_id}`.{schema_name}.INFORMATION_SCHEMA.TABLES where table_type NOT IN  ('VIEW', 'MATERIALIZED VIEW')
     """
 )
 
@@ -96,7 +96,7 @@ SELECT
   routine_name as name,
   routine_definition as definition,
   external_language as language
-FROM `{schema_name}`.INFORMATION_SCHEMA.ROUTINES
+FROM `{database_name}`.`{schema_name}`.INFORMATION_SCHEMA.ROUTINES
 WHERE routine_type in ('PROCEDURE', 'TABLE FUNCTION')
   AND routine_catalog = '{database_name}'
   AND routine_schema = '{schema_name}'
@@ -164,7 +164,7 @@ BIGQUERY_LIFE_CYCLE_QUERY = textwrap.dedent(
 select
 table_name as table_name,
 creation_time as created_at
-from `{schema_name}`.INFORMATION_SCHEMA.TABLES
+from `{database_name}`.`{schema_name}`.INFORMATION_SCHEMA.TABLES
 where table_schema = '{schema_name}'
 and table_catalog = '{database_name}'
 """
@@ -182,6 +182,18 @@ AND resource.labels.dataset_id = "{dataset}"
 AND timestamp >= "{start_date}"
 """
 
+BIGQUERY_GET_SCHEMA_NAMES = """
+SELECT schema_name FROM `{project}`.`region-{region}`.INFORMATION_SCHEMA.SCHEMATA;
+"""
+
+BIGQUERY_GET_VIEW_NAMES = """
+SELECT table_name FROM `{project}`.`{dataset}`.INFORMATION_SCHEMA.VIEWS;
+"""
+
+BIGQUERY_GET_MATERIALIZED_VIEW_NAMES = """
+SELECT table_name FROM `{project}`.`{dataset}`.INFORMATION_SCHEMA.MATERIALIZED_VIEWS;
+"""
+
 
 class BigQueryQueryResult(BaseModel):
     project_id: str

metadata/ingestion/source/database/common_db_source.py

@@ -61,10 +61,8 @@ from metadata.ingestion.connections.session import create_and_bind_thread_safe_s
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.models.patch_request import PatchedEntity, PatchRequest
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.connections import (
-    get_connection,
-    kill_active_connections,
-)
+from metadata.ingestion.source.connections import get_connection
+from metadata.ingestion.source.connections_utils import kill_active_connections
 from metadata.ingestion.source.database.database_service import DatabaseServiceSource
 from metadata.ingestion.source.database.sql_column_handler import SqlColumnHandlerMixin
 from metadata.ingestion.source.database.sqlalchemy_source import SqlAlchemySource

metadata/ingestion/source/database/database_service.py

@@ -57,9 +57,6 @@ from metadata.ingestion.api.delete import delete_entity_from_source
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.api.topology_runner import TopologyRunnerMixin
-from metadata.ingestion.connections.test_connections import (
-    raise_test_connection_exception,
-)
 from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.models.topology import (
@@ -68,7 +65,7 @@ from metadata.ingestion.models.topology import (
     TopologyContextManager,
     TopologyNode,
 )
-from metadata.ingestion.source.connections import get_test_connection_fn
+from metadata.ingestion.source.connections import test_connection_common
 from metadata.utils import fqn
 from metadata.utils.execution_time_tracker import calculate_execution_time
 from metadata.utils.filters import filter_by_schema
@@ -604,8 +601,6 @@ class DatabaseServiceSource(
         """
 
     def test_connection(self) -> None:
-        test_connection_fn = get_test_connection_fn(self.service_connection)
-        result = test_connection_fn(
+        test_connection_common(
             self.metadata, self.connection_obj, self.service_connection
         )
-        raise_test_connection_exception(result)

metadata/ingestion/source/database/datalake/metadata.py

@@ -248,7 +248,7 @@ class DatalakeSource(DatabaseServiceSource):
         if self.source_config.includeTables:
             for key_name in self.client.get_table_names(bucket_name, prefix):
                 table_name = self.standardize_table_name(bucket_name, key_name)
-
+                logger.info(f"Processing table: {table_name}")
                 if self.filter_dl_table(table_name):
                     continue
 

metadata/ingestion/source/database/dbt/metadata.py

@@ -83,7 +83,6 @@ from metadata.ingestion.source.database.dbt.dbt_service import (
 )
 from metadata.ingestion.source.database.dbt.dbt_utils import (
     check_ephemeral_node,
-    check_or_create_test_suite,
     create_test_case_parameter_definitions,
     create_test_case_parameter_values,
     generate_entity_link,
@@ -1024,9 +1023,6 @@ class DbtSource(DbtServiceSource):
                 logger.debug(f"Processing DBT Test Case for node: {manifest_node.name}")
                 entity_link_list = generate_entity_link(dbt_test)
                 for entity_link_str in entity_link_list:
-                    test_suite = check_or_create_test_suite(
-                        self.metadata, entity_link_str
-                    )
                     table_fqn = get_table_fqn(entity_link_str)
                     logger.debug(f"Table fqn found: {table_fqn}")
                     source_elements = table_fqn.split(fqn.FQN_SEPARATOR)
@@ -1056,7 +1052,6 @@ class DbtSource(DbtServiceSource):
                                     manifest_node.name
                                 ),
                                 entityLink=entity_link_str,
-                                testSuite=test_suite.fullyQualifiedName,
                                 parameterValues=create_test_case_parameter_values(
                                     dbt_test
                                 ),

metadata/ingestion/source/database/mysql/connection.py

@@ -20,6 +20,9 @@ from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
+from metadata.generated.schema.entity.services.connections.database.common.azureConfig import (
+    AzureConfigurationSource,
+)
 from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
     BasicAuth,
 )
@@ -34,6 +37,7 @@ from metadata.ingestion.connections.builders import (
     get_connection_args_common,
     get_connection_url_common,
 )
+from metadata.ingestion.connections.connection import BaseConnection
 from metadata.ingestion.connections.test_connections import (
     test_connection_db_schema_sources,
 )
@@ -45,51 +49,54 @@ from metadata.ingestion.source.database.mysql.queries import (
 from metadata.utils.constants import THREE_MIN
 
 
-def get_connection(connection: MysqlConnection) -> Engine:
-    """
-    Create connection
-    """
-    if hasattr(connection.authType, "azureConfig"):
-        azure_client = AzureClient(connection.authType.azureConfig).create_client()
-        if not connection.authType.azureConfig.scopes:
-            raise ValueError(
-                "Azure Scopes are missing, please refer https://learn.microsoft.com/"
-                "en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-micr"
-                "osoft-entra-access-token and fetch the resource associated with it,"
-                " for e.g. https://ossrdbms-aad.database.windows.net/.default"
+class MySQLConnection(BaseConnection[MysqlConnection, Engine]):
+    def get_client(self) -> Engine:
+        """
+        Return the SQLAlchemy Engine for MySQL.
+        """
+        connection = self.service_connection
+
+        if isinstance(connection.authType, AzureConfigurationSource):
+            if not connection.authType.azureConfig:
+                raise ValueError("Azure Config is missing")
+            azure_client = AzureClient(connection.authType.azureConfig).create_client()
+            if not connection.authType.azureConfig.scopes:
+                raise ValueError(
+                    "Azure Scopes are missing, please refer https://learn.microsoft.com/"
+                    "en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-micr"
+                    "osoft-entra-access-token and fetch the resource associated with it,"
+                    " for e.g. https://ossrdbms-aad.database.windows.net/.default"
+                )
+            access_token_obj = azure_client.get_token(
+                *connection.authType.azureConfig.scopes.split(",")
             )
-        access_token_obj = azure_client.get_token(
-            *connection.authType.azureConfig.scopes.split(",")
+            connection.authType = BasicAuth(password=access_token_obj.token)  # type: ignore
+        return create_generic_db_connection(
+            connection=connection,
+            get_connection_url_fn=get_connection_url_common,
+            get_connection_args_fn=get_connection_args_common,
         )
-        connection.authType = BasicAuth(password=access_token_obj.token)
-    return create_generic_db_connection(
-        connection=connection,
-        get_connection_url_fn=get_connection_url_common,
-        get_connection_args_fn=get_connection_args_common,
-    )
 
-
-def test_connection(
-    metadata: OpenMetadata,
-    engine: Engine,
-    service_connection: MysqlConnection,
-    automation_workflow: Optional[AutomationWorkflow] = None,
-    timeout_seconds: Optional[int] = THREE_MIN,
-) -> TestConnectionResult:
-    """
-    Test connection. This can be executed either as part
-    of a metadata workflow or during an Automation Workflow
-    """
-    queries = {
-        "GetQueries": MYSQL_TEST_GET_QUERIES
-        if not service_connection.useSlowLogs
-        else MYSQL_TEST_GET_QUERIES_SLOW_LOGS,
-    }
-    return test_connection_db_schema_sources(
-        metadata=metadata,
-        engine=engine,
-        service_connection=service_connection,
-        automation_workflow=automation_workflow,
-        timeout_seconds=timeout_seconds,
-        queries=queries,
-    )
+    def test_connection(
+        self,
+        metadata: OpenMetadata,
+        automation_workflow: Optional[AutomationWorkflow] = None,
+        timeout_seconds: Optional[int] = THREE_MIN,
+    ) -> TestConnectionResult:
+        """
+        Test connection. This can be executed either as part
+        of a metadata workflow or during an Automation Workflow
+        """
+        queries = {
+            "GetQueries": MYSQL_TEST_GET_QUERIES
+            if not self.service_connection.useSlowLogs
+            else MYSQL_TEST_GET_QUERIES_SLOW_LOGS,
+        }
+        return test_connection_db_schema_sources(
+            metadata=metadata,
+            engine=self.get_client(),
+            service_connection=self.service_connection,
+            automation_workflow=automation_workflow,
+            timeout_seconds=timeout_seconds,
+            queries=queries,
+        )

metadata/ingestion/source/database/mysql/service_spec.py

@@ -1,3 +1,4 @@
+from metadata.ingestion.source.database.mysql.connection import MySQLConnection
 from metadata.ingestion.source.database.mysql.lineage import MysqlLineageSource
 from metadata.ingestion.source.database.mysql.metadata import MysqlSource
 from metadata.ingestion.source.database.mysql.usage import MysqlUsageSource
@@ -7,4 +8,5 @@ ServiceSpec = DefaultDatabaseSpec(
     metadata_source_class=MysqlSource,
     lineage_source_class=MysqlLineageSource,
     usage_source_class=MysqlUsageSource,
+    connection_class=MySQLConnection,
 )

metadata/ingestion/source/database/query_parser_source.py

@@ -20,13 +20,10 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 )
 from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.api.steps import Source
-from metadata.ingestion.connections.test_connections import (
-    raise_test_connection_exception,
-)
 from metadata.ingestion.lineage.masker import masked_query_cache
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.connections import get_test_connection_fn
+from metadata.ingestion.source.connections import test_connection_common
 from metadata.utils.helpers import get_start_and_end, retry_with_docker_host
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.ssl_manager import get_ssl_connection
@@ -134,6 +131,4 @@ class QueryParserSource(Source, ABC):
         masked_query_cache.clear()
 
     def test_connection(self) -> None:
-        test_connection_fn = get_test_connection_fn(self.service_connection)
-        result = test_connection_fn(self.metadata, self.engine, self.service_connection)
-        raise_test_connection_exception(result)
+        test_connection_common(self.metadata, self.engine, self.service_connection)

metadata/ingestion/source/database/redshift/connection.py

@@ -40,7 +40,7 @@ from metadata.ingestion.connections.test_connections import (
     test_query,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.connections import kill_active_connections
+from metadata.ingestion.source.connections_utils import kill_active_connections
 from metadata.ingestion.source.database.redshift.queries import (
     REDSHIFT_GET_ALL_RELATIONS,
     REDSHIFT_GET_DATABASE_NAMES,

metadata/ingestion/source/database/sample_data.py

@@ -890,6 +890,7 @@ class SampleDataSource(
                 tags=table["tags"],
                 schemaDefinition=table.get("schemaDefinition"),
                 sourceUrl=table.get("sourceUrl"),
+                tablePartition=table.get("tablePartition"),
             )
 
             yield Either(right=table_and_db)
@@ -1602,7 +1603,6 @@ class SampleDataSource(
                         description=test_case["description"],
                         testDefinition=test_case["testDefinitionName"],
                         entityLink=test_case["entityLink"],
-                        testSuite=suite.fullyQualifiedName.root,
                         parameterValues=[
                             TestCaseParameterValue(**param_values)
                             for param_values in test_case["parameterValues"]

metadata/ingestion/source/database/saphana/lineage.py

@@ -29,11 +29,8 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 )
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException, Source
-from metadata.ingestion.connections.test_connections import (
-    raise_test_connection_exception,
-)
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.connections import get_test_connection_fn
+from metadata.ingestion.source.connections import test_connection_common
 from metadata.ingestion.source.database.saphana.cdata_parser import (
     ParsedLineage,
     parse_registry,
@@ -158,6 +155,4 @@ class SaphanaLineageSource(Source):
             )
 
     def test_connection(self) -> None:
-        test_connection_fn = get_test_connection_fn(self.service_connection)
-        result = test_connection_fn(self.metadata, self.engine, self.service_connection)
-        raise_test_connection_exception(result)
+        test_connection_common(self.metadata, self.engine, self.service_connection)

metadata/ingestion/source/database/sas/metadata.py

@@ -70,12 +70,9 @@ from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.common import Entity
 from metadata.ingestion.api.models import Either, StackTraceError
 from metadata.ingestion.api.steps import InvalidSourceException
-from metadata.ingestion.connections.test_connections import (
-    raise_test_connection_exception,
-)
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
+from metadata.ingestion.source.connections import get_connection, test_connection_common
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.ingestion.source.database.database_service import DatabaseServiceSource
 from metadata.ingestion.source.database.sas.client import SASClient
@@ -900,8 +897,6 @@ class SasSource(
         pass
 
     def test_connection(self) -> None:
-        test_connection_fn = get_test_connection_fn(self.service_connection)
-        result = test_connection_fn(
+        test_connection_common(
             self.metadata, self.connection_obj, self.service_connection
         )
-        raise_test_connection_exception(result)

metadata/ingestion/source/database/snowflake/metadata.py

@@ -561,11 +561,10 @@ class SnowflakeSource(
         table_list.extend(
             self._get_table_names_and_types(schema_name, table_type=TableType.External)
         )
-        logger.debug(f"Starting dynamic tables processing")
+
         table_list.extend(
             self._get_table_names_and_types(schema_name, table_type=TableType.Dynamic)
         )
-        logger.debug(f"Dynamic tables processing completed")
 
         if self.service_connection.includeTransientTables:
             table_list.extend(

metadata/ingestion/source/database/snowflake/utils.py

@@ -54,14 +54,12 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_INCREMENTAL_GET_WITHOUT_TRANSIENT_TABLE_NAMES,
 )
 from metadata.utils import fqn
-from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import (
     get_display_datatype,
     get_table_comment_wrapper,
     get_view_definition_wrapper,
 )
 
-logger = ingestion_logger()
 dialect = SnowflakeDialect()
 Query = str
 QueryMap = Dict[str, Query]
@@ -240,8 +238,6 @@ def get_table_names(self, connection, schema: str, **kw):
             for row in cursor
         ]
     )
-    if kw.get("dynamic_tables"):
-        logger.debug(f"Dynamic tables: {result}")
     return result