openmetadata-ingestion 1.8.9.0__py3-none-any.whl → 1.9.0.0__py3-none-any.whl

metadata/ingestion/source/database/trino/connection.py

@@ -13,24 +13,27 @@
 Source connection handler
 """
 from copy import deepcopy
-from typing import Optional
+from typing import Optional, cast
 from urllib.parse import quote_plus
 
 from requests import Session
 from sqlalchemy.engine import Engine
 from trino.auth import BasicAuthentication, JWTAuthentication, OAuth2Authentication
 
-from metadata.clients.azure_client import AzureClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
+from metadata.generated.schema.entity.services.connections.connectionBasicType import (
+    ConnectionArguments,
+)
 from metadata.generated.schema.entity.services.connections.database.common import (
+    azureConfig,
     basicAuth,
     jwtAuth,
     noConfigAuthenticationTypes,
 )
 from metadata.generated.schema.entity.services.connections.database.trinoConnection import (
-    TrinoConnection,
+    TrinoConnection as TrinoConnectionConfig,
 )
 from metadata.generated.schema.entity.services.connections.testConnectionResult import (
     TestConnectionResult,
@@ -41,6 +44,7 @@ from metadata.ingestion.connections.builders import (
     init_empty_connection_arguments,
     init_empty_connection_options,
 )
+from metadata.ingestion.connections.connection import BaseConnection
 from metadata.ingestion.connections.secrets import connection_with_options_secrets
 from metadata.ingestion.connections.test_connections import (
     test_connection_db_schema_sources,
@@ -48,151 +52,301 @@ from metadata.ingestion.connections.test_connections import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.trino.queries import TRINO_GET_DATABASE
 from metadata.utils.constants import THREE_MIN
+from metadata.utils.credentials import get_azure_access_token
 
 
-def get_connection_url(connection: TrinoConnection) -> str:
-    """
-    Prepare the connection url for trino
-    """
-    url = f"{connection.scheme.value}://"
-
-    # leaving username here as, even though with basic auth is used directly
-    # in BasicAuthentication class, it's often also required as a part of url.
-    # For example - it will be used by OAuth2Authentication to persist token in
-    # cache more efficiently (per user instead of per host)
-    if connection.username:
-        url += f"{quote_plus(connection.username)}@"
-
-    url += f"{connection.hostPort}"
-    if connection.catalog:
-        url += f"/{connection.catalog}"
-    if connection.connectionOptions is not None:
-        params = "&".join(
-            f"{key}={quote_plus(value)}"
-            for (key, value) in connection.connectionOptions.root.items()
-            if value
-        )
-        url = f"{url}?{params}"
-    return url
+# pylint: disable=unused-argument
+def _is_disconnect(self, e, connection, cursor):
+    """is_disconnect method for the Databricks dialect"""
+    if "JWT expired" in str(e):
+        return True
+    return False
 
 
-@connection_with_options_secrets
-def get_connection_args(connection: TrinoConnection):
-    if not connection.connectionArguments:
-        connection.connectionArguments = init_empty_connection_arguments()
+class TrinoConnection(BaseConnection[TrinoConnectionConfig, Engine]):
+    def __init__(self, connection: TrinoConnectionConfig):
+        super().__init__(connection)
 
-    if connection.proxies:
-        session = Session()
-        session.proxies = connection.proxies
+    def _get_client(self) -> Engine:
+        """
+        Create connection
+        """
+        # here we are creating a copy of connection, because we need to dynamically
+        # add auth params to connectionArguments, which we do no intend to store
+        # in original connection object and in OpenMetadata database
+        from trino.sqlalchemy.dialect import TrinoDialect
 
-        connection.connectionArguments.root["http_session"] = session
+        TrinoDialect.is_disconnect = _is_disconnect  # type: ignore
 
-    if isinstance(connection.authType, basicAuth.BasicAuth):
-        connection.connectionArguments.root["auth"] = BasicAuthentication(
-            connection.username,
-            connection.authType.password.get_secret_value()
-            if connection.authType.password
-            else None,
+        connection = self.service_connection
+        connection_copy = deepcopy(connection)
+
+        if hasattr(connection.authType, "azureConfig"):
+            auth_type = cast(azureConfig.AzureConfigurationSource, connection.authType)
+            access_token = get_azure_access_token(auth_type)
+            if not connection.connectionOptions:
+                connection.connectionOptions = init_empty_connection_options()
+            connection.connectionOptions.root["access_token"] = access_token
+
+        # Update the connection with the connection arguments
+        connection_copy.connectionArguments = self.build_connection_args(
+            connection_copy
         )
-        connection.connectionArguments.root["http_scheme"] = "https"
 
-    elif isinstance(connection.authType, jwtAuth.JwtAuth):
-        connection.connectionArguments.root["auth"] = JWTAuthentication(
-            connection.authType.jwt.get_secret_value()
+        return create_generic_db_connection(
+            connection=connection_copy,
+            get_connection_url_fn=self.get_connection_url,
+            get_connection_args_fn=get_connection_args_common,
         )
-        connection.connectionArguments.root["http_scheme"] = "https"
 
-    elif hasattr(connection.authType, "azureConfig"):
-        if not connection.authType.azureConfig.scopes:
-            raise ValueError(
-                "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default"
+    def test_connection(
+        self,
+        metadata: OpenMetadata,
+        automation_workflow: Optional[AutomationWorkflow] = None,
+        timeout_seconds: Optional[int] = THREE_MIN,
+    ) -> TestConnectionResult:
+        """
+        Test connection. This can be executed either as part
+        of a metadata workflow or during an Automation Workflow
+        """
+        queries = {
+            "GetDatabases": TRINO_GET_DATABASE,
+        }
+
+        return test_connection_db_schema_sources(
+            metadata=metadata,
+            engine=self.client,
+            service_connection=self.service_connection,
+            automation_workflow=automation_workflow,
+            queries=queries,
+            timeout_seconds=timeout_seconds,
+        )
+
+    def get_connection_dict(self) -> dict:
+        """
+        Return the connection dictionary for this service.
+        """
+        url = self.client.url
+        connection_copy = deepcopy(self.service_connection)
+
+        connection_dict = {
+            "driver": url.drivername,
+            "host": url.host,
+            "port": url.port,
+            "user": url.username,
+            "catalog": url.database,
+            "schema": url.query.get("schema"),
+        }
+
+        connection_dict.update(url.query)
+
+        if connection_copy.proxies:
+            connection_dict["http_session"] = connection_copy.proxies
+
+        if (
+            connection_copy.connectionArguments
+            and connection_copy.connectionArguments.root
+        ):
+            connection_with_options_secrets(lambda: connection_copy)
+            connection_dict.update(get_connection_args_common(connection_copy))
+
+        if isinstance(connection_copy.authType, basicAuth.BasicAuth):
+            connection_dict["auth"] = TrinoConnection.get_basic_auth_dict(
+                connection_copy
             )
+            connection_dict["http_scheme"] = "https"
 
-        azure_client = AzureClient(connection.authType.azureConfig).create_client()
+        elif isinstance(connection_copy.authType, jwtAuth.JwtAuth):
+            connection_dict["auth"] = TrinoConnection.get_jwt_auth_dict(connection_copy)
+            connection_dict["http_scheme"] = "https"
 
-        access_token_obj = azure_client.get_token(
-            *connection.authType.azureConfig.scopes.split(",")
-        )
+        elif hasattr(connection_copy.authType, "azureConfig"):
+            connection_dict["auth"] = TrinoConnection.get_azure_auth_dict(
+                connection_copy
+            )
+            connection_dict["http_scheme"] = "https"
+
+        elif (
+            connection_copy.authType
+            == noConfigAuthenticationTypes.NoConfigAuthenticationTypes.OAuth2
+        ):
+            connection_dict["auth"] = TrinoConnection.get_oauth2_auth_dict(
+                connection_copy
+            )
+            connection_dict["http_scheme"] = "https"
+
+        return connection_dict
+
+    @staticmethod
+    def get_connection_url(connection: TrinoConnectionConfig) -> str:
+        """
+        Prepare the connection url for trino
+        """
+
+        url = f"{connection.scheme.value}://"
 
-        connection.connectionArguments.root["auth"] = JWTAuthentication(
-            access_token_obj.token
+        # leaving username here as, even though with basic auth is used directly
+        # in BasicAuthentication class, it's often also required as a part of url.
+        # For example - it will be used by OAuth2Authentication to persist token in
+        # cache more efficiently (per user instead of per host)
+        if connection.username:
+            url += f"{quote_plus(connection.username)}@"
+
+        url += f"{connection.hostPort}"
+        if connection.catalog:
+            url += f"/{connection.catalog}"
+        if connection.connectionOptions is not None:
+            params = "&".join(
+                f"{key}={quote_plus(value)}"
+                for (key, value) in connection.connectionOptions.root.items()
+                if value
+            )
+            url = f"{url}?{params}"
+        return url
+
+    @staticmethod
+    @connection_with_options_secrets
+    def build_connection_args(connection: TrinoConnectionConfig) -> ConnectionArguments:
+        """
+        Get the connection args for the trino connection
+        """
+        connection_args: ConnectionArguments = (
+            connection.connectionArguments or init_empty_connection_arguments()
         )
-        connection.connectionArguments.root["http_scheme"] = "https"
+        assert connection_args.root is not None
+
+        if connection.verify:
+            connection_args.root["verify"] = {"verify": connection.verify}
+
+        if connection.proxies:
+            session = Session()
+            session.proxies = connection.proxies
+
+            connection_args.root["http_session"] = session
+
+        if isinstance(connection.authType, basicAuth.BasicAuth):
+            TrinoConnection.set_basic_auth(connection, connection_args)
 
-    elif (
-        connection.authType
-        == noConfigAuthenticationTypes.NoConfigAuthenticationTypes.OAuth2
-    ):
-        connection.connectionArguments.root["auth"] = OAuth2Authentication()
-        connection.connectionArguments.root["http_scheme"] = "https"
+        elif isinstance(connection.authType, jwtAuth.JwtAuth):
+            TrinoConnection.set_jwt_auth(connection, connection_args)
 
-    return get_connection_args_common(connection)
+        elif hasattr(connection.authType, "azureConfig"):
+            TrinoConnection.set_azure_auth(connection, connection_args)
 
+        elif (
+            connection.authType
+            == noConfigAuthenticationTypes.NoConfigAuthenticationTypes.OAuth2
+        ):
+            TrinoConnection.set_oauth2_auth(connection, connection_args)
 
-def get_connection(connection: TrinoConnection) -> Engine:
-    """
-    Create connection
-    """
-    # here we are creating a copy of connection, because we need to dynamically
-    # add auth params to connectionArguments, which we do no intend to store
-    # in original connection object and in OpenMetadata database
-    from trino.sqlalchemy.dialect import TrinoDialect
+        return connection_args
 
-    TrinoDialect.is_disconnect = _is_disconnect
+    @staticmethod
+    def get_basic_auth_dict(connection: TrinoConnectionConfig) -> dict:
+        """
+        Get the basic auth dictionary for the trino connection
+        """
+        auth_type = cast(basicAuth.BasicAuth, connection.authType)
+        return {
+            "authType": "basic",
+            "username": connection.username,
+            "password": auth_type.password.get_secret_value()
+            if auth_type.password
+            else None,
+        }
+
+    @staticmethod
+    def set_basic_auth(
+        connection: TrinoConnectionConfig, connection_args: ConnectionArguments
+    ) -> None:
+        """
+        Get the basic auth dictionary for the trino connection
+        """
+        assert connection_args.root is not None
+        auth_type = cast(basicAuth.BasicAuth, connection.authType)
 
-    connection_copy = deepcopy(connection)
-    if connection_copy.verify:
-        connection_copy.connectionArguments = (
-            connection_copy.connectionArguments or init_empty_connection_arguments()
+        connection_args.root["auth"] = BasicAuthentication(
+            connection.username,
+            auth_type.password.get_secret_value() if auth_type.password else None,
         )
-        connection.connectionArguments.root["verify"] = {"verify": connection.verify}
-    if hasattr(connection.authType, "azureConfig"):
-        azure_client = AzureClient(connection.authType.azureConfig).create_client()
-        if not connection.authType.azureConfig.scopes:
-            raise ValueError(
-                "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default"
-            )
-        access_token_obj = azure_client.get_token(
-            *connection.authType.azureConfig.scopes.split(",")
+        connection_args.root["http_scheme"] = "https"
+
+    @staticmethod
+    def get_jwt_auth_dict(connection: TrinoConnectionConfig) -> dict:
+        """
+        Get the jwt auth dictionary for the trino connection
+        """
+        auth_type = cast(jwtAuth.JwtAuth, connection.authType)
+
+        return {
+            "authType": "jwt",
+            "jwt": auth_type.jwt.get_secret_value(),
+        }
+
+    @staticmethod
+    def set_jwt_auth(
+        connection: TrinoConnectionConfig, connection_args: ConnectionArguments
+    ) -> None:
+        """
+        Set the jwt auth for the trino connection
+        """
+        assert connection_args.root is not None
+        auth_type = cast(jwtAuth.JwtAuth, connection.authType)
+
+        connection_args.root["auth"] = JWTAuthentication(
+            auth_type.jwt.get_secret_value()
         )
-        if not connection.connectionOptions:
-            connection.connectionOptions = init_empty_connection_options()
-        connection.connectionOptions.root["access_token"] = access_token_obj.token
-    return create_generic_db_connection(
-        connection=connection_copy,
-        get_connection_url_fn=get_connection_url,
-        get_connection_args_fn=get_connection_args,
-    )
-
-
-def test_connection(
-    metadata: OpenMetadata,
-    engine: Engine,
-    service_connection: TrinoConnection,
-    automation_workflow: Optional[AutomationWorkflow] = None,
-    timeout_seconds: Optional[int] = THREE_MIN,
-) -> TestConnectionResult:
-    """
-    Test connection. This can be executed either as part
-    of a metadata workflow or during an Automation Workflow
-    """
-    queries = {
-        "GetDatabases": TRINO_GET_DATABASE,
-    }
-
-    return test_connection_db_schema_sources(
-        metadata=metadata,
-        engine=engine,
-        service_connection=service_connection,
-        automation_workflow=automation_workflow,
-        queries=queries,
-        timeout_seconds=timeout_seconds,
-    )
+        connection_args.root["http_scheme"] = "https"
 
+    @staticmethod
+    def get_azure_auth_dict(connection: TrinoConnectionConfig) -> dict:
+        """
+        Get the azure auth dictionary for the trino connection
+        """
+        return {
+            "authType": "jwt",
+            "jwt": TrinoConnection.get_azure_token(connection),
+        }
 
-# pylint: disable=unused-argument
-def _is_disconnect(self, e, connection, cursor):
-    """is_disconnect method for the Databricks dialect"""
-    if "JWT expired" in str(e):
-        return True
-    return False
+    @staticmethod
+    def set_azure_auth(
+        connection: TrinoConnectionConfig, connection_args: ConnectionArguments
+    ) -> None:
+        """
+        Set the azure auth for the trino connection
+        """
+        assert connection_args.root is not None
+
+        connection_args.root["auth"] = JWTAuthentication(
+            TrinoConnection.get_azure_token(connection)
+        )
+        connection_args.root["http_scheme"] = "https"
+
+    @staticmethod
+    def get_oauth2_auth_dict(connection: TrinoConnectionConfig) -> dict:
+        """
+        Get the oauth2 auth dictionary for the trino connection
+        """
+        return {
+            "authType": "oauth2",
+        }
+
+    @staticmethod
+    def set_oauth2_auth(
+        connection: TrinoConnectionConfig, connection_args: ConnectionArguments
+    ) -> None:
+        """
+        Set the oauth2 auth for the trino connection
+        """
+        assert connection_args.root is not None
+
+        connection_args.root["auth"] = OAuth2Authentication()
+        connection_args.root["http_scheme"] = "https"
+
+    @staticmethod
+    def get_azure_token(connection: TrinoConnectionConfig) -> str:
+        """
+        Get the azure token for the trino connection
+        """
+        auth_type = cast(azureConfig.AzureConfigurationSource, connection.authType)
+        return get_azure_access_token(auth_type)

metadata/ingestion/source/database/trino/service_spec.py

@@ -1,3 +1,4 @@
+from metadata.ingestion.source.database.trino.connection import TrinoConnection
 from metadata.ingestion.source.database.trino.lineage import TrinoLineageSource
 from metadata.ingestion.source.database.trino.metadata import TrinoSource
 from metadata.ingestion.source.database.trino.usage import TrinoUsageSource
@@ -13,4 +14,5 @@ ServiceSpec = DefaultDatabaseSpec(
     usage_source_class=TrinoUsageSource,
     profiler_class=TrinoProfilerInterface,
     sampler_class=TrinoSampler,
+    connection_class=TrinoConnection,
 )

metadata/ingestion/source/database/unitycatalog/connection.py

@@ -17,6 +17,7 @@ from typing import Optional
 
 from databricks.sdk import WorkspaceClient
 from sqlalchemy.engine import Engine
+from sqlalchemy.exc import DatabaseError
 
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
@@ -34,13 +35,13 @@ from metadata.ingestion.connections.builders import (
 )
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.database.unitycatalog.client import UnityCatalogClient
 from metadata.ingestion.source.database.unitycatalog.models import DatabricksTable
 from metadata.ingestion.source.database.unitycatalog.queries import (
     UNITY_CATALOG_GET_ALL_SCHEMA_TAGS,
     UNITY_CATALOG_GET_ALL_TABLE_COLUMNS_TAGS,
     UNITY_CATALOG_GET_ALL_TABLE_TAGS,
     UNITY_CATALOG_GET_CATALOGS_TAGS,
+    UNITY_CATALOG_SQL_STATEMENT_TEST,
 )
 from metadata.utils.constants import THREE_MIN
 from metadata.utils.db_utils import get_host_from_host_port
@@ -93,8 +94,20 @@ def test_connection(
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    client = UnityCatalogClient(service_connection)
     table_obj = DatabricksTable()
+    engine = get_sqlalchemy_connection(service_connection)
+
+    def test_database_query(engine: Engine, statement: str):
+        """
+        Method used to execute the given query and fetch a result
+        to test if user has access to the tables specified
+        in the sql statement
+        """
+        try:
+            connection = engine.connect()
+            connection.execute(statement).fetchone()
+        except DatabaseError as soe:
+            logger.debug(f"Failed to fetch catalogs due to: {soe}")
 
     def get_catalogs(connection: WorkspaceClient, table_obj: DatabricksTable):
         for catalog in connection.catalogs.list():
@@ -149,7 +162,11 @@ def test_connection(
         "GetSchemas": partial(get_schemas, connection, table_obj),
         "GetTables": partial(get_tables, connection, table_obj),
         "GetViews": partial(get_tables, connection, table_obj),
-        "GetQueries": client.test_query_api_access,
+        "GetQueries": partial(
+            test_database_query,
+            engine=engine,
+            statement=UNITY_CATALOG_SQL_STATEMENT_TEST,
+        ),
         "GetTags": partial(get_tags, service_connection, table_obj),
     }
 

metadata/ingestion/source/database/unitycatalog/queries.py

@@ -1,7 +1,19 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 """
-SQL queries for Unity Catalog
+SQL Queries used during ingestion
 """
 
+import textwrap
+
 UNITY_CATALOG_GET_CATALOGS_TAGS = """
 SELECT * FROM `{database}`.information_schema.catalog_tags;
 """
@@ -17,3 +29,27 @@ SELECT * FROM `{database}`.information_schema.table_tags WHERE schema_name = '{s
 UNITY_CATALOG_GET_ALL_TABLE_COLUMNS_TAGS = """
 SELECT * FROM `{database}`.information_schema.column_tags WHERE schema_name = '{schema}';
 """
+
+UNITY_CATALOG_SQL_STATEMENT = textwrap.dedent(
+    """
+    SELECT
+      statement_type AS query_type,
+      statement_text AS query_text,
+      executed_by AS user_name,
+      start_time AS start_time,
+      null AS database_name,
+      null AS schema_name,
+      end_time AS end_time,
+      total_duration_ms/1000 AS duration
+    from system.query.history
+    WHERE statement_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
+    AND statement_text NOT LIKE '/* {{"app": "dbt", %%}} */%%'
+    AND start_time between to_timestamp('{start_time}') and to_timestamp('{end_time}')
+    {filters}
+    LIMIT {result_limit}
+    """
+)
+
+UNITY_CATALOG_SQL_STATEMENT_TEST = """
+ SELECT statement_text from  system.query.history LIMIT 1
+"""

metadata/ingestion/source/database/unitycatalog/query_parser.py

@@ -27,6 +27,9 @@ from metadata.ingestion.source.database.databricks.query_parser import (
 )
 from metadata.ingestion.source.database.query_parser_source import QueryParserSource
 from metadata.ingestion.source.database.unitycatalog.client import UnityCatalogClient
+from metadata.ingestion.source.database.unitycatalog.connection import (
+    get_sqlalchemy_connection,
+)
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -55,6 +58,7 @@ class UnityCatalogQueryParserSource(
     def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
         self._init_super(config=config, metadata=metadata)
         self.client = UnityCatalogClient(self.service_connection)
+        self.sql_client = get_sqlalchemy_connection(self.service_connection)
 
     @classmethod
     def create(

metadata/ingestion/source/database/unitycatalog/usage.py

@@ -11,12 +11,10 @@
 """
 unity catalog usage module
 """
-import traceback
-from datetime import datetime
-from typing import Iterable
 
-from metadata.generated.schema.type.basic import DateTime
-from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery
+from metadata.ingestion.source.database.unitycatalog.queries import (
+    UNITY_CATALOG_SQL_STATEMENT,
+)
 from metadata.ingestion.source.database.unitycatalog.query_parser import (
     UnityCatalogQueryParserSource,
 )
@@ -35,36 +33,24 @@ class UnitycatalogUsageSource(UnityCatalogQueryParserSource, UsageSource):
     the same API for fetching Usage Queries
     """
 
-    def yield_table_queries(self) -> Iterable[TableQuery]:
+    sql_stmt = UNITY_CATALOG_SQL_STATEMENT
+
+    filters = """
+        AND statement_type NOT IN ('SHOW', 'DESCRIBE', 'USE')
+    """
+
+    def get_engine(self):
+        yield self.sql_client
+
+    def get_sql_statement(self, start_time, end_time):
         """
-        Method to yield TableQueries
+        returns sql statement to fetch query logs.
+
+        Override if we have specific parameters
         """
-        queries = []
-        data = self.client.list_query_history(
-            start_date=self.start,
-            end_date=self.end,
+        return self.sql_stmt.format(
+            start_time=start_time,
+            end_time=end_time,
+            filters=self.get_filters(),
+            result_limit=self.source_config.resultLimit,
         )
-        for row in data or []:
-            try:
-                if self.client.is_query_valid(row):
-                    queries.append(
-                        TableQuery(
-                            dialect=self.dialect.value,
-                            query=row.get("query_text"),
-                            userName=row.get("user_name"),
-                            startTime=str(row.get("query_start_time_ms")),
-                            endTime=str(row.get("execution_end_time_ms")),
-                            analysisDate=DateTime(datetime.now()),
-                            serviceName=self.config.serviceName,
-                            duration=row.get("duration")
-                            if row.get("duration")
-                            else None,
-                        )
-                    )
-            except Exception as err:
-                logger.debug(traceback.format_exc())
-                logger.warning(
-                    f"Failed to process query {row.get('query_text')} due to: {err}"
-                )
-
-        yield TableQueries(queries=queries)