openmetadata-ingestion 1.7.0.1__py3-none-any.whl → 1.7.1.1__py3-none-any.whl

metadata/ingestion/source/database/mysql/queries.py

@@ -36,8 +36,37 @@ LIMIT {result_limit};
 """
 )
 
+
+MYSQL_SQL_STATEMENT_SLOW_LOGS = textwrap.dedent(
+    """
+SELECT 
+	NULL `database_name`,
+	sql_text `query_text`,
+	start_time `start_time`,
+    NULL `end_time`,
+	NULL  `duration`,
+	NULL `schema_name`,
+    NULL `query_type`,
+    NULL `user_name`,
+    NULL `aborted`
+FROM mysql.slow_log
+WHERE start_time between '{start_time}' and '{end_time}'
+    AND sql_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
+    AND sql_text NOT LIKE '/* {{"app": "dbt", %%}} */%%'
+    {filters}
+ORDER BY start_time desc
+LIMIT {result_limit};
+"""
+)
+
 MYSQL_TEST_GET_QUERIES = textwrap.dedent(
     """
 SELECT `argument` from mysql.general_log limit 1;
 """
 )
+
+MYSQL_TEST_GET_QUERIES_SLOW_LOGS = textwrap.dedent(
+    """
+SELECT `sql_text` from mysql.slow_log limit 1;
+"""
+)

metadata/ingestion/source/database/mysql/query_parser.py

@@ -12,6 +12,7 @@
 Mysql query parser module
 """
 from abc import ABC
+from datetime import datetime
 from typing import Optional
 
 from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
@@ -22,6 +23,10 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 )
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.mysql.queries import (
+    MYSQL_SQL_STATEMENT,
+    MYSQL_SQL_STATEMENT_SLOW_LOGS,
+)
 from metadata.ingestion.source.database.query_parser_source import QueryParserSource
 
 
@@ -44,3 +49,29 @@ class MysqlQueryParserSource(QueryParserSource, ABC):
                 f"Expected MysqlConnection, but got {connection}"
             )
         return cls(config, metadata)
+
+    def get_sql_statement(self, start_time: datetime, end_time: datetime) -> str:
+        """
+        returns sql statement to fetch query logs.
+
+        Override if we have specific parameters
+        """
+        if self.service_connection.useSlowLogs:
+            self.sql_stmt = MYSQL_SQL_STATEMENT_SLOW_LOGS
+        else:
+            self.sql_stmt = MYSQL_SQL_STATEMENT
+        return self.sql_stmt.format(
+            start_time=start_time,
+            end_time=end_time,
+            filters=self.get_filters(),
+            result_limit=self.source_config.resultLimit,
+        )
+
+    def get_filters(self) -> str:
+        if self.service_connection.useSlowLogs:
+            sql_column = "sql_text"
+        else:
+            sql_column = "argument"
+        if self.source_config.filterCondition:
+            return f"{self.filters.format(sql_column=sql_column)} AND {self.source_config.filterCondition}"
+        return self.filters.format(sql_column=sql_column)

metadata/ingestion/source/database/oracle/queries.py

@@ -31,14 +31,14 @@ ORACLE_ALL_VIEW_DEFINITIONS = textwrap.dedent(
 SELECT
 LOWER(view_name) AS "view_name",
 LOWER(owner) AS "schema",
-DBMS_METADATA.GET_DDL('VIEW', view_name, owner) AS view_def
+text AS view_def
 FROM DBA_VIEWS
 WHERE owner NOT IN ('SYSTEM', 'SYS')
 UNION ALL
 SELECT
 LOWER(mview_name) AS "view_name",
 LOWER(owner) AS "schema",
-DBMS_METADATA.GET_DDL('MATERIALIZED_VIEW', mview_name, owner) AS view_def
+query AS view_def
 FROM DBA_MVIEWS
 WHERE owner NOT IN ('SYSTEM', 'SYS')
 """

metadata/ingestion/source/database/postgres/metadata.py

@@ -71,6 +71,7 @@ from metadata.ingestion.source.database.postgres.utils import (
     get_columns,
     get_etable_owner,
     get_foreign_keys,
+    get_schema_names,
     get_table_comment,
     get_table_owner,
     get_view_definition,
@@ -115,6 +116,7 @@ Inspector.get_table_ddl = get_table_ddl
 Inspector.get_table_owner = get_etable_owner
 
 PGDialect.get_foreign_keys = get_foreign_keys
+PGDialect.get_schema_names = get_schema_names
 
 
 class PostgresSource(CommonDbSourceService, MultiDBSource):
@@ -281,7 +283,7 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
         for row in results:
             try:
                 stored_procedure = PostgresStoredProcedure.model_validate(
-                    dict(row._mapping)
+                    dict(row._mapping)  # pylint: disable=protected-access
                 )
                 yield stored_procedure
             except Exception as exc:

metadata/ingestion/source/database/postgres/queries.py

@@ -196,6 +196,13 @@ POSTGRES_GET_SERVER_VERSION = """
 show server_version_num
 """
 
+# pylint: disable=anomalous-backslash-in-string
+POSTGRES_GET_SCHEMA_NAMES = """
+SELECT nspname FROM pg_namespace
+    WHERE nspname NOT LIKE 'pg\_%'
+    ORDER BY nspname
+"""
+
 POSTGRES_FETCH_FK = """
     SELECT r.conname,
         pg_catalog.pg_get_constraintdef(r.oid, true) as condef,

metadata/ingestion/source/database/postgres/utils.py

@@ -26,6 +26,7 @@ from sqlalchemy.sql import sqltypes
 from metadata.ingestion.source.database.postgres.queries import (
     POSTGRES_COL_IDENTITY,
     POSTGRES_FETCH_FK,
+    POSTGRES_GET_SCHEMA_NAMES,
     POSTGRES_GET_SERVER_VERSION,
     POSTGRES_SQL_COLUMNS,
     POSTGRES_TABLE_COMMENTS,
@@ -43,10 +44,8 @@ logger = utils_logger()
 
 OLD_POSTGRES_VERSION = "130000"
 
-
-def get_etable_owner(
-    self, connection, table_name=None, schema=None
-):  # pylint: disable=unused-argument
+# pylint: disable=unused-argument,too-many-arguments,invalid-name,too-many-locals
+def get_etable_owner(self, connection, table_name=None, schema=None):
     """Return all owners.
 
     :param schema: Optional, retrieve names from a non-default schema.
@@ -67,6 +66,16 @@ def get_etable_owner(
 def get_foreign_keys(
     self, connection, table_name, schema=None, postgresql_ignore_search_path=False, **kw
 ):
+    """
+    Args:
+        connection (_type_): _description_
+        table_name (_type_): _description_
+        schema (_type_, optional): _description_. Defaults to None.
+        postgresql_ignore_search_path (bool, optional): _description_. Defaults to False.
+
+    Returns:
+        _type_: _description_
+    """
     preparer = self.identifier_preparer
     table_oid = self.get_table_oid(
         connection, table_name, schema, info_cache=kw.get("info_cache")
@@ -87,7 +96,7 @@ def get_foreign_keys(
     t = sql.text(POSTGRES_FETCH_FK).columns(
         conname=sqltypes.Unicode, condef=sqltypes.Unicode, con_db_name=sqltypes.Unicode
     )
-    c = connection.execute(t, dict(table=table_oid))
+    c = connection.execute(t, {"table": table_oid})
     fkeys = []
     for conname, condef, conschema, con_db_name in c.fetchall():
         m = re.search(FK_REGEX, condef).groups()
@@ -109,7 +118,7 @@ def get_foreign_keys(
         ) = m
 
         if deferrable is not None:
-            deferrable = True if deferrable == "DEFERRABLE" else False
+            deferrable = deferrable == "DEFERRABLE"
         constrained_columns = tuple(re.split(r"\s*,\s*", constrained_columns))
         constrained_columns = [
             preparer._unquote_identifier(x) for x in constrained_columns
@@ -161,9 +170,7 @@ def get_foreign_keys(
 
 
 @reflection.cache
-def get_table_owner(
-    self, connection, table_name, schema=None, **kw
-):  # pylint: disable=unused-argument
+def get_table_owner(self, connection, table_name, schema=None, **kw):
     return get_table_owner_wrapper(
         self,
         connection=connection,
@@ -174,9 +181,7 @@ def get_table_owner(
 
 
 @reflection.cache
-def get_table_comment(
-    self, connection, table_name, schema=None, **kw
-):  # pylint: disable=unused-argument
+def get_table_comment(self, connection, table_name, schema=None, **kw):
     return get_table_comment_wrapper(
         self,
         connection,
@@ -187,9 +192,7 @@ def get_table_comment(
 
 
 @reflection.cache
-def get_columns(  # pylint: disable=too-many-locals
-    self, connection, table_name, schema=None, **kw
-):
+def get_columns(self, connection, table_name, schema=None, **kw):
     """
     Overriding the dialect method to add raw_data_type in response
     """
@@ -368,7 +371,7 @@ def _handle_array_type(attype):
     )
 
 
-# pylint: disable=too-many-statements,too-many-branches,too-many-locals,too-many-arguments
+# pylint: disable=too-many-statements,too-many-branches
 def get_column_info(
     self,
     name,
@@ -477,9 +480,7 @@ def get_column_info(
 
 
 @reflection.cache
-def get_view_definition(
-    self, connection, table_name, schema=None, **kw
-):  # pylint: disable=unused-argument
+def get_view_definition(self, connection, table_name, schema=None, **kw):
     return get_view_definition_wrapper(
         self,
         connection,
@@ -515,3 +516,11 @@ def get_postgres_time_column_name(engine) -> str:
     ):
         time_column_name = "total_time"
     return time_column_name
+
+
+@reflection.cache
+def get_schema_names(self, connection, **kw):
+    result = connection.execute(
+        sql.text(POSTGRES_GET_SCHEMA_NAMES).columns(nspname=sqltypes.Unicode)
+    )
+    return [name for name, in result]

metadata/ingestion/source/database/snowflake/data_diff/data_diff.py

@@ -0,0 +1,37 @@
+"""Snowflake spec for data diff"""
+
+from typing import Optional, cast
+
+from metadata.data_quality.validations.models import TableParameter
+from metadata.data_quality.validations.runtime_param_setter.base_diff_params_setter import (
+    BaseTableParameter,
+)
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.ingestion.source.database.snowflake.metadata import SnowflakeConnection
+
+
+class SnowflakeTableParameter(BaseTableParameter):
+    """SnowflakeTableParameter class for setting runtime parameters for data diff"""
+
+    def get(
+        self,
+        service: DatabaseService,
+        entity: Table,
+        key_columns,
+        extra_columns,
+        case_sensitive_columns,
+        service_url: Optional[str],
+    ) -> TableParameter:
+        table_param: TableParameter = super().get(
+            service,
+            entity,
+            key_columns,
+            extra_columns,
+            case_sensitive_columns,
+            service_url,
+        )
+        connection_config = cast(SnowflakeConnection, service.connection.config)
+        table_param.privateKey = connection_config.privateKey
+        table_param.passPhrase = connection_config.snowflakePrivatekeyPassphrase
+        return table_param

metadata/ingestion/source/database/snowflake/metadata.py

@@ -169,6 +169,8 @@ class SnowflakeSource(
     Database metadata from Snowflake Source
     """
 
+    service_connection: SnowflakeConnection
+
     def __init__(
         self,
         config,
@@ -492,6 +494,7 @@ class SnowflakeSource(
         snowflake_tables = self.inspector.get_table_names(
             schema=schema_name,
             incremental=self.incremental,
+            account_usage=self.service_connection.accountUsageSchema,
             **table_type_to_params_map[table_type],
         )
 
@@ -640,6 +643,7 @@ class SnowflakeSource(
         snowflake_views = self.inspector.get_view_names(
             schema=schema_name,
             incremental=self.incremental,
+            account_usage=self.service_connection.accountUsageSchema,
             materialized_views=materialized_views,
         )
 
@@ -894,3 +898,13 @@ class SnowflakeSource(
             logger.debug(f"Failed to fetch schema definition for {table_name}: {exc}")
 
         return None
+
+    def get_life_cycle_query(self):
+        """
+        Get the life cycle query
+        """
+        return self.life_cycle_query.format(
+            database_name=self.context.get().database,
+            schema_name=self.context.get().database_schema,
+            account_usage=self.service_connection.accountUsageSchema,
+        )

metadata/ingestion/source/database/snowflake/queries.py

@@ -60,7 +60,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where TABLE_CATALOG = '{database}'
       and TABLE_SCHEMA = '{schema}'
       and TABLE_TYPE = 'EXTERNAL TABLE'
@@ -86,7 +86,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where TABLE_CATALOG = '{database}'
     and TABLE_SCHEMA = '{schema}'
     and TABLE_TYPE = 'BASE TABLE'
@@ -111,7 +111,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where  TABLE_CATALOG = '{database}'
     and TABLE_SCHEMA = '{schema}'
     and TABLE_TYPE = 'VIEW'
@@ -134,7 +134,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where  TABLE_CATALOG = '{database}'
     and TABLE_SCHEMA = '{schema}'
     and TABLE_TYPE = 'MATERIALIZED VIEW'
@@ -171,7 +171,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where TABLE_CATALOG = '{database}'
     and TABLE_SCHEMA = '{schema}'
     and TABLE_TYPE = 'BASE TABLE'
@@ -197,7 +197,7 @@ from (
         ROW_NUMBER() over (
             partition by TABLE_NAME order by LAST_DDL desc
         ) as ROW_NUMBER
-    from snowflake.account_usage.tables
+    from {account_usage}.tables
     where TABLE_CATALOG = '{database}'
     and TABLE_SCHEMA = '{schema}'
     and TABLE_TYPE = 'BASE TABLE'
@@ -297,7 +297,7 @@ SNOWFLAKE_LIFE_CYCLE_QUERY = textwrap.dedent(
 select
 table_name as table_name,
 created as created_at
-from snowflake.account_usage.tables
+from {account_usage}.tables
 where table_schema = '{schema_name}'
 and table_catalog = '{database_name}'
 """
@@ -414,6 +414,10 @@ FROM information_schema.views
 WHERE view_definition is not null
 """
 
+SNOWFLAKE_GET_VIEW_DDL = """
+SELECT GET_DDL('VIEW','{view_name}') AS \"text\"
+"""
+
 SNOWFLAKE_GET_STREAM_DEFINITION = """
 SELECT GET_DDL('STREAM','{stream_name}') AS \"text\"
 """

metadata/ingestion/source/database/snowflake/service_spec.py

@@ -1,6 +1,9 @@
 from metadata.data_quality.interface.sqlalchemy.snowflake.test_suite_interface import (
     SnowflakeTestSuiteInterface,
 )
+from metadata.ingestion.source.database.snowflake.data_diff.data_diff import (
+    SnowflakeTableParameter,
+)
 from metadata.ingestion.source.database.snowflake.lineage import SnowflakeLineageSource
 from metadata.ingestion.source.database.snowflake.metadata import SnowflakeSource
 from metadata.ingestion.source.database.snowflake.profiler.profiler import (
@@ -17,4 +20,5 @@ ServiceSpec = DefaultDatabaseSpec(
     profiler_class=SnowflakeProfiler,
     test_suite_class=SnowflakeTestSuiteInterface,
     sampler_class=SnowflakeSampler,
+    data_diff=SnowflakeTableParameter,
 )

metadata/ingestion/source/database/snowflake/utils.py

@@ -41,6 +41,7 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_GET_STREAM_NAMES,
     SNOWFLAKE_GET_TABLE_DDL,
     SNOWFLAKE_GET_TRANSIENT_NAMES,
+    SNOWFLAKE_GET_VIEW_DDL,
     SNOWFLAKE_GET_VIEW_DEFINITION,
     SNOWFLAKE_GET_VIEW_NAMES,
     SNOWFLAKE_GET_WITHOUT_TRANSIENT_TABLE_NAMES,
@@ -188,7 +189,11 @@ def _get_query_map(
 
 
 def _get_query_parameters(
-    self, connection, schema: str, incremental: Optional[IncrementalConfig]
+    self,
+    connection,
+    schema: str,
+    incremental: Optional[IncrementalConfig],
+    account_usage: Optional[str] = None,
 ):
     """Returns the proper query parameters depending if the extraction is Incremental or Full"""
     parameters = {"schema": fqn.unquote_name(schema)}
@@ -199,6 +204,7 @@ def _get_query_parameters(
             **parameters,
             "date": incremental.start_timestamp,
             "database": database,
+            "account_usage": account_usage or "SNOWFLAKE.ACCOUNT_USAGE",
         }
 
     return parameters
@@ -207,9 +213,12 @@ def _get_query_parameters(
 def get_table_names(self, connection, schema: str, **kw):
     """Return the Table names to process based on the incremental setup."""
     incremental = kw.get("incremental")
+    account_usage = kw.get("account_usage")
 
     queries = _get_query_map(incremental, TABLE_QUERY_MAPS)
-    parameters = _get_query_parameters(self, connection, schema, incremental)
+    parameters = _get_query_parameters(
+        self, connection, schema, incremental, account_usage
+    )
 
     query = queries["default"]
 
@@ -234,9 +243,12 @@ def get_table_names(self, connection, schema: str, **kw):
 
 def get_view_names(self, connection, schema, **kw):
     incremental = kw.get("incremental")
+    account_usage = kw.get("account_usage")
 
     queries = _get_query_map(incremental, VIEW_QUERY_MAPS)
-    parameters = _get_query_parameters(self, connection, schema, incremental)
+    parameters = _get_query_parameters(
+        self, connection, schema, incremental, account_usage
+    )
 
     if kw.get("materialized_views"):
         query = queries["materialized_views"]
@@ -275,13 +287,29 @@ def get_stream_names(self, connection, schema, **kw):
 def get_view_definition(
     self, connection, table_name, schema=None, **kw
 ):  # pylint: disable=unused-argument
-    return get_view_definition_wrapper(
+    view_definition = get_view_definition_wrapper(
         self,
         connection,
         table_name=table_name,
         schema=schema,
         query=SNOWFLAKE_GET_VIEW_DEFINITION,
     )
+    if view_definition:
+        return view_definition
+
+    # If the view definition is not found via optimized query,
+    # we need to get the view definition from the view ddl
+
+    schema = schema or self.default_schema_name
+    view_name = f"{schema}.{table_name}" if schema else table_name
+    cursor = connection.execute(SNOWFLAKE_GET_VIEW_DDL.format(view_name=view_name))
+    try:
+        result = cursor.fetchone()
+        if result:
+            return result[0]
+    except Exception:
+        pass
+    return None
 
 
 @reflection.cache

metadata/ingestion/source/database/vertica/queries.py

@@ -28,20 +28,6 @@ import textwrap
 # v_catalog.comments with v_catalog.columns.
 VERTICA_GET_COLUMNS = textwrap.dedent(
     """
-        WITH column_projection as (
-          SELECT 
-            column_id,
-            proj.projection_name
-        FROM v_catalog.columns col,
-          v_catalog.projections proj
-        where lower(table_name) = '{table}' 
-            AND {schema_condition}
-            AND proj.projection_id in (
-              select 
-                min(projection_id) 
-              from v_catalog.projections sub_proj
-              where col.table_id=sub_proj.anchor_table_id
-        ))
         select
           column_name,
           data_type,
@@ -50,12 +36,11 @@ VERTICA_GET_COLUMNS = textwrap.dedent(
           comment 
         from 
           v_catalog.columns col
-          LEFT JOIN column_projection proj ON proj.column_id = col.column_id
-          LEFT JOIN v_catalog.comments com ON com.object_type = 'COLUMN'
-          AND com.object_name = CONCAT(
-            CONCAT(proj.projection_name, '.'),
-            col.column_name
-          )
+          LEFT JOIN v_catalog.comments cm
+          ON col.table_schema = cm.object_schema
+          AND col.table_name = cm.object_name
+          AND col.column_name = cm.child_object
+          AND cm.object_type = 'COLUMN'
         WHERE 
           lower(table_name) = '{table}'
           AND {schema_condition}

metadata/ingestion/source/pipeline/airbyte/constants.py

@@ -0,0 +1,29 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Constants for Airbyte
+"""
+
+from enum import Enum
+
+
+class AirbyteSource(Enum):
+    MYSQL = "MySQL"
+    POSTGRES = "Postgres"
+    MSSQL = "Microsoft SQL Server (MSSQL)"
+    MONGODB = "MongoDb"
+
+
+class AirbyteDestination(Enum):
+    MYSQL = "MySQL"
+    POSTGRES = "Postgres"
+    MSSQL = "MS SQL Server"