openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/source/database/saphana/lineage.py

@@ -37,6 +37,7 @@ from metadata.ingestion.source.database.saphana.cdata_parser import (
 )
 from metadata.ingestion.source.database.saphana.models import SapHanaLineageModel
 from metadata.ingestion.source.database.saphana.queries import SAPHANA_LINEAGE
+from metadata.utils.filters import filter_by_table
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.ssl_manager import get_ssl_connection
 
@@ -108,6 +109,17 @@ class SaphanaLineageSource(Source):
                 try:
                     lineage_model = SapHanaLineageModel.validate(dict(row))
 
+                    if filter_by_table(
+                        self.source_config.tableFilterPattern,
+                        lineage_model.name,
+                    ):
+                        self.status.filter(
+                            lineage_model.name,
+                            "View Object Filtered Out",
+                        )
+                        continue
+
+                    logger.debug(f"Processing lineage for view: {lineage_model.name}")
                     yield from self.parse_cdata(
                         metadata=self.metadata, lineage_model=lineage_model
                     )
@@ -138,17 +150,18 @@ class SaphanaLineageSource(Source):
             if to_entity:
                 yield from parsed_lineage.to_request(
                     metadata=metadata,
+                    engine=self.engine,
                     service_name=self.config.serviceName,
                     to_entity=to_entity,
                 )
         except Exception as exc:
             error = (
                 f"Error parsing CDATA XML for {lineage_model.object_suffix} at "
-                + f"{lineage_model.package_id}/{lineage_model.object_name} due to [{exc}]"
+                + f"{lineage_model.name} due to [{exc}]"
             )
             self.status.failed(
                 error=StackTraceError(
-                    name=lineage_model.object_name,
+                    name=lineage_model.name,
                     error=error,
                     stackTrace=traceback.format_exc(),
                 )

metadata/ingestion/source/database/saphana/queries.py

@@ -21,3 +21,10 @@ SELECT
 FROM _SYS_REPO.ACTIVE_OBJECT
 WHERE OBJECT_SUFFIX IN ('analyticview', 'attributeview', 'calculationview');
 """
+
+SAPHANA_SCHEMA_MAPPING = """
+SELECT
+  PHYSICAL_SCHEMA
+FROM _SYS_BI.M_SCHEMA_MAPPING
+WHERE AUTHORING_SCHEMA = '{authoring_schema}';
+"""

metadata/ingestion/source/database/snowflake/connection.py

@@ -53,6 +53,7 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_TEST_GET_VIEWS,
 )
 from metadata.utils.constants import THREE_MIN
+from metadata.utils.filters import filter_by_database
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -72,6 +73,12 @@ def _init_database(engine_wrapper: SnowflakeEngineWrapper):
         if not engine_wrapper.database_name:
             databases = engine_wrapper.engine.execute(SNOWFLAKE_GET_DATABASES)
             for database in databases:
+                if filter_by_database(
+                    engine_wrapper.service_connection.databaseFilterPattern,
+                    database.name,
+                ):
+                    continue
+
                 engine_wrapper.database_name = database.name
                 break
     else:

metadata/ingestion/source/database/snowflake/constants.py

@@ -14,6 +14,9 @@ Snowflake constants
 
 from sqlalchemy.sql.sqltypes import BOOLEANTYPE, VARCHAR
 
+from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureType
+from metadata.generated.schema.entity.data.table import TableType
+
 DEFAULT_STREAM_COLUMNS = [
     {
         "name": "METADATA$ACTION",
@@ -46,3 +49,21 @@ DEFAULT_STREAM_COLUMNS = [
         "primary_key": False,
     },
 ]
+
+SNOWFLAKE_TAG_DESCRIPTION = "SNOWFLAKE TAG VALUE"
+SNOWFLAKE_CLASSIFICATION_DESCRIPTION = "SNOWFLAKE TAG NAME"
+
+TABLE_TYPE_URL_MAP = {
+    TableType.Regular: "table",
+    TableType.Transient: "table",
+    TableType.Dynamic: "dynamic-table",
+    TableType.External: "external-table",
+    TableType.View: "view",
+    TableType.MaterializedView: "materialized-view",
+    TableType.Stream: "stream",
+}
+
+PROCEDURE_TYPE_URL_MAP = {
+    StoredProcedureType.StoredProcedure: "procedure",
+    StoredProcedureType.UDF: "user-function",
+}

metadata/ingestion/source/database/snowflake/lineage.py

@@ -13,7 +13,7 @@ Snowflake lineage module
 """
 
 import traceback
-from typing import Dict, Iterator, List
+from typing import Iterator
 
 from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.source.database.lineage_source import LineageSource
@@ -22,10 +22,10 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_SQL_STATEMENT,
 )
 from metadata.ingestion.source.database.snowflake.query_parser import (
+    SNOWFLAKE_QUERY_BATCH_SIZE,
     SnowflakeQueryParserSource,
 )
 from metadata.ingestion.source.database.stored_procedures_mixin import (
-    QueryByProcedure,
     StoredProcedureLineageMixin,
 )
 from metadata.utils.helpers import get_start_and_end
@@ -55,21 +55,17 @@ class SnowflakeLineageSource(
 
     stored_procedure_query = SNOWFLAKE_GET_STORED_PROCEDURE_QUERIES
 
-    def get_stored_procedure_queries_dict(self) -> Dict[str, List[QueryByProcedure]]:
+    def get_stored_procedure_sql_statement(self) -> str:
         """
-        Return the dictionary associating stored procedures to the
-        queries they triggered
+        Return the SQL statement to get the stored procedure queries
         """
         start, _ = get_start_and_end(self.source_config.queryLogDuration)
         query = self.stored_procedure_query.format(
             start_date=start,
             account_usage=self.service_connection.accountUsageSchema,
         )
-        queries_dict = self.procedure_queries_dict(
-            query=query,
-        )
 
-        return queries_dict
+        return query
 
     def yield_table_query(self) -> Iterator[TableQuery]:
         """
@@ -77,26 +73,48 @@ class SnowflakeLineageSource(
         yield a TableQuery with query parsing info
         """
         for engine in self.get_engine():
-            rows = []
-            with engine.connect() as conn:
-                rows = conn.execution_options(
-                    stream_results=True, max_row_buffer=100
-                ).execute(
-                    self.get_sql_statement(start_time=self.start, end_time=self.end)
+            offset = 0
+            total_fetched = 0
+            max_results = self.source_config.resultLimit
+            while total_fetched < max_results:
+                batch_size = min(
+                    SNOWFLAKE_QUERY_BATCH_SIZE, max_results - total_fetched
                 )
-            # exit from active connection after fetching rows & during
-            # further process of `yield_query_lineage`
-            for row in rows:
-                query_dict = dict(row)
-                query_dict.update({k.lower(): v for k, v in query_dict.items()})
-                try:
-                    yield TableQuery(
-                        dialect=self.dialect.value,
-                        query=query_dict["query_text"],
-                        databaseName=self.get_database_name(query_dict),
-                        serviceName=self.config.serviceName,
-                        databaseSchema=self.get_schema_name(query_dict),
+                rows = []
+                row_count = 0
+                with engine.connect() as conn:
+                    rows = conn.execution_options(
+                        stream_results=True, max_row_buffer=100
+                    ).execute(
+                        self.get_sql_statement(
+                            start_time=self.start,
+                            end_time=self.end,
+                            offset=offset,
+                            limit=batch_size,
+                        )
                     )
-                except Exception as exc:
-                    logger.debug(traceback.format_exc())
-                    logger.warning(f"Error processing query_dict {query_dict}: {exc}")
+                for row in rows:
+                    query_dict = dict(row)
+                    query_dict.update({k.lower(): v for k, v in query_dict.items()})
+                    row_count += 1
+                    try:
+                        yield TableQuery(
+                            dialect=self.dialect.value,
+                            query=query_dict["query_text"],
+                            databaseName=self.get_database_name(query_dict),
+                            serviceName=self.config.serviceName,
+                            databaseSchema=self.get_schema_name(query_dict),
+                        )
+                    except Exception as exc:
+                        logger.debug(traceback.format_exc())
+                        logger.warning(
+                            f"Error processing query_dict {query_dict}: {exc}"
+                        )
+                total_fetched += row_count
+                if row_count < batch_size:
+                    break
+                offset += batch_size
+                logger.info(
+                    f"Fetching next page with offset {offset} (fetched {total_fetched}/{max_results}) "
+                    f"for lineage queries"
+                )

metadata/ingestion/source/database/snowflake/metadata.py

@@ -20,6 +20,7 @@ import sqlalchemy.types as sqltypes
 import sqlparse
 from snowflake.sqlalchemy.custom_types import VARIANT, StructuredType
 from snowflake.sqlalchemy.snowdialect import SnowflakeDialect, ischema_names
+from sqlalchemy import exc as sa_exc
 from sqlalchemy.engine.reflection import Inspector
 from sqlparse.sql import Function, Identifier, Token
 
@@ -54,6 +55,7 @@ from metadata.generated.schema.type.basic import (
     SourceUrl,
 )
 from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
+from metadata.generated.schema.type.tagLabel import TagLabel
 from metadata.ingestion.api.delete import delete_entity_by_name
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
@@ -73,6 +75,10 @@ from metadata.ingestion.source.database.incremental_metadata_extraction import (
 from metadata.ingestion.source.database.multi_db_source import MultiDBSource
 from metadata.ingestion.source.database.snowflake.constants import (
     DEFAULT_STREAM_COLUMNS,
+    PROCEDURE_TYPE_URL_MAP,
+    SNOWFLAKE_CLASSIFICATION_DESCRIPTION,
+    SNOWFLAKE_TAG_DESCRIPTION,
+    TABLE_TYPE_URL_MAP,
 )
 from metadata.ingestion.source.database.snowflake.models import (
     STORED_PROC_LANGUAGE_MAP,
@@ -81,7 +87,8 @@ from metadata.ingestion.source.database.snowflake.models import (
 from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_DESC_FUNCTION,
     SNOWFLAKE_DESC_STORED_PROCEDURE,
-    SNOWFLAKE_FETCH_ALL_TAGS,
+    SNOWFLAKE_FETCH_SCHEMA_TAGS,
+    SNOWFLAKE_FETCH_TABLE_TAGS,
     SNOWFLAKE_GET_CLUSTER_KEY,
     SNOWFLAKE_GET_CURRENT_ACCOUNT,
     SNOWFLAKE_GET_DATABASE_COMMENTS,
@@ -123,7 +130,7 @@ from metadata.utils.sqlalchemy_utils import (
     get_all_table_ddls,
     get_all_view_definitions,
 )
-from metadata.utils.tag_utils import get_ometa_tag_and_classification
+from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_label
 
 
 class MAP(StructuredType):
@@ -203,6 +210,7 @@ class SnowflakeSource(
         self.schema_desc_map = {}
         self.database_desc_map = {}
         self.external_location_map = {}
+        self.schema_tags_map = {}
 
         self._account: Optional[str] = None
         self._org_name: Optional[str] = None
@@ -299,6 +307,32 @@ class SnowflakeSource(
             for row in results
         }
 
+    def set_schema_tags_map(self, database_name: str) -> None:
+        """Fetch and store all schema-level tags for the current database"""
+        self.schema_tags_map.clear()
+        if not self.source_config.includeTags:
+            return
+
+        try:
+            results = self.engine.execute(
+                SNOWFLAKE_FETCH_SCHEMA_TAGS.format(
+                    database_name=database_name,
+                    account_usage=self.service_connection.accountUsageSchema,
+                )
+            ).all()
+
+            for row in results:
+                schema_name = row.SCHEMA_NAME
+                if schema_name not in self.schema_tags_map:
+                    self.schema_tags_map[schema_name] = []
+                self.schema_tags_map[schema_name].append(
+                    {"tag_name": row.TAG_NAME, "tag_value": row.TAG_VALUE}
+                )
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch schema tags: {exc}")
+
     def get_schema_description(self, schema_name: str) -> Optional[str]:
         """
         Method to fetch the schema description
@@ -329,6 +363,7 @@ class SnowflakeSource(
             self.set_schema_description_map()
             self.set_database_description_map()
             self.set_external_location_map(configured_db)
+            self.set_schema_tags_map(configured_db)
             yield configured_db
         else:
             for new_database in self.get_database_names_raw():
@@ -357,6 +392,7 @@ class SnowflakeSource(
                     self.set_schema_description_map()
                     self.set_database_description_map()
                     self.set_external_location_map(new_database)
+                    self.set_schema_tags_map(new_database)
                     yield new_database
                 except Exception as exc:
                     logger.debug(traceback.format_exc())
@@ -451,11 +487,14 @@ class SnowflakeSource(
     def yield_tag(
         self, schema_name: str
     ) -> Iterable[Either[OMetaTagAndClassification]]:
+        """
+        Yield tags for tables/columns and schemas.
+        """
         if self.source_config.includeTags:
             result = []
             try:
                 result = self.connection.execute(
-                    SNOWFLAKE_FETCH_ALL_TAGS.format(
+                    SNOWFLAKE_FETCH_TABLE_TAGS.format(
                         database_name=self.context.get().database,
                         schema_name=schema_name,
                         account_usage=self.service_connection.accountUsageSchema,
@@ -469,20 +508,15 @@ class SnowflakeSource(
                         f"Error fetching tags {exc}. Trying with quoted names"
                     )
                     result = self.connection.execute(
-                        SNOWFLAKE_FETCH_ALL_TAGS.format(
+                        SNOWFLAKE_FETCH_TABLE_TAGS.format(
                             database_name=f'"{self.context.get().database}"',
                             schema_name=f'"{self.context.get().database_schema}"',
                             account_usage=self.service_connection.accountUsageSchema,
                         )
                     )
                 except Exception as inner_exc:
-                    yield Either(
-                        left=StackTraceError(
-                            name="Tags and Classifications",
-                            error=f"Failed to fetch tags due to [{inner_exc}]",
-                            stackTrace=traceback.format_exc(),
-                        )
-                    )
+                    logger.debug(traceback.format_exc())
+                    logger.error(f"Failed to fetch tags due to [{inner_exc}]")
 
             for res in result:
                 row = list(res)
@@ -495,12 +529,32 @@ class SnowflakeSource(
                     ),
                     tags=[row[1]],
                     classification_name=row[0],
-                    tag_description="SNOWFLAKE TAG VALUE",
-                    classification_description="SNOWFLAKE TAG NAME",
+                    tag_description=SNOWFLAKE_TAG_DESCRIPTION,
+                    classification_description=SNOWFLAKE_CLASSIFICATION_DESCRIPTION,
                     metadata=self.metadata,
                     system_tags=True,
                 )
 
+            # Yield schema-level tags
+            if schema_name in self.schema_tags_map:
+                schema_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DatabaseSchema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=schema_name,
+                )
+                for tag_info in self.schema_tags_map[schema_name]:
+                    yield from get_ometa_tag_and_classification(
+                        tag_fqn=FullyQualifiedEntityName(schema_fqn),
+                        tags=[tag_info["tag_value"]],
+                        classification_name=tag_info["tag_name"],
+                        tag_description=SNOWFLAKE_TAG_DESCRIPTION,
+                        classification_description=SNOWFLAKE_CLASSIFICATION_DESCRIPTION,
+                        metadata=self.metadata,
+                        system_tags=True,
+                    )
+
     def _get_table_names_and_types(
         self, schema_name: str, table_type: TableType = TableType.Regular
     ) -> List[TableNameAndType]:
@@ -621,7 +675,7 @@ class SnowflakeSource(
         self, database_name: Optional[str] = None, schema_name: Optional[str] = None
     ) -> str:
         url = (
-            f"https://app.snowflake.com/{self.org_name.lower()}"
+            f"https://{self.service_connection.snowflakeSourceHost}/{self.org_name.lower()}"
             f"/{self.account.lower()}/#/data/databases/{database_name}"
         )
         if schema_name:
@@ -637,11 +691,11 @@ class SnowflakeSource(
         table_type: Optional[TableType] = None,
     ) -> Optional[str]:
         """
-        Method to get the source url for snowflake
+        Method to get the source url for snowflake tables
         """
         try:
             if self.account and self.org_name:
-                tab_type = "view" if table_type == TableType.View else "table"
+                tab_type = TABLE_TYPE_URL_MAP.get(table_type, "table")
                 url = self._get_source_url_root(
                     database_name=database_name, schema_name=schema_name
                 )
@@ -653,6 +707,41 @@ class SnowflakeSource(
             logger.error(f"Unable to get source url: {exc}")
         return None
 
+    def get_procedure_source_url(
+        self,
+        database_name: Optional[str] = None,
+        schema_name: Optional[str] = None,
+        procedure_name: Optional[str] = None,
+        procedure_signature: Optional[str] = None,
+        procedure_type: Optional[str] = None,
+    ) -> Optional[str]:
+        """
+        Method to get the source url for snowflake stored procedures
+        """
+        try:
+            if self.account and self.org_name:
+                url = self._get_source_url_root(
+                    database_name=database_name, schema_name=schema_name
+                )
+
+                # Convert string procedure type to enum and get URL mapping
+                proc_type_enum = (
+                    StoredProcedureType(procedure_type)
+                    if procedure_type
+                    else StoredProcedureType.StoredProcedure
+                )
+                tab_type = PROCEDURE_TYPE_URL_MAP.get(proc_type_enum, "procedure")
+
+                if procedure_name:
+                    full_name = f"{procedure_name}{procedure_signature or ''}"
+                    url = f"{url}/{tab_type}/{full_name}"
+
+                return url
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.error(f"Unable to get procedure source url: {exc}")
+        return None
+
     def _get_view_names_and_types(
         self, schema_name: str, materialized_views: bool = False
     ) -> List[TableNameAndType]:
@@ -793,12 +882,13 @@ class SnowflakeSource(
                     schema_name=self.context.get().database_schema,
                 ),
                 sourceUrl=SourceUrl(
-                    self._get_source_url_root(
+                    self.get_procedure_source_url(
                         database_name=self.context.get().database,
                         schema_name=self.context.get().database_schema,
+                        procedure_name=stored_procedure.name,
+                        procedure_signature=stored_procedure.signature,
+                        procedure_type=stored_procedure.procedure_type,
                     )
-                    + f"/procedure/{stored_procedure.name}"
-                    + f"{stored_procedure.signature if stored_procedure.signature else ''}"
                 ),
             )
             yield Either(right=stored_procedure_request)
@@ -847,6 +937,8 @@ class SnowflakeSource(
         """
         Get columns of table/view/stream
         """
+        # For streams, we will use source table/view's columns
+        # since stream does not define columns separately in Snowflake
         if table_type == TableType.Stream:
             cursor = self.connection.execute(
                 SNOWFLAKE_GET_STREAM.format(stream_name=table_name, schema=schema_name)
@@ -855,12 +947,28 @@ class SnowflakeSource(
                 result = cursor.fetchone()
                 if result:
                     table_name = result[6].split(".")[-1]
+                    # Can't fetch source of stream is source is dropped or no priviledge
+                    if table_name == "No privilege or table dropped":
+                        logger.warning(
+                            f"Couldn't fetch columns of stream [{result and result[1]}] "
+                            f"(schema: '{schema_name}', db: '{db_name}') due to error on"
+                            f" source: [{table_name}]. Result: {result}"
+                        )
+                        return []
             except Exception:
                 pass
 
-        columns = inspector.get_columns(
-            table_name, schema_name, table_type=table_type, db_name=db_name
-        )
+        try:
+            columns = inspector.get_columns(
+                table_name, schema_name, table_type=table_type, db_name=db_name
+            )
+        except sa_exc.NoSuchTableError:
+            logger.warning(
+                f"Table [{table_name}] (schema: '{schema_name}', db: '{db_name}') not found."
+                " Unable to fetch columns. Please check if the configured Snowflake user has"
+                " necessary grants on this table."
+            )
+            return []
 
         if table_type == TableType.Stream:
             columns = [*columns, *DEFAULT_STREAM_COLUMNS]
@@ -945,3 +1053,49 @@ class SnowflakeSource(
         logger.debug(
             f"Processing ownership is not supported for {self.service_connection.type.name}"
         )
+
+    def get_schema_tag_labels(self, schema_name: str) -> Optional[List[TagLabel]]:
+        """
+        Return tags for schema entity including Snowflake schema-level tags.
+        """
+        schema_tags = []
+
+        if schema_name in self.schema_tags_map:
+            for tag_info in self.schema_tags_map[schema_name]:
+                tag_label = get_tag_label(
+                    metadata=self.metadata,
+                    tag_name=tag_info["tag_value"],
+                    classification_name=tag_info["tag_name"],
+                )
+                if tag_label:
+                    schema_tags.append(tag_label)
+
+        # Include parent tags from context
+        parent_tags = super().get_schema_tag_labels(schema_name) or []
+        for tag in parent_tags:
+            if tag not in schema_tags:
+                schema_tags.append(tag)
+
+        return schema_tags if schema_tags else None
+
+    def get_tag_labels(self, table_name: str) -> Optional[List[TagLabel]]:
+        """
+        Override to include schema-level tags inherited by tables.
+        This method combines:
+        1. Tags directly assigned to the table (from parent implementation)
+        2. Tags inherited from the schema level
+        """
+        table_tags = super().get_tag_labels(table_name) or []
+
+        schema_name = self.context.get().database_schema
+        if schema_name and schema_name in self.schema_tags_map:
+            for tag_info in self.schema_tags_map[schema_name]:
+                tag_label = get_tag_label(
+                    metadata=self.metadata,
+                    tag_name=tag_info["tag_value"],
+                    classification_name=tag_info["tag_name"],
+                )
+                if tag_label and tag_label not in table_tags:
+                    table_tags.append(tag_label)
+
+        return table_tags if table_tags else None

metadata/ingestion/source/database/snowflake/queries.py

@@ -31,13 +31,15 @@ SNOWFLAKE_SQL_STATEMENT = textwrap.dedent(
     AND query_text NOT LIKE '/* {{"app": "dbt", %%}} */%%'
     AND start_time between to_timestamp_ltz('{start_time}') and to_timestamp_ltz('{end_time}')
     {filters}
+    ORDER BY start_time
     LIMIT {result_limit}
+    OFFSET {offset}
     """
 )
 
 SNOWFLAKE_SESSION_TAG_QUERY = 'ALTER SESSION SET QUERY_TAG="{query_tag}"'
 
-SNOWFLAKE_FETCH_ALL_TAGS = textwrap.dedent(
+SNOWFLAKE_FETCH_TABLE_TAGS = textwrap.dedent(
     """
     select TAG_NAME, TAG_VALUE, OBJECT_DATABASE, OBJECT_SCHEMA, OBJECT_NAME, COLUMN_NAME
     from {account_usage}.tag_references
@@ -46,6 +48,18 @@ SNOWFLAKE_FETCH_ALL_TAGS = textwrap.dedent(
 """
 )
 
+SNOWFLAKE_FETCH_SCHEMA_TAGS = textwrap.dedent(
+    """
+    select TAG_NAME, TAG_VALUE, OBJECT_NAME as SCHEMA_NAME
+    from {account_usage}.tag_references
+    where OBJECT_DATABASE = '{database_name}'
+      and OBJECT_SCHEMA IS NULL
+      and OBJECT_NAME IS NOT NULL
+      and COLUMN_NAME IS NULL
+      and DOMAIN = 'SCHEMA'
+"""
+)
+
 SNOWFLAKE_GET_EXTERNAL_TABLE_NAMES = """
 select TABLE_NAME, NULL from information_schema.tables
 where TABLE_SCHEMA = '{schema}' AND TABLE_TYPE = 'EXTERNAL TABLE'

metadata/ingestion/source/database/snowflake/query_parser.py

@@ -32,6 +32,7 @@ from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 SNOWFLAKE_ABORTED_CODE = "1969"
+SNOWFLAKE_QUERY_BATCH_SIZE = 1000
 
 
 class SnowflakeQueryParserSource(QueryParserSource, ABC):
@@ -51,18 +52,27 @@ class SnowflakeQueryParserSource(QueryParserSource, ABC):
             )
         return cls(config, metadata)
 
-    def get_sql_statement(self, start_time: datetime, end_time: datetime) -> str:
+    def get_sql_statement(
+        self,
+        start_time: datetime,
+        end_time: datetime,
+        offset: int = 0,
+        limit: int = None,
+    ) -> str:
         """
         returns sql statement to fetch query logs
         """
+        if limit is None:
+            limit = self.config.sourceConfig.config.resultLimit
         return self.sql_stmt.format(
             start_time=start_time,
             end_time=end_time,
-            result_limit=self.config.sourceConfig.config.resultLimit,
+            result_limit=limit,
             filters=self.get_filters(),
             account_usage=self.service_connection.accountUsageSchema,
             credit_cost=self.service_connection.creditCost
             * self.service_connection.creditCost,
+            offset=offset,
         )
 
     def check_life_cycle_query(