openmetadata-ingestion 1.6.7.1__py3-none-any.whl → 1.7.0.0rc1__py3-none-any.whl

metadata/ingestion/source/database/sample_data.py

@@ -709,6 +709,7 @@ class SampleDataSource(
         db = CreateDatabaseRequest(
             name=self.mysql_database["name"],
             service=self.mysql_database_service.fullyQualifiedName,
+            sourceUrl=self.mysql_database.get("sourceUrl"),
         )
 
         yield Either(right=db)
@@ -726,6 +727,7 @@ class SampleDataSource(
         schema = CreateDatabaseSchemaRequest(
             name=self.mysql_database_schema["name"],
             database=database_object.fullyQualifiedName,
+            sourceUrl=self.mysql_database_schema.get("sourceUrl"),
         )
         yield Either(right=schema)
 
@@ -749,6 +751,7 @@ class SampleDataSource(
                 databaseSchema=database_schema_object.fullyQualifiedName,
                 tableConstraints=table.get("tableConstraints"),
                 tableType=table["tableType"],
+                sourceUrl=table.get("sourceUrl"),
             )
             yield Either(right=table_request)
 
@@ -759,6 +762,7 @@ class SampleDataSource(
             name=self.glue_database["name"],
             description=self.glue_database["description"],
             service=self.glue_database_service.fullyQualifiedName,
+            sourceUrl=self.glue_database.get("sourceUrl"),
         )
 
         yield Either(right=db)
@@ -777,6 +781,7 @@ class SampleDataSource(
             name=self.glue_database_schema["name"],
             description=self.glue_database_schema["description"],
             database=database_object.fullyQualifiedName,
+            sourceUrl=self.glue_database_schema.get("sourceUrl"),
         )
         yield Either(right=schema)
 
@@ -800,6 +805,7 @@ class SampleDataSource(
                 databaseSchema=database_schema_object.fullyQualifiedName,
                 tableConstraints=table.get("tableConstraints"),
                 tableType=table["tableType"],
+                sourceUrl=table.get("sourceUrl"),
             )
             yield Either(right=table_request)
 
@@ -823,6 +829,7 @@ class SampleDataSource(
                 databaseSchema=database_schema_object.fullyQualifiedName,
                 tableConstraints=table.get("tableConstraints"),
                 tableType=table["tableType"],
+                sourceUrl=table.get("sourceUrl"),
             )
             yield Either(right=table_request)
 
@@ -853,6 +860,7 @@ class SampleDataSource(
             name=self.database_schema["name"],
             description=self.database_schema["description"],
             database=database_object.fullyQualifiedName,
+            sourceUrl=self.database_schema.get("sourceUrl"),
         )
         yield Either(right=schema)
 
@@ -881,6 +889,7 @@ class SampleDataSource(
                 tableConstraints=table.get("tableConstraints"),
                 tags=table["tags"],
                 schemaDefinition=table.get("schemaDefinition"),
+                sourceUrl=table.get("sourceUrl"),
             )
 
             yield Either(right=table_and_db)
@@ -953,6 +962,7 @@ class SampleDataSource(
             name=self.database_schema["name"],
             description=self.database_schema["description"],
             database=database_object.fullyQualifiedName,
+            sourceUrl=self.database_schema.get("sourceUrl"),
         )
         yield Either(right=schema)
 
@@ -980,6 +990,7 @@ class SampleDataSource(
                 ),
                 databaseSchema=database_schema_object.fullyQualifiedName,
                 tags=stored_procedure["tags"],
+                sourceUrl=stored_procedure.get("sourceUrl"),
             )
 
             yield Either(right=stored_procedure)

metadata/ingestion/source/database/sample_usage.py

@@ -92,6 +92,8 @@ class SampleUsageSource(UsageSource):
                     databaseName="ecommerce_db",
                     serviceName=self.config.serviceName,
                     databaseSchema="shopify",
+                    cost=row.get("cost"),
+                    userName=row.get("user"),
                 )
                 for row in self.query_logs
             ]

metadata/ingestion/source/database/saphana/lineage.py

@@ -159,5 +159,5 @@ class SaphanaLineageSource(Source):
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        result = test_connection_fn(self.engine)
+        result = test_connection_fn(self.metadata, self.engine, self.service_connection)
         raise_test_connection_exception(result)

metadata/ingestion/source/database/singlestore/service_spec.py

@@ -1,3 +1,6 @@
+from metadata.ingestion.source.database.singlestore.lineage import (
+    SinglestoreLineageSource,
+)
 from metadata.ingestion.source.database.singlestore.metadata import SinglestoreSource
 from metadata.profiler.interface.sqlalchemy.single_store.profiler_interface import (
     SingleStoreProfilerInterface,
@@ -5,5 +8,7 @@ from metadata.profiler.interface.sqlalchemy.single_store.profiler_interface impo
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
 ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=SinglestoreSource, profiler_class=SingleStoreProfilerInterface
+    metadata_source_class=SinglestoreSource,
+    profiler_class=SingleStoreProfilerInterface,
+    lineage_source_class=SinglestoreLineageSource,
 )

metadata/ingestion/source/database/snowflake/connection.py

@@ -47,6 +47,7 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_GET_DATABASES,
     SNOWFLAKE_TEST_FETCH_TAG,
     SNOWFLAKE_TEST_GET_QUERIES,
+    SNOWFLAKE_TEST_GET_STREAMS,
     SNOWFLAKE_TEST_GET_TABLES,
     SNOWFLAKE_TEST_GET_VIEWS,
 )
@@ -192,6 +193,11 @@ def test_connection(
             statement=SNOWFLAKE_TEST_GET_VIEWS,
             engine_wrapper=engine_wrapper,
         ),
+        "GetStreams": partial(
+            test_table_query,
+            statement=SNOWFLAKE_TEST_GET_STREAMS,
+            engine_wrapper=engine_wrapper,
+        ),
         "GetQueries": partial(
             test_query,
             statement=SNOWFLAKE_TEST_GET_QUERIES.format(

metadata/ingestion/source/database/snowflake/constants.py

@@ -0,0 +1,48 @@
+#  Copyright 2025 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Snowflake constants
+"""
+
+from sqlalchemy.sql.sqltypes import BOOLEANTYPE, VARCHAR
+
+DEFAULT_STREAM_COLUMNS = [
+    {
+        "name": "METADATA$ACTION",
+        "type": VARCHAR(length=16777216),
+        "nullable": True,
+        "default": None,
+        "autoincrement": False,
+        "system_data_type": "TEXT(16777216)",
+        "comment": None,
+        "primary_key": False,
+    },
+    {
+        "name": "METADATA$ISUPDATE",
+        "type": BOOLEANTYPE,
+        "nullable": True,
+        "default": None,
+        "autoincrement": False,
+        "system_data_type": "BOOLEAN",
+        "comment": None,
+        "primary_key": False,
+    },
+    {
+        "name": "METADATA$ROW_ID",
+        "type": VARCHAR(length=16777216),
+        "nullable": True,
+        "default": None,
+        "autoincrement": False,
+        "system_data_type": "TEXT(16777216)",
+        "comment": None,
+        "primary_key": False,
+    },
+]

metadata/ingestion/source/database/snowflake/metadata.py

@@ -69,6 +69,9 @@ from metadata.ingestion.source.database.incremental_metadata_extraction import (
     IncrementalConfig,
 )
 from metadata.ingestion.source.database.multi_db_source import MultiDBSource
+from metadata.ingestion.source.database.snowflake.constants import (
+    DEFAULT_STREAM_COLUMNS,
+)
 from metadata.ingestion.source.database.snowflake.models import (
     STORED_PROC_LANGUAGE_MAP,
     SnowflakeStoredProcedure,
@@ -86,6 +89,7 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_GET_ORGANIZATION_NAME,
     SNOWFLAKE_GET_SCHEMA_COMMENTS,
     SNOWFLAKE_GET_STORED_PROCEDURES,
+    SNOWFLAKE_GET_STREAM,
     SNOWFLAKE_LIFE_CYCLE_QUERY,
     SNOWFLAKE_SESSION_TAG_QUERY,
 )
@@ -96,6 +100,9 @@ from metadata.ingestion.source.database.snowflake.utils import (
     get_pk_constraint,
     get_schema_columns,
     get_schema_foreign_keys,
+    get_stream_definition,
+    get_stream_names,
+    get_stream_names_reflection,
     get_table_comment,
     get_table_ddl,
     get_table_names,
@@ -115,6 +122,7 @@ from metadata.utils.tag_utils import get_ometa_tag_and_classification
 ischema_names["VARIANT"] = VARIANT
 ischema_names["GEOGRAPHY"] = create_sqlalchemy_type("GEOGRAPHY")
 ischema_names["GEOMETRY"] = create_sqlalchemy_type("GEOMETRY")
+ischema_names["VECTOR"] = create_sqlalchemy_type("VECTOR")
 
 logger = ingestion_logger()
 
@@ -122,6 +130,7 @@ logger = ingestion_logger()
 SnowflakeDialect._json_deserializer = json.loads  # pylint: disable=protected-access
 SnowflakeDialect.get_table_names = get_table_names
 SnowflakeDialect.get_view_names = get_view_names
+SnowflakeDialect.get_stream_names = get_stream_names
 SnowflakeDialect.get_all_table_comments = get_all_table_comments
 SnowflakeDialect.normalize_name = normalize_names
 SnowflakeDialect.get_table_comment = get_table_comment
@@ -132,6 +141,7 @@ SnowflakeDialect._get_schema_columns = (  # pylint: disable=protected-access
 )
 Inspector.get_table_names = get_table_names_reflection
 Inspector.get_view_names = get_view_names_reflection
+Inspector.get_stream_names = get_stream_names_reflection
 SnowflakeDialect._current_database_schema = (  # pylint: disable=protected-access
     _current_database_schema
 )
@@ -140,6 +150,7 @@ SnowflakeDialect.get_foreign_keys = get_foreign_keys
 SnowflakeDialect.get_columns = get_columns
 Inspector.get_all_table_ddls = get_all_table_ddls
 Inspector.get_table_ddl = get_table_ddl
+Inspector.get_stream_definition = get_stream_definition
 SnowflakeDialect._get_schema_foreign_keys = get_schema_foreign_keys
 
 
@@ -303,9 +314,11 @@ class SnowflakeSource(
 
                 if filter_by_database(
                     self.source_config.databaseFilterPattern,
-                    database_fqn
-                    if self.source_config.useFqnForFiltering
-                    else new_database,
+                    (
+                        database_fqn
+                        if self.source_config.useFqnForFiltering
+                        else new_database
+                    ),
                 ):
                     self.status.filter(database_fqn, "Database Filtered Out")
                     continue
@@ -494,6 +507,33 @@ class SnowflakeSource(
             for table in snowflake_tables.get_not_deleted()
         ]
 
+    def _get_stream_names_and_types(self, schema_name: str) -> List[TableNameAndType]:
+        table_type = TableType.Stream
+
+        snowflake_streams = self.inspector.get_stream_names(
+            schema=schema_name,
+            incremental=self.incremental,
+        )
+
+        self.context.get_global().deleted_tables.extend(
+            [
+                fqn.build(
+                    metadata=self.metadata,
+                    entity_type=Table,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=schema_name,
+                    table_name=stream.name,
+                )
+                for stream in snowflake_streams.get_deleted()
+            ]
+        )
+
+        return [
+            TableNameAndType(name=stream.name, type_=table_type)
+            for stream in snowflake_streams.get_not_deleted()
+        ]
+
     def query_table_names_and_types(
         self, schema_name: str
     ) -> Iterable[TableNameAndType]:
@@ -522,6 +562,9 @@ class SnowflakeSource(
                 )
             )
 
+        if self.service_connection.includeStreams:
+            table_list.extend(self._get_stream_names_and_types(schema_name))
+
         return table_list
 
     def _get_org_name(self) -> Optional[str]:
@@ -749,3 +792,71 @@ class SnowflakeSource(
                 )
         else:
             yield from super().mark_tables_as_deleted()
+
+    def _get_columns_internal(
+        self,
+        schema_name: str,
+        table_name: str,
+        db_name: str,
+        inspector: Inspector,
+        table_type: TableType = None,
+    ):
+        """
+        Get columns of table/view/stream
+        """
+        if table_type == TableType.Stream:
+            cursor = self.connection.execute(
+                SNOWFLAKE_GET_STREAM.format(stream_name=table_name, schema=schema_name)
+            )
+            try:
+                result = cursor.fetchone()
+                if result:
+                    table_name = result[6].split(".")[-1]
+            except Exception:
+                pass
+
+        columns = inspector.get_columns(
+            table_name, schema_name, table_type=table_type, db_name=db_name
+        )
+
+        if table_type == TableType.Stream:
+            columns = [*columns, *DEFAULT_STREAM_COLUMNS]
+
+        return columns
+
+    def get_schema_definition(
+        self,
+        table_type: TableType,
+        table_name: str,
+        schema_name: str,
+        inspector: Inspector,
+    ) -> Optional[str]:
+        """
+        Get the DDL statement, View Definition or Stream Definition for a table
+        """
+        try:
+            schema_definition = None
+            if table_type in (TableType.View, TableType.MaterializedView):
+                schema_definition = inspector.get_view_definition(
+                    table_name, schema_name
+                )
+            elif table_type == TableType.Stream:
+                schema_definition = inspector.get_stream_definition(
+                    self.connection, table_name, schema_name
+                )
+            elif hasattr(inspector, "get_table_ddl") and self.source_config.includeDDL:
+                schema_definition = inspector.get_table_ddl(
+                    self.connection, table_name, schema_name
+                )
+            schema_definition = (
+                str(schema_definition).strip()
+                if schema_definition is not None
+                else None
+            )
+            return schema_definition
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.debug(f"Failed to fetch schema definition for {table_name}: {exc}")
+
+        return None

metadata/ingestion/source/database/snowflake/models.py

@@ -85,8 +85,8 @@ class SnowflakeStoredProcedure(BaseModel):
 
 
 class SnowflakeTable(BaseModel):
-    """Models the items returned from the Table and View Queries used to get the entities to process.
-    :name: Holds the table/view name.
+    """Models the items returned from the Table, View and Stream Queries used to get the entities to process.
+    :name: Holds the table/view/stream name.
     :deleted: Holds either a datetime if the table was deleted or None.
     """
 
@@ -95,7 +95,7 @@ class SnowflakeTable(BaseModel):
 
 
 class SnowflakeTableList(BaseModel):
-    """Understands how to return the deleted and not deleted tables/views from a given list."""
+    """Understands how to return the deleted and not deleted tables/views/streams from a given list."""
 
     tables: List[SnowflakeTable]
 

metadata/ingestion/source/database/snowflake/queries.py

@@ -24,7 +24,8 @@ SNOWFLAKE_SQL_STATEMENT = textwrap.dedent(
       schema_name "schema_name",
       start_time "start_time",
       end_time "end_time",
-      total_elapsed_time "duration"
+      total_elapsed_time "duration",
+      CREDITS_USED_CLOUD_SERVICES * {credit_cost} as "cost"
     from {account_usage}.query_history
     WHERE query_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
     AND query_text NOT LIKE '/* {{"app": "dbt", %%}} */%%'
@@ -142,6 +143,18 @@ from (
 where ROW_NUMBER = 1
 """
 
+SNOWFLAKE_GET_STREAM_NAMES = """
+SHOW STREAMS IN SCHEMA "{schema}"
+"""
+
+SNOWFLAKE_INCREMENTAL_GET_STREAM_NAMES = """
+SHOW STREAMS IN SCHEMA "{schema}"
+"""
+
+SNOWFLAKE_GET_STREAM = """
+SHOW STREAMS LIKE '{stream_name}' IN SCHEMA "{schema}"
+"""
+
 SNOWFLAKE_GET_TRANSIENT_NAMES = """
 select TABLE_NAME, NULL from information_schema.tables
 where TABLE_SCHEMA = '{schema}'
@@ -249,6 +262,10 @@ SNOWFLAKE_TEST_GET_VIEWS = """
 SELECT TABLE_NAME FROM "{database_name}".information_schema.views LIMIT 1
 """
 
+SNOWFLAKE_TEST_GET_STREAMS = """
+SHOW STREAMS IN DATABASE "{database_name}"
+"""
+
 SNOWFLAKE_GET_DATABASES = "SHOW DATABASES"
 
 
@@ -388,6 +405,15 @@ ORDER BY PROCEDURE_START_TIME DESC
 SNOWFLAKE_GET_TABLE_DDL = """
 SELECT GET_DDL('TABLE','{table_name}') AS \"text\"
 """
+
+SNOWFLAKE_GET_VIEW_DEFINITION = """
+SELECT GET_DDL('VIEW','{view_name}') AS \"text\"
+"""
+
+SNOWFLAKE_GET_STREAM_DEFINITION = """
+SELECT GET_DDL('STREAM','{stream_name}') AS \"text\"
+"""
+
 SNOWFLAKE_QUERY_LOG_QUERY = """
     SELECT
         QUERY_ID,

metadata/ingestion/source/database/snowflake/query_parser.py

@@ -61,6 +61,8 @@ class SnowflakeQueryParserSource(QueryParserSource, ABC):
             result_limit=self.config.sourceConfig.config.resultLimit,
             filters=self.get_filters(),
             account_usage=self.service_connection.accountUsageSchema,
+            credit_cost=self.service_connection.creditCost
+            * self.service_connection.creditCost,
         )
 
     def check_life_cycle_query(