openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.9.2.0__py3-none-any.whl

metadata/ingestion/source/database/saphana/lineage.py

@@ -37,6 +37,7 @@ from metadata.ingestion.source.database.saphana.cdata_parser import (
 )
 from metadata.ingestion.source.database.saphana.models import SapHanaLineageModel
 from metadata.ingestion.source.database.saphana.queries import SAPHANA_LINEAGE
+from metadata.utils.filters import filter_by_table
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.ssl_manager import get_ssl_connection
 
@@ -108,6 +109,16 @@ class SaphanaLineageSource(Source):
                 try:
                     lineage_model = SapHanaLineageModel.validate(dict(row))
 
+                    if filter_by_table(
+                        self.source_config.tableFilterPattern,
+                        lineage_model.object_name,
+                    ):
+                        self.status.filter(
+                            lineage_model.object_name,
+                            "View Object Filtered Out",
+                        )
+                        continue
+
                     yield from self.parse_cdata(
                         metadata=self.metadata, lineage_model=lineage_model
                     )
@@ -138,6 +149,7 @@ class SaphanaLineageSource(Source):
             if to_entity:
                 yield from parsed_lineage.to_request(
                     metadata=metadata,
+                    engine=self.engine,
                     service_name=self.config.serviceName,
                     to_entity=to_entity,
                 )

metadata/ingestion/source/database/saphana/queries.py

@@ -21,3 +21,10 @@ SELECT
 FROM _SYS_REPO.ACTIVE_OBJECT
 WHERE OBJECT_SUFFIX IN ('analyticview', 'attributeview', 'calculationview');
 """
+
+SAPHANA_SCHEMA_MAPPING = """
+SELECT
+  PHYSICAL_SCHEMA
+FROM _SYS_BI.M_SCHEMA_MAPPING
+WHERE AUTHORING_SCHEMA = '{authoring_schema}';
+"""

metadata/ingestion/source/database/snowflake/constants.py

@@ -46,3 +46,6 @@ DEFAULT_STREAM_COLUMNS = [
         "primary_key": False,
     },
 ]
+
+SNOWFLAKE_TAG_DESCRIPTION = "SNOWFLAKE TAG VALUE"
+SNOWFLAKE_CLASSIFICATION_DESCRIPTION = "SNOWFLAKE TAG NAME"

metadata/ingestion/source/database/snowflake/metadata.py

@@ -54,6 +54,7 @@ from metadata.generated.schema.type.basic import (
     SourceUrl,
 )
 from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
+from metadata.generated.schema.type.tagLabel import TagLabel
 from metadata.ingestion.api.delete import delete_entity_by_name
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
@@ -73,6 +74,8 @@ from metadata.ingestion.source.database.incremental_metadata_extraction import (
 from metadata.ingestion.source.database.multi_db_source import MultiDBSource
 from metadata.ingestion.source.database.snowflake.constants import (
     DEFAULT_STREAM_COLUMNS,
+    SNOWFLAKE_CLASSIFICATION_DESCRIPTION,
+    SNOWFLAKE_TAG_DESCRIPTION,
 )
 from metadata.ingestion.source.database.snowflake.models import (
     STORED_PROC_LANGUAGE_MAP,
@@ -81,7 +84,8 @@ from metadata.ingestion.source.database.snowflake.models import (
 from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_DESC_FUNCTION,
     SNOWFLAKE_DESC_STORED_PROCEDURE,
-    SNOWFLAKE_FETCH_ALL_TAGS,
+    SNOWFLAKE_FETCH_SCHEMA_TAGS,
+    SNOWFLAKE_FETCH_TABLE_TAGS,
     SNOWFLAKE_GET_CLUSTER_KEY,
     SNOWFLAKE_GET_CURRENT_ACCOUNT,
     SNOWFLAKE_GET_DATABASE_COMMENTS,
@@ -123,7 +127,7 @@ from metadata.utils.sqlalchemy_utils import (
     get_all_table_ddls,
     get_all_view_definitions,
 )
-from metadata.utils.tag_utils import get_ometa_tag_and_classification
+from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_label
 
 
 class MAP(StructuredType):
@@ -203,6 +207,7 @@ class SnowflakeSource(
         self.schema_desc_map = {}
         self.database_desc_map = {}
         self.external_location_map = {}
+        self.schema_tags_map = {}
 
         self._account: Optional[str] = None
         self._org_name: Optional[str] = None
@@ -299,6 +304,32 @@ class SnowflakeSource(
             for row in results
         }
 
+    def set_schema_tags_map(self, database_name: str) -> None:
+        """Fetch and store all schema-level tags for the current database"""
+        self.schema_tags_map.clear()
+        if not self.source_config.includeTags:
+            return
+
+        try:
+            results = self.engine.execute(
+                SNOWFLAKE_FETCH_SCHEMA_TAGS.format(
+                    database_name=database_name,
+                    account_usage=self.service_connection.accountUsageSchema,
+                )
+            ).all()
+
+            for row in results:
+                schema_name = row.SCHEMA_NAME
+                if schema_name not in self.schema_tags_map:
+                    self.schema_tags_map[schema_name] = []
+                self.schema_tags_map[schema_name].append(
+                    {"tag_name": row.TAG_NAME, "tag_value": row.TAG_VALUE}
+                )
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch schema tags: {exc}")
+
     def get_schema_description(self, schema_name: str) -> Optional[str]:
         """
         Method to fetch the schema description
@@ -329,6 +360,7 @@ class SnowflakeSource(
             self.set_schema_description_map()
             self.set_database_description_map()
             self.set_external_location_map(configured_db)
+            self.set_schema_tags_map(configured_db)
             yield configured_db
         else:
             for new_database in self.get_database_names_raw():
@@ -357,6 +389,7 @@ class SnowflakeSource(
                     self.set_schema_description_map()
                     self.set_database_description_map()
                     self.set_external_location_map(new_database)
+                    self.set_schema_tags_map(new_database)
                     yield new_database
                 except Exception as exc:
                     logger.debug(traceback.format_exc())
@@ -451,11 +484,14 @@ class SnowflakeSource(
     def yield_tag(
         self, schema_name: str
     ) -> Iterable[Either[OMetaTagAndClassification]]:
+        """
+        Yield tags for tables/columns and schemas.
+        """
         if self.source_config.includeTags:
             result = []
             try:
                 result = self.connection.execute(
-                    SNOWFLAKE_FETCH_ALL_TAGS.format(
+                    SNOWFLAKE_FETCH_TABLE_TAGS.format(
                         database_name=self.context.get().database,
                         schema_name=schema_name,
                         account_usage=self.service_connection.accountUsageSchema,
@@ -469,7 +505,7 @@ class SnowflakeSource(
                         f"Error fetching tags {exc}. Trying with quoted names"
                     )
                     result = self.connection.execute(
-                        SNOWFLAKE_FETCH_ALL_TAGS.format(
+                        SNOWFLAKE_FETCH_TABLE_TAGS.format(
                             database_name=f'"{self.context.get().database}"',
                             schema_name=f'"{self.context.get().database_schema}"',
                             account_usage=self.service_connection.accountUsageSchema,
@@ -495,12 +531,32 @@ class SnowflakeSource(
                     ),
                     tags=[row[1]],
                     classification_name=row[0],
-                    tag_description="SNOWFLAKE TAG VALUE",
-                    classification_description="SNOWFLAKE TAG NAME",
+                    tag_description=SNOWFLAKE_TAG_DESCRIPTION,
+                    classification_description=SNOWFLAKE_CLASSIFICATION_DESCRIPTION,
                     metadata=self.metadata,
                     system_tags=True,
                 )
 
+            # Yield schema-level tags
+            if schema_name in self.schema_tags_map:
+                schema_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DatabaseSchema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=schema_name,
+                )
+                for tag_info in self.schema_tags_map[schema_name]:
+                    yield from get_ometa_tag_and_classification(
+                        tag_fqn=FullyQualifiedEntityName(schema_fqn),
+                        tags=[tag_info["tag_value"]],
+                        classification_name=tag_info["tag_name"],
+                        tag_description=SNOWFLAKE_TAG_DESCRIPTION,
+                        classification_description=SNOWFLAKE_CLASSIFICATION_DESCRIPTION,
+                        metadata=self.metadata,
+                        system_tags=True,
+                    )
+
     def _get_table_names_and_types(
         self, schema_name: str, table_type: TableType = TableType.Regular
     ) -> List[TableNameAndType]:
@@ -945,3 +1001,49 @@ class SnowflakeSource(
         logger.debug(
             f"Processing ownership is not supported for {self.service_connection.type.name}"
         )
+
+    def get_schema_tag_labels(self, schema_name: str) -> Optional[List[TagLabel]]:
+        """
+        Return tags for schema entity including Snowflake schema-level tags.
+        """
+        schema_tags = []
+
+        if schema_name in self.schema_tags_map:
+            for tag_info in self.schema_tags_map[schema_name]:
+                tag_label = get_tag_label(
+                    metadata=self.metadata,
+                    tag_name=tag_info["tag_value"],
+                    classification_name=tag_info["tag_name"],
+                )
+                if tag_label:
+                    schema_tags.append(tag_label)
+
+        # Include parent tags from context
+        parent_tags = super().get_schema_tag_labels(schema_name) or []
+        for tag in parent_tags:
+            if tag not in schema_tags:
+                schema_tags.append(tag)
+
+        return schema_tags if schema_tags else None
+
+    def get_tag_labels(self, table_name: str) -> Optional[List[TagLabel]]:
+        """
+        Override to include schema-level tags inherited by tables.
+        This method combines:
+        1. Tags directly assigned to the table (from parent implementation)
+        2. Tags inherited from the schema level
+        """
+        table_tags = super().get_tag_labels(table_name) or []
+
+        schema_name = self.context.get().database_schema
+        if schema_name and schema_name in self.schema_tags_map:
+            for tag_info in self.schema_tags_map[schema_name]:
+                tag_label = get_tag_label(
+                    metadata=self.metadata,
+                    tag_name=tag_info["tag_value"],
+                    classification_name=tag_info["tag_name"],
+                )
+                if tag_label and tag_label not in table_tags:
+                    table_tags.append(tag_label)
+
+        return table_tags if table_tags else None

metadata/ingestion/source/database/snowflake/queries.py

@@ -37,7 +37,7 @@ SNOWFLAKE_SQL_STATEMENT = textwrap.dedent(
 
 SNOWFLAKE_SESSION_TAG_QUERY = 'ALTER SESSION SET QUERY_TAG="{query_tag}"'
 
-SNOWFLAKE_FETCH_ALL_TAGS = textwrap.dedent(
+SNOWFLAKE_FETCH_TABLE_TAGS = textwrap.dedent(
     """
     select TAG_NAME, TAG_VALUE, OBJECT_DATABASE, OBJECT_SCHEMA, OBJECT_NAME, COLUMN_NAME
     from {account_usage}.tag_references
@@ -46,6 +46,18 @@ SNOWFLAKE_FETCH_ALL_TAGS = textwrap.dedent(
 """
 )
 
+SNOWFLAKE_FETCH_SCHEMA_TAGS = textwrap.dedent(
+    """
+    select TAG_NAME, TAG_VALUE, OBJECT_NAME as SCHEMA_NAME
+    from {account_usage}.tag_references
+    where OBJECT_DATABASE = '{database_name}'
+      and OBJECT_SCHEMA IS NULL
+      and OBJECT_NAME IS NOT NULL
+      and COLUMN_NAME IS NULL
+      and DOMAIN = 'SCHEMA'
+"""
+)
+
 SNOWFLAKE_GET_EXTERNAL_TABLE_NAMES = """
 select TABLE_NAME, NULL from information_schema.tables
 where TABLE_SCHEMA = '{schema}' AND TABLE_TYPE = 'EXTERNAL TABLE'

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -207,10 +207,18 @@ class StoredProcedureLineageMixin(ABC):
                 ).graph
 
             self.stored_procedure_query_lineage = True
+            # Prepare service names for lineage processing
+            service_names = [self.service_name]
+            if (
+                self.source_config.processCrossDatabaseLineage
+                and self.source_config.crossDatabaseServiceNames
+            ):
+                service_names.extend(self.source_config.crossDatabaseServiceNames)
+
             for either_lineage in get_lineage_by_query(
                 self.metadata,
                 query=query_by_procedure.query_text,
-                service_name=self.service_name,
+                service_names=service_names,
                 database_name=query_by_procedure.query_database_name,
                 schema_name=query_by_procedure.query_schema_name,
                 dialect=ConnectionTypeDialectMapper.dialect_of(

metadata/ingestion/source/pipeline/airflow/connection.py

@@ -27,7 +27,7 @@ from metadata.generated.schema.entity.services.connections.database.mysqlConnect
     MysqlConnection as MysqlConnectionConfig,
 )
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
-    PostgresConnection,
+    PostgresConnection as PostgresConnectionConfig,
 )
 from metadata.generated.schema.entity.services.connections.database.sqliteConnection import (
     SQLiteConnection,
@@ -70,16 +70,16 @@ def _(_: BackendConnection) -> Engine:
 def _(airflow_connection: MysqlConnectionConfig) -> Engine:
     from metadata.ingestion.source.database.mysql.connection import MySQLConnection
 
-    return MySQLConnection(airflow_connection).get_client()
+    return MySQLConnection(airflow_connection)._get_client()
 
 
 @_get_connection.register
-def _(airflow_connection: PostgresConnection) -> Engine:
+def _(airflow_connection: PostgresConnectionConfig) -> Engine:
     from metadata.ingestion.source.database.postgres.connection import (
-        get_connection as get_postgres_connection,
+        PostgresConnection,
     )
 
-    return get_postgres_connection(airflow_connection)
+    return PostgresConnection(airflow_connection)._get_client()
 
 
 @_get_connection.register
@@ -131,7 +131,9 @@ def test_connection(
 
     def test_pipeline_details_access(session):
         try:
-            result = session.query(SerializedDagModel).first()
+            # Query only the dag_id column to avoid version compatibility issues
+            # The data_compressed column doesn't exist in Airflow 2.2.5
+            result = session.query(SerializedDagModel.dag_id).first()
             return result
         except Exception as e:
             raise AirflowPipelineDetailsAccessError(

metadata/ingestion/source/pipeline/nifi/metadata.py

@@ -11,18 +11,29 @@
 """
 Nifi source to extract metadata
 """
+import math
 import traceback
 from collections import defaultdict
+from datetime import datetime
 from typing import Dict, Iterable, List, Optional
 
 from pydantic import BaseModel, ValidationError
 
 from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
-from metadata.generated.schema.entity.data.pipeline import Pipeline, Task
+from metadata.generated.schema.entity.data.pipeline import (
+    Pipeline,
+    PipelineStatus,
+    StatusType,
+    Task,
+    TaskStatus,
+)
 from metadata.generated.schema.entity.services.connections.pipeline.nifiConnection import (
     NifiConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
@@ -39,6 +50,7 @@ from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.pipeline_service import PipelineServiceSource
+from metadata.utils import fqn
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
@@ -49,6 +61,13 @@ PROCESS_GROUP_FLOW = "processGroupFlow"
 BREADCRUMB = "breadcrumb"
 PARENT_BREADCRUMB = "parentBreadcrumb"
 
+STATUS_MAP = {
+    "Running": StatusType.Successful.value,
+    "Stopped": StatusType.Pending.value,
+    "Invalid": StatusType.Failed.value,
+    "Disabled": StatusType.Skipped.value,
+}
+
 
 class NifiProcessor(BaseModel):
     """
@@ -59,6 +78,7 @@ class NifiProcessor(BaseModel):
     name: Optional[str] = None
     type_: str
     uri: str
+    run_status: Optional[str] = None
 
 
 class NifiProcessorConnections(BaseModel):
@@ -129,7 +149,7 @@ class NifiSource(PipelineServiceSource):
         try:
             return [
                 Task(
-                    name=processor.id_,
+                    name=str(processor.id_),
                     displayName=processor.name,
                     sourceUrl=SourceUrl(
                         f"{clean_uri(self.service_connection.hostPort)}{processor.uri}"
@@ -173,10 +193,46 @@ class NifiSource(PipelineServiceSource):
         self, pipeline_details: NifiPipelineDetails
     ) -> Iterable[Either[OMetaPipelineStatus]]:
         """
-        Method to get task & pipeline status.
-        Based on the latest refresh data.
-        https://github.com/open-metadata/OpenMetadata/issues/6955
+        Method to get task & pipeline status with execution history.
         """
+        try:
+
+            pipeline_fqn = fqn.build(
+                metadata=self.metadata,
+                entity_type=Pipeline,
+                service_name=self.context.get().pipeline_service,
+                pipeline_name=self.context.get().pipeline,
+            )
+
+            for task in pipeline_details.processors:
+                task_status = TaskStatus(
+                    name=str(task.id_),
+                    executionStatus=STATUS_MAP.get(task.run_status, StatusType.Pending),
+                )
+
+                pipeline_status = PipelineStatus(
+                    executionStatus=task_status.executionStatus,
+                    taskStatus=[task_status],
+                    timestamp=math.floor(
+                        (datetime.now().timestamp()) * 1000  # timestamp in milliseconds
+                    ),
+                )
+
+                yield Either(
+                    right=OMetaPipelineStatus(
+                        pipeline_fqn=pipeline_fqn,
+                        pipeline_status=pipeline_status,
+                    )
+                )
+
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=pipeline_details.name,
+                    error=f"Wild error ingesting pipeline status {pipeline_details} - {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
 
     def yield_pipeline_lineage_details(
         self, pipeline_details: NifiPipelineDetails
@@ -223,6 +279,7 @@ class NifiSource(PipelineServiceSource):
                 uri=processor.get("uri"),
                 name=processor["component"].get("name"),
                 type_=processor["component"].get("type"),
+                run_status=processor.get("status", {}).get("runStatus"),
             )
             for processor in processor_list
         ]

metadata/profiler/orm/functions/table_metric_computer.py

@@ -38,18 +38,9 @@ from metadata.utils.logger import profiler_interface_registry_logger
 logger = profiler_interface_registry_logger()
 
 
-@inject
-def get_row_count_metric(metrics: Inject[Type[MetricRegistry]] = None):
-    if metrics is None:
-        raise DependencyNotFoundError(
-            "MetricRegistry dependency not found. Please ensure the MetricRegistry is properly registered."
-        )
-    return metrics.ROW_COUNT().name()
-
-
 COLUMN_COUNT = "columnCount"
 COLUMN_NAMES = "columnNames"
-ROW_COUNT = get_row_count_metric()
+ROW_COUNT = "rowCount"
 SIZE_IN_BYTES = "sizeInBytes"
 CREATE_DATETIME = "createDateTime"
 

metadata/profiler/processor/core.py

@@ -51,12 +51,12 @@ from metadata.profiler.metrics.core import (
     TMetric,
 )
 from metadata.profiler.metrics.static.row_count import RowCount
-from metadata.profiler.orm.functions.table_metric_computer import CREATE_DATETIME
 from metadata.profiler.orm.registry import NOT_COMPUTE
 from metadata.profiler.processor.metric_filter import MetricFilter
 from metadata.utils.logger import profiler_logger
 
 logger = profiler_logger()
+CREATE_DATETIME = "createDateTime"
 
 
 class MissingMetricException(Exception):

metadata/utils/db_utils.py

@@ -14,7 +14,7 @@ Helpers module for db sources
 """
 
 import traceback
-from typing import Iterable
+from typing import Iterable, List, Union
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.table import Table
@@ -50,13 +50,16 @@ def get_host_from_host_port(uri: str) -> str:
 def get_view_lineage(
     view: TableView,
     metadata: OpenMetadata,
-    service_name: str,
+    service_names: Union[str, List[str]],
     connection_type: str,
     timeout_seconds: int = LINEAGE_PARSING_TIMEOUT,
 ) -> Iterable[Either[AddLineageRequest]]:
     """
     Method to generate view lineage
+    Now supports cross-database lineage by accepting a list of service names.
     """
+    if isinstance(service_names, str):
+        service_names = [service_names]
     table_name = view.table_name
     schema_name = view.schema_name
     db_name = view.db_name
@@ -65,7 +68,7 @@ def get_view_lineage(
     table_fqn = fqn.build(
         metadata,
         entity_type=Table,
-        service_name=service_name,
+        service_name=service_names[0],  # Use first service for table entity lookup
         database_name=db_name,
         schema_name=schema_name,
         table_name=table_name,
@@ -95,7 +98,7 @@ def get_view_lineage(
             yield from get_lineage_by_query(
                 metadata,
                 query=view_definition,
-                service_name=service_name,
+                service_names=service_names,
                 database_name=db_name,
                 schema_name=schema_name,
                 dialect=dialect,
@@ -108,7 +111,7 @@ def get_view_lineage(
             yield from get_lineage_via_table_entity(
                 metadata,
                 table_entity=table_entity,
-                service_name=service_name,
+                service_names=service_names,
                 database_name=db_name,
                 schema_name=schema_name,
                 query=view_definition,

metadata/utils/helpers.py

@@ -105,6 +105,11 @@ om_chart_type_dict = {
     "pie": ChartType.Pie,
     "text": ChartType.Text,
     "scatter": ChartType.Scatter,
+    "gauge": ChartType.Gauge,
+    "map": ChartType.Map,
+    "graph": ChartType.Graph,
+    "heatmap": ChartType.Heatmap,
+    "timeline": ChartType.Timeline,
 }
 
 
@@ -183,7 +188,7 @@ def replace_special_with(raw: str, replacement: str) -> str:
     return re.sub(r"[^a-zA-Z0-9]", replacement, raw)
 
 
-def get_standard_chart_type(raw_chart_type: str) -> ChartType.Other:
+def get_standard_chart_type(raw_chart_type: str) -> ChartType:
     """
     Get standard chart type supported by OpenMetadata based on raw chart type input
     :param raw_chart_type: raw chart type to be standardize