openmetadata-ingestion 1.9.1.0__py3-none-any.whl → 1.9.2.1__py3-none-any.whl

metadata/ingestion/source/database/snowflake/queries.py

@@ -37,7 +37,7 @@ SNOWFLAKE_SQL_STATEMENT = textwrap.dedent(
 
 SNOWFLAKE_SESSION_TAG_QUERY = 'ALTER SESSION SET QUERY_TAG="{query_tag}"'
 
-SNOWFLAKE_FETCH_ALL_TAGS = textwrap.dedent(
+SNOWFLAKE_FETCH_TABLE_TAGS = textwrap.dedent(
     """
     select TAG_NAME, TAG_VALUE, OBJECT_DATABASE, OBJECT_SCHEMA, OBJECT_NAME, COLUMN_NAME
     from {account_usage}.tag_references
@@ -46,6 +46,18 @@ SNOWFLAKE_FETCH_ALL_TAGS = textwrap.dedent(
 """
 )
 
+SNOWFLAKE_FETCH_SCHEMA_TAGS = textwrap.dedent(
+    """
+    select TAG_NAME, TAG_VALUE, OBJECT_NAME as SCHEMA_NAME
+    from {account_usage}.tag_references
+    where OBJECT_DATABASE = '{database_name}'
+      and OBJECT_SCHEMA IS NULL
+      and OBJECT_NAME IS NOT NULL
+      and COLUMN_NAME IS NULL
+      and DOMAIN = 'SCHEMA'
+"""
+)
+
 SNOWFLAKE_GET_EXTERNAL_TABLE_NAMES = """
 select TABLE_NAME, NULL from information_schema.tables
 where TABLE_SCHEMA = '{schema}' AND TABLE_TYPE = 'EXTERNAL TABLE'

metadata/ingestion/source/database/unitycatalog/connection.py

@@ -111,16 +111,15 @@ def test_connection(
 
     def get_catalogs(connection: WorkspaceClient, table_obj: DatabricksTable):
         for catalog in connection.catalogs.list():
-            table_obj.catalog_name = catalog.name
-            break
+            if catalog.name != "__databricks_internal":
+                table_obj.catalog_name = catalog.name
+                return
 
     def get_schemas(connection: WorkspaceClient, table_obj: DatabricksTable):
-        for catalog in connection.catalogs.list():
-            for schema in connection.schemas.list(catalog_name=catalog.name):
-                if schema.name:
-                    table_obj.schema_name = schema.name
-                    table_obj.catalog_name = catalog.name
-                    return
+        for schema in connection.schemas.list(catalog_name=table_obj.catalog_name):
+            if schema.name:
+                table_obj.schema_name = schema.name
+                return
 
     def get_tables(connection: WorkspaceClient, table_obj: DatabricksTable):
         if table_obj.catalog_name and table_obj.schema_name:

metadata/ingestion/source/database/unitycatalog/metadata.py

@@ -92,7 +92,7 @@ logger = ingestion_logger()
 UNITY_CATALOG_TAG = "UNITY CATALOG TAG"
 UNITY_CATALOG_TAG_CLASSIFICATION = "UNITY CATALOG TAG CLASSIFICATION"
 
-
+# pylint: disable=protected-access
 class UnitycatalogSource(
     ExternalTableLineageMixin, DatabaseServiceSource, MultiDBSource
 ):
@@ -306,6 +306,8 @@ class UnitycatalogSource(
                 if table.table_type:
                     if table.table_type.value.lower() == TableType.View.value.lower():
                         table_type: TableType = TableType.View
+                    if table.table_type.value.lower() == "materialized_view":
+                        table_type: TableType = TableType.MaterializedView
                     elif (
                         table.table_type.value.lower()
                         == TableType.External.value.lower()
@@ -427,7 +429,6 @@ class UnitycatalogSource(
             )
             if referred_table_fqn:
                 for parent_column in column.parent_columns:
-                    # pylint: disable=protected-access
                     col_fqn = fqn._build(referred_table_fqn, parent_column, quote=False)
                     if col_fqn:
                         referred_column_fqns.append(FullyQualifiedEntityName(col_fqn))
@@ -563,7 +564,7 @@ class UnitycatalogSource(
                             yield from get_ometa_tag_and_classification(
                                 tag_fqn=FullyQualifiedEntityName(
                                     fqn._build(*tag_fqn_builder(tag))
-                                ),  # pylint: disable=protected-access
+                                ),
                                 tags=[tag.tag_value],
                                 classification_name=tag.tag_name,
                                 tag_description=UNITY_CATALOG_TAG,
@@ -617,7 +618,7 @@ class UnitycatalogSource(
                             yield from get_ometa_tag_and_classification(
                                 tag_fqn=FullyQualifiedEntityName(
                                     fqn._build(*tag_fqn_builder(tag))
-                                ),  # pylint: disable=protected-access
+                                ),
                                 tags=[tag.tag_value],
                                 classification_name=tag.tag_name,
                                 tag_description=UNITY_CATALOG_TAG,

metadata/ingestion/source/pipeline/nifi/metadata.py

@@ -11,18 +11,29 @@
 """
 Nifi source to extract metadata
 """
+import math
 import traceback
 from collections import defaultdict
+from datetime import datetime
 from typing import Dict, Iterable, List, Optional
 
 from pydantic import BaseModel, ValidationError
 
 from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
-from metadata.generated.schema.entity.data.pipeline import Pipeline, Task
+from metadata.generated.schema.entity.data.pipeline import (
+    Pipeline,
+    PipelineStatus,
+    StatusType,
+    Task,
+    TaskStatus,
+)
 from metadata.generated.schema.entity.services.connections.pipeline.nifiConnection import (
     NifiConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
@@ -39,6 +50,7 @@ from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.pipeline_service import PipelineServiceSource
+from metadata.utils import fqn
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
@@ -49,6 +61,13 @@ PROCESS_GROUP_FLOW = "processGroupFlow"
 BREADCRUMB = "breadcrumb"
 PARENT_BREADCRUMB = "parentBreadcrumb"
 
+STATUS_MAP = {
+    "Running": StatusType.Successful.value,
+    "Stopped": StatusType.Pending.value,
+    "Invalid": StatusType.Failed.value,
+    "Disabled": StatusType.Skipped.value,
+}
+
 
 class NifiProcessor(BaseModel):
     """
@@ -59,6 +78,7 @@ class NifiProcessor(BaseModel):
     name: Optional[str] = None
     type_: str
     uri: str
+    run_status: Optional[str] = None
 
 
 class NifiProcessorConnections(BaseModel):
@@ -129,7 +149,7 @@ class NifiSource(PipelineServiceSource):
         try:
             return [
                 Task(
-                    name=processor.id_,
+                    name=str(processor.id_),
                     displayName=processor.name,
                     sourceUrl=SourceUrl(
                         f"{clean_uri(self.service_connection.hostPort)}{processor.uri}"
@@ -173,10 +193,46 @@ class NifiSource(PipelineServiceSource):
         self, pipeline_details: NifiPipelineDetails
     ) -> Iterable[Either[OMetaPipelineStatus]]:
         """
-        Method to get task & pipeline status.
-        Based on the latest refresh data.
-        https://github.com/open-metadata/OpenMetadata/issues/6955
+        Method to get task & pipeline status with execution history.
         """
+        try:
+
+            pipeline_fqn = fqn.build(
+                metadata=self.metadata,
+                entity_type=Pipeline,
+                service_name=self.context.get().pipeline_service,
+                pipeline_name=self.context.get().pipeline,
+            )
+
+            for task in pipeline_details.processors:
+                task_status = TaskStatus(
+                    name=str(task.id_),
+                    executionStatus=STATUS_MAP.get(task.run_status, StatusType.Pending),
+                )
+
+                pipeline_status = PipelineStatus(
+                    executionStatus=task_status.executionStatus,
+                    taskStatus=[task_status],
+                    timestamp=math.floor(
+                        (datetime.now().timestamp()) * 1000  # timestamp in milliseconds
+                    ),
+                )
+
+                yield Either(
+                    right=OMetaPipelineStatus(
+                        pipeline_fqn=pipeline_fqn,
+                        pipeline_status=pipeline_status,
+                    )
+                )
+
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=pipeline_details.name,
+                    error=f"Wild error ingesting pipeline status {pipeline_details} - {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
 
     def yield_pipeline_lineage_details(
         self, pipeline_details: NifiPipelineDetails
@@ -223,6 +279,7 @@ class NifiSource(PipelineServiceSource):
                 uri=processor.get("uri"),
                 name=processor["component"].get("name"),
                 type_=processor["component"].get("type"),
+                run_status=processor.get("status", {}).get("runStatus"),
             )
             for processor in processor_list
         ]

metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py

@@ -23,6 +23,7 @@ from metadata.generated.schema.entity.data.table import (
     ColumnName,
     DataType,
     SystemProfile,
+    TableType,
 )
 from metadata.generated.schema.entity.services.databaseService import (
     DatabaseServiceType,
@@ -51,6 +52,11 @@ class DatabricksProfilerInterface(SQAProfilerInterface):
         *args,
         **kwargs,
     ) -> List[SystemProfile]:
+        if self.table_entity.tableType in (TableType.View, TableType.MaterializedView):
+            logger.debug(
+                f"Skipping {metrics.name()} metric for view {runner.table_name}"
+            )
+            return []
         logger.debug(f"Computing {metrics.name()} metric for {runner.table_name}")
         self.system_metrics_class = cast(
             Type[DatabricksSystemMetricsComputer], self.system_metrics_class

metadata/profiler/orm/functions/table_metric_computer.py

@@ -38,18 +38,9 @@ from metadata.utils.logger import profiler_interface_registry_logger
 logger = profiler_interface_registry_logger()
 
 
-@inject
-def get_row_count_metric(metrics: Inject[Type[MetricRegistry]] = None):
-    if metrics is None:
-        raise DependencyNotFoundError(
-            "MetricRegistry dependency not found. Please ensure the MetricRegistry is properly registered."
-        )
-    return metrics.ROW_COUNT().name()
-
-
 COLUMN_COUNT = "columnCount"
 COLUMN_NAMES = "columnNames"
-ROW_COUNT = get_row_count_metric()
+ROW_COUNT = "rowCount"
 SIZE_IN_BYTES = "sizeInBytes"
 CREATE_DATETIME = "createDateTime"
 

metadata/profiler/processor/core.py

@@ -51,12 +51,12 @@ from metadata.profiler.metrics.core import (
     TMetric,
 )
 from metadata.profiler.metrics.static.row_count import RowCount
-from metadata.profiler.orm.functions.table_metric_computer import CREATE_DATETIME
 from metadata.profiler.orm.registry import NOT_COMPUTE
 from metadata.profiler.processor.metric_filter import MetricFilter
 from metadata.utils.logger import profiler_logger
 
 logger = profiler_logger()
+CREATE_DATETIME = "createDateTime"
 
 
 class MissingMetricException(Exception):

metadata/utils/fqn.py

@@ -15,6 +15,7 @@ ES indexes definitions
 """
 import hashlib
 import re
+import traceback
 from typing import Dict, List, Optional, Type, TypeVar, Union
 
 from antlr4.CommonTokenStream import CommonTokenStream
@@ -53,6 +54,9 @@ from metadata.generated.schema.tests.testSuite import TestSuite
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils.dispatch import class_register
 from metadata.utils.elasticsearch import get_entity_from_es_result
+from metadata.utils.logger import utils_logger
+
+logger = utils_logger()
 
 T = TypeVar("T", bound=BaseModel)
 
@@ -144,12 +148,32 @@ def build(
     :param kwargs: required to build the FQN
     :return: FQN as a string
     """
+    # Transform table_name and column_name if they exist and contain special characters
+    if kwargs.get("table_name") or kwargs.get("column_name"):
+        from metadata.ingestion.models.custom_basemodel_validation import (  # pylint: disable=import-outside-toplevel
+            replace_separators,
+        )
+
+        table_name = kwargs.get("table_name")
+        if table_name and isinstance(table_name, str):
+            kwargs["table_name"] = replace_separators(table_name)
+
+        column_name = kwargs.get("column_name")
+        if column_name and isinstance(column_name, str):
+            kwargs["column_name"] = replace_separators(column_name)
+
     func = fqn_build_registry.registry.get(entity_type.__name__)
-    if not func:
+    try:
+        if not func:
+            raise FQNBuildingException(
+                f"Invalid Entity Type {entity_type.__name__}. FQN builder not implemented."
+            )
+        return func(metadata, **kwargs)
+    except Exception as e:
+        logger.debug(traceback.format_exc())
         raise FQNBuildingException(
-            f"Invalid Entity Type {entity_type.__name__}. FQN builder not implemented."
+            f"Error building FQN for {entity_type.__name__}: {e}"
         )
-    return func(metadata, **kwargs)
 
 
 @fqn_build_registry.add(Table)