openmetadata-ingestion 1.6.5.2.dev0__py3-none-any.whl → 1.6.6.0__py3-none-any.whl

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-03-11T12:59:30+00:00
+#   timestamp: 2025-03-13T17:32:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-03-11T12:59:30+00:00
+#   timestamp: 2025-03-13T17:32:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-03-11T12:59:30+00:00
+#   timestamp: 2025-03-13T17:32:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-03-11T12:59:30+00:00
+#   timestamp: 2025-03-13T17:32:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-03-11T12:59:30+00:00
+#   timestamp: 2025-03-13T17:32:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-03-11T12:59:30+00:00
+#   timestamp: 2025-03-13T17:32:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-03-11T12:59:30+00:00
+#   timestamp: 2025-03-13T17:32:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-03-11T12:59:30+00:00
+#   timestamp: 2025-03-13T17:32:30+00:00
 
 from __future__ import annotations
 

metadata/ingestion/models/ometa_lineage.py

@@ -12,13 +12,17 @@
 Custom wrapper for Lineage Request
 """
 
-from typing import Optional
+from typing import Optional, Type, TypeVar
 
 from pydantic import BaseModel
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 
+T = TypeVar("T", bound=BaseModel)
+
 
 class OMetaLineageRequest(BaseModel):
     override_lineage: Optional[bool] = False
     lineage_request: AddLineageRequest
+    entity_fqn: Optional[str] = None
+    entity: Optional[Type[T]] = None

metadata/ingestion/models/patch_request.py

@@ -145,7 +145,13 @@ ALLOWED_COMMON_PATCH_FIELDS = {
     "fileFormats": True,
 }
 
-RESTRICT_UPDATE_LIST = ["description", "tags", "owners", "displayName"]
+RESTRICT_UPDATE_LIST = [
+    "description",
+    "tags",
+    "owners",
+    "displayName",
+    "tableConstraints",
+]
 
 ARRAY_ENTITY_FIELDS = ["columns", "tasks", "fields"]
 

metadata/ingestion/ometa/mixins/es_mixin.py

@@ -391,6 +391,7 @@ class ESMixin(Generic[T]):
     def yield_es_view_def(
         self,
         service_name: str,
+        incremental: bool = False,
     ) -> Iterable[TableView]:
         """
         Get the view definition from ES
@@ -451,6 +452,23 @@ class ESMixin(Generic[T]):
                 }
             }
         }
+        if incremental:
+            query.get("query").get("bool").get("must").append(
+                {
+                    "bool": {
+                        "should": [
+                            {"term": {"processedLineage": False}},
+                            {
+                                "bool": {
+                                    "must_not": {
+                                        "exists": {"field": "processedLineage"}
+                                    }
+                                }
+                            },
+                        ]
+                    }
+                }
+            )
         query = json.dumps(query)
         for response in self._paginate_es_internal(
             entity=Table,

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -406,3 +406,24 @@ class OMetaLineageMixin(Generic[T]):
                     logger.error(
                         f"Error while adding lineage: {lineage_request.left.error}"
                     )
+
+    def patch_lineage_processed_flag(
+        self,
+        entity: Type[T],
+        fqn: str,
+    ) -> None:
+
+        try:
+            original_entity = self.get_by_name(entity=entity, fqn=fqn)
+            if not original_entity:
+                return
+
+            updated_entity = original_entity.model_copy(deep=True)
+            updated_entity.processedLineage = True
+
+            self.patch(
+                entity=entity, source=original_entity, destination=updated_entity
+            )
+        except Exception as exc:
+            logger.debug(f"Error while patching lineage processed flag: {exc}")
+            logger.debug(traceback.format_exc())

metadata/ingestion/sink/metadata_rest.py

@@ -288,7 +288,16 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
                     entity_id=str(add_lineage.lineage_request.edge.toEntity.id.root),
                     source=add_lineage.lineage_request.edge.lineageDetails.source.value,
                 )
-        return self._run_dispatch(add_lineage.lineage_request)
+        lineage_response = self._run_dispatch(add_lineage.lineage_request)
+        if (
+            lineage_response
+            and lineage_response.right is not None
+            and add_lineage.entity_fqn
+            and add_lineage.entity
+        ):
+            self.metadata.patch_lineage_processed_flag(
+                entity=add_lineage.entity, fqn=add_lineage.entity_fqn
+            )
 
     def _create_role(self, create_role: CreateRoleRequest) -> Optional[Role]:
         """

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -1042,17 +1042,19 @@ class LookerSource(DashboardServiceSource):
                     continue
 
                 description = self.build_chart_description(chart)
+                if chart.query is not None:
+                    source_url = chart.query.share_url
+                elif getattr(chart.result_maker, "query", None) is not None:
+                    source_url = chart.result_maker.query.share_url
+                else:
+                    source_url = f"{clean_uri(self.service_connection.hostPort)}/merge?mid={chart.merge_result_id}"
                 yield Either(
                     right=CreateChartRequest(
                         name=EntityName(chart.id),
                         displayName=chart.title or chart.id,
                         description=Markdown(description) if description else None,
                         chartType=get_standard_chart_type(chart.type).value,
-                        sourceUrl=SourceUrl(chart.query.share_url)
-                        if chart.query is not None
-                        else SourceUrl(
-                            f"{clean_uri(self.service_connection.hostPort)}/merge?mid={chart.merge_result_id}"
-                        ),
+                        sourceUrl=SourceUrl(source_url),
                         service=self.context.get().dashboard_service,
                     )
                 )

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -10,6 +10,7 @@
 #  limitations under the License.
 """PowerBI source module"""
 
+import re
 import traceback
 from typing import Any, Iterable, List, Optional, Union
 
@@ -610,6 +611,87 @@ class PowerbiSource(DashboardServiceSource):
             logger.debug(f"Error to get data_model_column_fqn {exc}")
             logger.debug(traceback.format_exc())
 
+    def _parse_redshift_source(self, source_expression: str) -> Optional[dict]:
+        try:
+            db_match = re.search(
+                r'AmazonRedshift\.Database\("[^"]+","([^"]+)"\)', source_expression
+            )
+            if not db_match:
+                # not valid redshift source
+                return None
+            schema_table_match = re.findall(r'\[Name="([^"]+)"\]', source_expression)
+
+            database = db_match.group(1) if db_match else None
+            schema = table = None
+            if isinstance(schema_table_match, list):
+                schema = schema_table_match[0] if len(schema_table_match) > 0 else None
+                table = schema_table_match[1] if len(schema_table_match) > 1 else None
+
+            if (
+                schema and table
+            ):  # atlease 2 entities schema and table should be fetched
+                return {"database": database, "schema": schema, "table": table}
+            return None
+        except Exception as exc:
+            logger.debug(f"Error to parse redshift table source: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def _parse_snowflake_source(self, source_expression: str) -> Optional[dict]:
+        try:
+            if "Snowflake.Databases" not in source_expression:
+                # Not a snowflake valid expression
+                return None
+            db_match = re.search(
+                r'\[Name="([^"]+)",Kind="Database"\]', source_expression
+            )
+            schema_match = re.search(
+                r'\[Name="([^"]+)",Kind="Schema"\]', source_expression
+            )
+            view_match = re.search(r'\[Name="([^"]+)",Kind="View"\]', source_expression)
+            table_match = re.search(
+                r'\[Name="([^"]+)",Kind="Table"\]', source_expression
+            )
+
+            database = db_match.group(1) if db_match else None
+            schema = schema_match.group(1) if schema_match else None
+            view = view_match.group(1) if view_match else None
+            table = table_match.group(1) if table_match else None
+
+            if schema and (
+                table or view
+            ):  # atlease 2 entities schema and table should be fetched
+                return {
+                    "database": database,
+                    "schema": schema,
+                    "table": table if table else view,
+                }
+            return None
+        except Exception as exc:
+            logger.debug(f"Error to parse snowflake table source: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def _parse_table_info_from_source_exp(self, table: PowerBiTable) -> dict:
+        try:
+            if not isinstance(table.source, list):
+                return {}
+            source_expression = table.source[0].expression
+
+            # parse snowflake source
+            table_info = self._parse_snowflake_source(source_expression)
+            if isinstance(table_info, dict):
+                return table_info
+            # parse redshift source
+            table_info = self._parse_redshift_source(source_expression)
+            if isinstance(table_info, dict):
+                return table_info
+            return {}
+        except Exception as exc:
+            logger.debug(f"Error to parse table source: {exc}")
+            logger.debug(traceback.format_exc())
+        return {}
+
     def _get_table_and_datamodel_lineage(
         self,
         db_service_name: str,
@@ -620,13 +702,14 @@ class PowerbiSource(DashboardServiceSource):
         Method to create lineage between table and datamodels
         """
         try:
+            table_info = self._parse_table_info_from_source_exp(table)
             table_fqn = fqn.build(
                 self.metadata,
                 entity_type=Table,
                 service_name=db_service_name,
-                database_name=None,
-                schema_name=None,
-                table_name=table.name,
+                database_name=table_info.get("database"),
+                schema_name=table_info.get("schema"),
+                table_name=table_info.get("table") or table.name,
             )
             table_entity = self.metadata.get_by_name(
                 entity=Table,

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -97,6 +97,14 @@ class PowerBiColumns(BaseModel):
     columnType: Optional[str] = None
 
 
+class PowerBITableSource(BaseModel):
+    """
+    PowerBI Table Source
+    """
+
+    expression: str
+
+
 class PowerBiTable(BaseModel):
     """
     PowerBI Table Model
@@ -106,6 +114,7 @@ class PowerBiTable(BaseModel):
     name: str
     columns: Optional[List[PowerBiColumns]] = None
     description: Optional[str] = None
+    source: Optional[List[PowerBITableSource]] = None
 
 
 class TablesResponse(BaseModel):

metadata/ingestion/source/dashboard/superset/mixin.py

@@ -227,6 +227,17 @@ class SupersetSourceMixin(DashboardServiceSource):
         """clean datatype of column fetched from superset"""
         return datatype.replace("()", "")
 
+    def parse_array_data_type(self, col_parse: dict) -> Optional[str]:
+        """
+        Set arrayDataType to UNKNOWN for Snowflake table array columns
+        to prevent validation error requiring non-null arrayDataType
+        """
+        if col_parse["dataType"] == "ARRAY" and not col_parse.get("arrayDataType"):
+            return DataType.UNKNOWN
+        if col_parse.get("arrayDataType"):
+            return DataType(col_parse["arrayDataType"])
+        return None
+
     def get_column_info(
         self, data_source: List[Union[DataSourceResult, FetchColumn]]
     ) -> Optional[List[Column]]:
@@ -247,9 +258,7 @@ class SupersetSourceMixin(DashboardServiceSource):
                     parsed_fields = Column(
                         dataTypeDisplay=field.type,
                         dataType=col_parse["dataType"],
-                        arrayDataType=DataType(col_parse["arrayDataType"])
-                        if col_parse.get("arrayDataType")
-                        else None,
+                        arrayDataType=self.parse_array_data_type(col_parse),
                         children=list(col_parse["children"])
                         if col_parse.get("children")
                         else None,

metadata/ingestion/source/database/common_db_source.py

@@ -252,7 +252,11 @@ class CommonDbSourceService(
         )
         source_url = (
             SourceUrl(source_url)
-            if (source_url := self.get_source_url(database_name=schema_name))
+            if (
+                source_url := self.get_source_url(
+                    database_name=self.context.get().database, schema_name=schema_name
+                )
+            )
             else None
         )
 
@@ -708,7 +712,7 @@ class CommonDbSourceService(
                 table_constraints.extend(
                     constraint
                     for constraint in foreign_table_constraints
-                    if constraint not in table_constraints
+                    if constraint and constraint not in table_constraints
                 )
             else:
                 table_constraints = foreign_table_constraints

metadata/ingestion/source/database/lineage_source.py

@@ -265,11 +265,22 @@ class LineageSource(QueryParserSource, ABC):
                     timeout_seconds=self.source_config.parsingTimeoutLimit,
                 ):
                     if lineage.right is not None:
+                        view_fqn = fqn.build(
+                            metadata=self.metadata,
+                            entity_type=Table,
+                            service_name=self.service_name,
+                            database_name=view.db_name,
+                            schema_name=view.schema_name,
+                            table_name=view.table_name,
+                            skip_es_search=True,
+                        )
                         queue.put(
                             Either(
                                 right=OMetaLineageRequest(
                                     lineage_request=lineage.right,
                                     override_lineage=self.source_config.overrideViewLineage,
+                                    entity_fqn=view_fqn,
+                                    entity=Table,
                                 )
                             )
                         )
@@ -281,7 +292,11 @@ class LineageSource(QueryParserSource, ABC):
 
     def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
         logger.info("Processing View Lineage")
-        producer_fn = partial(self.metadata.yield_es_view_def, self.config.serviceName)
+        producer_fn = partial(
+            self.metadata.yield_es_view_def,
+            self.config.serviceName,
+            self.source_config.incrementalLineageProcessing,
+        )
         processor_fn = self.view_lineage_generator
         yield from self.generate_lineage_in_thread(producer_fn, processor_fn)
 

metadata/ingestion/source/database/redshift/metadata.py

@@ -84,6 +84,7 @@ from metadata.ingestion.source.database.redshift.utils import (
     _get_pg_column_info,
     _get_schema_column_info,
     get_columns,
+    get_redshift_columns,
     get_table_comment,
     get_view_definition,
 )
@@ -109,18 +110,15 @@ STANDARD_TABLE_TYPES = {
     "v": TableType.View,
 }
 
-
-RedshiftDialectMixin._get_column_info = (  # pylint: disable=protected-access
-    _get_column_info
-)
-RedshiftDialectMixin._get_schema_column_info = (  # pylint: disable=protected-access
-    _get_schema_column_info
-)
+# pylint: disable=protected-access
+RedshiftDialectMixin._get_column_info = _get_column_info
+RedshiftDialectMixin._get_schema_column_info = _get_schema_column_info
 RedshiftDialectMixin.get_columns = get_columns
-PGDialect._get_column_info = _get_pg_column_info  # pylint: disable=protected-access
+PGDialect._get_column_info = _get_pg_column_info
 RedshiftDialect.get_all_table_comments = get_all_table_comments
 RedshiftDialect.get_table_comment = get_table_comment
 RedshiftDialect.get_view_definition = get_view_definition
+RedshiftDialect._get_redshift_columns = get_redshift_columns
 RedshiftDialect._get_all_relation_info = (  # pylint: disable=protected-access
     _get_all_relation_info
 )

metadata/ingestion/source/database/redshift/utils.py

@@ -34,6 +34,7 @@ from metadata.ingestion.source.database.redshift.queries import (
     REDSHIFT_TABLE_COMMENTS,
 )
 from metadata.utils.execution_time_tracker import calculate_execution_time
+from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import get_table_comment_wrapper
 
 sa_version = Version(sa.__version__)
@@ -45,6 +46,9 @@ ischema_names.update({"binary varying": sqltypes.VARBINARY})
 ischema_names.update(REDSHIFT_ISCHEMA_NAMES)
 
 
+logger = ingestion_logger()
+
+
 # pylint: disable=protected-access
 @calculate_execution_time()
 @reflection.cache
@@ -414,3 +418,25 @@ def get_view_definition(self, connection, view_name, schema=None, **kw):
             f"CREATE VIEW {view.schema}.{view.relname} AS {view_definition}"
         )
     return view_definition
+
+
+def get_redshift_columns(self, connection, table_name, schema=None, **kw):
+    try:
+        info_cache = kw.get("info_cache")
+        all_schema_columns = self._get_schema_column_info(
+            connection,
+            schema,
+            info_cache=info_cache,
+        )
+        key = RelationKey(table_name, schema, connection)
+        if key not in all_schema_columns.keys():
+            key = key.unquoted()
+        return all_schema_columns[key]
+    except KeyError:
+        schema_name = schema or "public"
+        logger.error(
+            f"Fetching columns for table {schema_name}.{table_name} failed,"
+            " if this is a view with no schema binding, please make sure user has"
+            f' USAGE privilege on schema "{schema_name}"'
+        )
+        return []

metadata/ingestion/source/database/snowflake/lineage.py

@@ -49,7 +49,6 @@ class SnowflakeLineageSource(
             OR (QUERY_TYPE = 'INSERT' and query_text ILIKE '%%insert%%into%%select%%')
             OR (QUERY_TYPE = 'ALTER' and query_text ILIKE '%%alter%%table%%swap%%')
             OR (QUERY_TYPE = 'CREATE_TABLE' and query_text ILIKE '%%clone%%')
-            OR (QUERY_TYPE = 'CREATE_VIEW' and query_text ILIKE '%%create%%temporary%%view%%')
         )
     """
 

metadata/ingestion/source/database/snowflake/metadata.py

@@ -115,7 +115,6 @@ from metadata.utils.tag_utils import get_ometa_tag_and_classification
 ischema_names["VARIANT"] = VARIANT
 ischema_names["GEOGRAPHY"] = create_sqlalchemy_type("GEOGRAPHY")
 ischema_names["GEOMETRY"] = create_sqlalchemy_type("GEOMETRY")
-ischema_names["VECTOR"] = create_sqlalchemy_type("VECTOR")
 
 logger = ingestion_logger()
 

metadata/ingestion/source/database/snowflake/utils.py

@@ -300,8 +300,7 @@ def get_schema_columns(self, connection, schema, **kw):
             sa_util.warn(
                 f"Did not recognize type '{coltype}' of column '{column_name}'"
             )
-            col_type = sqltypes.NullType
-            type_instance = col_type()
+            col_type = sqltypes.NULLTYPE
         else:
             if issubclass(col_type, FLOAT):
                 col_type_kw["precision"] = numeric_precision
@@ -311,7 +310,8 @@ def get_schema_columns(self, connection, schema, **kw):
                 col_type_kw["scale"] = numeric_scale
             elif issubclass(col_type, (sqltypes.String, sqltypes.BINARY)):
                 col_type_kw["length"] = character_maximum_length
-            type_instance = col_type(**col_type_kw)
+
+        type_instance = col_type(**col_type_kw)
 
         current_table_pks = schema_primary_keys.get(table_name)
 

metadata/ingestion/source/database/sql_column_handler.py

@@ -96,7 +96,7 @@ class SqlColumnHandlerMixin:
             col_type = ColumnTypeParser.get_column_type(column["type"])
             # For arrays, we'll get the item type if possible, or parse the string representation of the column
             # if SQLAlchemy does not provide any further information
-            if col_type == "ARRAY" and getattr(column["type"], "item_type"):
+            if col_type == "ARRAY" and getattr(column["type"], "item_type", None):
                 arr_data_type = ColumnTypeParser.get_column_type(
                     column["type"].item_type
                 )

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -38,6 +38,7 @@ from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.status import Status
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query
+from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.models.topology import Queue
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils.logger import ingestion_logger
@@ -221,7 +222,19 @@ class StoredProcedureLineageMixin(ABC):
                     query_by_procedure=procedure_and_query.query_by_procedure,
                     procedure=procedure_and_query.procedure,
                 ):
-                    queue.put(lineage)
+                    if lineage and lineage.right is not None:
+                        queue.put(
+                            Either(
+                                right=OMetaLineageRequest(
+                                    override_lineage=False,
+                                    lineage_request=lineage.right,
+                                    entity=StoredProcedure,
+                                    entity_fqn=procedure_and_query.procedure.fullyQualifiedName.root,
+                                )
+                            )
+                        )
+                    else:
+                        queue.put(lineage)
             except Exception as exc:
                 logger.debug(traceback.format_exc())
                 logger.warning(
@@ -260,8 +273,11 @@ class StoredProcedureLineageMixin(ABC):
                 }
             }
         }
+        if self.source_config.incrementalLineageProcessing:
+            query.get("query").get("bool").get("must").append(
+                {"bool": {"should": [{"term": {"processedLineage": False}}]}}
+            )
         query_filter = json.dumps(query)
-
         logger.info("Processing Lineage for Stored Procedures")
         # First, get all the query history
         queries_dict = self.get_stored_procedure_queries_dict()

metadata/ingestion/source/database/usage_source.py

@@ -122,15 +122,15 @@ class UsageSource(QueryParserSource, ABC):
                             row = dict(row)
                             try:
                                 row.update({k.lower(): v for k, v in row.items()})
-                                logger.debug(f"Processing row: {query}")
+                                logger.debug(f"Processing row: {row}")
                                 query_type = row.get("query_type")
-                                query = self.format_query(row["query_text"])
+                                query_text = self.format_query(row["query_text"])
                                 queries.append(
                                     TableQuery(
-                                        query=query,
+                                        query=query_text,
                                         query_type=query_type,
                                         exclude_usage=self.check_life_cycle_query(
-                                            query_type=query_type, query_text=query
+                                            query_type=query_type, query_text=query_text
                                         ),
                                         dialect=self.dialect.value,
                                         userName=row["user_name"],

metadata/pii/processor.py

@@ -129,12 +129,13 @@ class PIIProcessor(Processor):
         if column_has_pii_tag is True:
             return None
 
-        # Scan by column name. If no results there, check the sample data, if any
-        tag_and_confidence = self.name_scanner.scan(column.name.root) or (
+        # We'll scan first by sample data to prioritize the NER scanner
+        # If we find nothing, we'll check the column name
+        tag_and_confidence = (
             self.ner_scanner.scan([row[idx] for row in table_data.rows])
             if table_data
             else None
-        )
+        ) or self.name_scanner.scan(column.name.root)
 
         if (
             tag_and_confidence