openmetadata-ingestion 1.5.5.0__py3-none-any.whl → 1.5.6.0__py3-none-any.whl

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-09-25T04:53:10+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -23,7 +23,7 @@ from pydantic import BaseModel
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.type.basic import FullyQualifiedEntityName, Uuid
-from metadata.generated.schema.type.entityLineage import EntitiesEdge
+from metadata.generated.schema.type.entityLineage import ColumnLineage, EntitiesEdge
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.lineage.parser import LINEAGE_PARSING_TIMEOUT
@@ -139,6 +139,18 @@ class OMetaLineageMixin(Generic[T]):
                         )
                     )
 
+                    serialized_col_details = []
+                    for col_lin in data.edge.lineageDetails.columnsLineage or []:
+                        serialized_col_details.append(ColumnLineage(**col_lin))
+                    data.edge.lineageDetails.columnsLineage = serialized_col_details
+
+                    serialized_col_details_og = []
+                    for col_lin in original.edge.lineageDetails.columnsLineage or []:
+                        serialized_col_details_og.append(ColumnLineage(**col_lin))
+                    original.edge.lineageDetails.columnsLineage = (
+                        serialized_col_details_og
+                    )
+
                     # Keep the pipeline information from the original
                     # lineage if available
                     if (

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -544,12 +544,12 @@ class PowerbiSource(DashboardServiceSource):
                         to_entity=report_entity, from_entity=datamodel_entity
                     )
 
-                    # create the lineage between table and datamodel
-                    yield from self.create_table_datamodel_lineage(
-                        db_service_name=db_service_name,
-                        tables=dataset.tables,
-                        datamodel_entity=datamodel_entity,
-                    )
+                    for table in dataset.tables or []:
+                        yield self._get_table_and_datamodel_lineage(
+                            db_service_name=db_service_name,
+                            table=table,
+                            datamodel_entity=datamodel_entity,
+                        )
 
                     # create the lineage between table and datamodel using the pbit files
                     if self.client.file_client:
@@ -678,20 +678,6 @@ class PowerbiSource(DashboardServiceSource):
                 )
             )
 
-    def create_table_datamodel_lineage(
-        self,
-        db_service_name: str,
-        tables: Optional[List[PowerBiTable]],
-        datamodel_entity: Optional[DashboardDataModel],
-    ) -> Iterable[Either[CreateDashboardRequest]]:
-        """Method to create lineage between table and datamodels"""
-        for table in tables or []:
-            yield self._get_table_and_datamodel_lineage(
-                db_service_name=db_service_name,
-                table=table,
-                datamodel_entity=datamodel_entity,
-            )
-
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: Union[PowerBIDashboard, PowerBIReport],

metadata/ingestion/source/database/dbt/dbt_service.py

@@ -160,7 +160,7 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
         # This step is necessary as the manifest file may not always adhere to the schema definition
         # and the presence of other nodes can hinder the ingestion process from progressing any further.
         # Therefore, we are only retaining the essential data for further processing.
-        required_manifest_keys = ["nodes", "sources", "metadata"]
+        required_manifest_keys = {"nodes", "sources", "metadata"}
         manifest_dict.update(
             {
                 key: {}
@@ -169,6 +169,52 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
             }
         )
 
+        required_nodes_keys = {
+            "schema_",
+            "schema",
+            "name",
+            "resource_type",
+            "path",
+            "unique_id",
+            "fqn",
+            "alias",
+            "checksum",
+            "config",
+            "column_name",
+            "test_metadata",
+            "original_file_path",
+            "root_path",
+            "database",
+            "tags",
+            "description",
+            "columns",
+            "meta",
+            "owner",
+            "created_at",
+            "group",
+            "sources",
+            "compiled",
+            "docs",
+            "version",
+            "latest_version",
+            "package_name",
+            "depends_on",
+            "compiled_code",
+            "compiled_sql",
+            "raw_code",
+            "raw_sql",
+            "language",
+        }
+
+        for node, value in manifest_dict.get(  # pylint: disable=unused-variable
+            "nodes"
+        ).items():
+            keys_to_delete = [
+                key for key in value if key.lower() not in required_nodes_keys
+            ]
+            for key in keys_to_delete:
+                del value[key]
+
     def get_dbt_files(self) -> Iterable[DbtFiles]:
         dbt_files = get_dbt_details(self.source_config.dbtConfigSource)
         for dbt_file in dbt_files:

metadata/ingestion/source/database/iceberg/catalog/rest.py

@@ -49,7 +49,7 @@ class IcebergRestCatalog(IcebergCatalogBase):
 
         parameters = {
             "warehouse": catalog.warehouseLocation,
-            "uri": catalog.connection.uri,
+            "uri": str(catalog.connection.uri),
             "credential": credential,
             "token": catalog.connection.token.get_secret_value()
             if catalog.connection.token

metadata/ingestion/source/metadata/alationsink/metadata.py

@@ -14,11 +14,17 @@ AlationSink source to extract metadata
 """
 
 import traceback
-from typing import Iterable, Optional
+from typing import Iterable, List, Optional
 
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
-from metadata.generated.schema.entity.data.table import Column, Constraint, Table
+from metadata.generated.schema.entity.data.table import (
+    Column,
+    Constraint,
+    ConstraintType,
+    Table,
+    TableConstraint,
+)
 from metadata.generated.schema.entity.services.connections.metadata.alationSinkConnection import (
     AlationSinkConnection,
 )
@@ -108,7 +114,9 @@ class AlationsinkSource(Source):
                     ),
                 ),
                 db_username="Test",
-                title=model_str(om_database.name),
+                title=om_database.displayName
+                if om_database.displayName
+                else model_str(om_database.name),
                 description=model_str(om_database.description),
             )
         except Exception as exc:
@@ -129,7 +137,9 @@ class AlationsinkSource(Source):
                 key=fqn._build(  # pylint: disable=protected-access
                     str(alation_datasource_id), model_str(om_schema.name)
                 ),
-                title=model_str(om_schema.name),
+                title=om_schema.displayName
+                if om_schema.displayName
+                else model_str(om_schema.name),
                 description=model_str(om_schema.description),
             )
         except Exception as exc:
@@ -150,7 +160,9 @@ class AlationsinkSource(Source):
                 key=fqn._build(  # pylint: disable=protected-access
                     str(alation_datasource_id), schema_name, model_str(om_table.name)
                 ),
-                title=model_str(om_table.name),
+                title=om_table.displayName
+                if om_table.displayName
+                else model_str(om_table.name),
                 description=model_str(om_table.description),
                 table_type=TABLE_TYPE_MAPPER.get(om_table.tableType, "TABLE"),
                 sql=om_table.schemaDefinition,
@@ -162,14 +174,60 @@ class AlationsinkSource(Source):
             )
         return None
 
-    def _get_column_index(self, om_column: Column) -> Optional[ColumnIndex]:
+    def _update_foreign_key(
+        self,
+        alation_datasource_id: int,
+        om_column: Column,
+        table_constraints: Optional[List[TableConstraint]],
+        column_index: ColumnIndex,
+    ):
+        """
+        Method to update the foreign key metadata in columns index
+        """
+        try:
+            for table_constraint in table_constraints or []:
+                if table_constraint.constraintType == ConstraintType.FOREIGN_KEY:
+                    for i, constraint_column in enumerate(
+                        table_constraint.columns or []
+                    ):
+                        if constraint_column == model_str(om_column.name):
+                            column_index.isForeignKey = True
+                            # update the service name of OM with the alation datasource id in the column FQN
+                            splitted_col_fqn = fqn.split(
+                                model_str(table_constraint.referredColumns[i])
+                            )
+                            splitted_col_fqn[0] = str(alation_datasource_id)
+                            column_index.referencedColumnId = (
+                                fqn._build(  # pylint: disable=protected-access
+                                    *splitted_col_fqn
+                                )
+                            )
+                            break
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Failed to update foreign key for {model_str(om_column.name)}: {exc}"
+            )
+
+    def _get_column_index(
+        self,
+        alation_datasource_id: int,
+        om_column: Column,
+        table_constraints: Optional[List[TableConstraint]],
+    ) -> Optional[ColumnIndex]:
         """
         Method to get the alation column index
         """
         column_index = ColumnIndex()
         try:
+            # Attach the primary key
             if om_column.constraint == Constraint.PRIMARY_KEY:
-                return ColumnIndex(isPrimaryKey=True)
+                column_index.isPrimaryKey = True
+
+            # Attach the foreign key
+            self._update_foreign_key(
+                alation_datasource_id, om_column, table_constraints, column_index
+            )
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(
@@ -199,6 +257,7 @@ class AlationsinkSource(Source):
         schema_name: str,
         table_name: str,
         om_column: Column,
+        table_constraints: Optional[List[TableConstraint]],
     ) -> Optional[CreateColumnRequest]:
         """
         Method to form the CreateColumnRequest object
@@ -211,13 +270,19 @@ class AlationsinkSource(Source):
                     table_name,
                     model_str(om_column.name),
                 ),
-                column_type=om_column.dataType.value.lower(),
-                title=model_str(om_column.name),
+                column_type=om_column.dataTypeDisplay.lower()
+                if om_column.dataTypeDisplay
+                else om_column.dataType.value.lower(),
+                title=om_column.displayName
+                if om_column.displayName
+                else model_str(om_column.name),
                 description=model_str(om_column.description),
                 position=str(om_column.ordinalPosition)
                 if om_column.ordinalPosition
                 else None,
-                index=self._get_column_index(om_column),
+                index=self._get_column_index(
+                    alation_datasource_id, om_column, table_constraints
+                ),
                 nullable=self._check_nullable_column(om_column),
             )
         except Exception as exc:
@@ -241,6 +306,7 @@ class AlationsinkSource(Source):
                     schema_name=schema_name,
                     table_name=model_str(om_table.name),
                     om_column=om_column,
+                    table_constraints=om_table.tableConstraints,
                 )
                 if create_column_request:
                     create_requests.root.append(create_column_request)
@@ -266,6 +332,7 @@ class AlationsinkSource(Source):
                     entity=Table,
                     skip_on_failure=True,
                     params={"database": model_str(om_schema.fullyQualifiedName)},
+                    fields=["tableConstraints, columns"],
                 )
             )
             create_requests = CreateTableRequestList(root=[])

metadata/ingestion/source/pipeline/dbtcloud/metadata.py

@@ -314,7 +314,7 @@ class DbtcloudSource(PipelineServiceSource):
                 timestamp=Timestamp(
                     datetime_to_ts(
                         datetime.strptime(
-                            pipeline_details.created_at, "%Y-%m-%dT%H:%M:%S.%f%z"
+                            pipeline_details.created_at, "%Y-%m-%d %H:%M:%S.%f%z"
                         )
                         if pipeline_details.created_at
                         else None

metadata/profiler/metrics/composed/null_ratio.py

@@ -50,8 +50,9 @@ class NullRatio(ComposedMetric):
         results of other Metrics
         """
 
-        count = res.get(Count.name())
-        null_count = res.get(NullCount.name())
-        if count + null_count == 0:
+        count = res.get(Count.name(), 0)
+        null_count = res.get(NullCount.name(), 0)
+        total = count + null_count
+        if total == 0:
             return None
-        return null_count / (null_count + count)
+        return null_count / total

metadata/profiler/orm/functions/table_metric_computer.py

@@ -176,6 +176,8 @@ class SnowflakeTableMetricComputer(BaseTableMetricComputer):
         )
 
         rest = self._runner._session.execute(query).first()
+        if not rest:
+            return None
         if rest.rowCount is None:
             # if we don't have any row count, fallback to the base logic
             return super().compute()

metadata/profiler/orm/types/custom_hex_byte_string.py

@@ -22,6 +22,7 @@ from sqlalchemy.sql.sqltypes import String, TypeDecorator
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
+NULL_BYTE = "\x00"
 
 
 class HexByteString(TypeDecorator):
@@ -63,10 +64,22 @@ class HexByteString(TypeDecorator):
         detected_encoding = chardet.detect(bytes_value).get("encoding")
         if detected_encoding:
             try:
-                value = bytes_value.decode(encoding=detected_encoding)
-                return value
+                # Decode the bytes value with the detected encoding and replace errors with "?"
+                # if bytes cannot be decoded e.g. b"\x66\x67\x67\x9c", if detected_encoding="utf-8"
+                # will result in 'foo�' (instead of failing)
+                str_value = bytes_value.decode(
+                    encoding=detected_encoding, errors="replace"
+                )
+                # Replace NULL_BYTE with empty string to avoid errors with
+                # the database client (should be O(n))
+                str_value = (
+                    str_value.replace(NULL_BYTE, "")
+                    if NULL_BYTE in str_value
+                    else str_value
+                )
+                return str_value
             except Exception as exc:
-                logger.debug("Failed to parse bytes valud as string: %s", exc)
+                logger.debug("Failed to parse bytes value as string: %s", exc)
                 logger.debug(traceback.format_exc())
 
         return value.hex()