openmetadata-ingestion 1.8.9.0__py3-none-any.whl → 1.9.0.0__py3-none-any.whl

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/layerPaging.py

@@ -0,0 +1,24 @@
+# generated by datamodel-codegen:
+#   filename:  type/layerPaging.json
+#   timestamp: 2025-08-08T13:50:05+00:00
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+
+class LayerPaging(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    entityUpstreamCount: Annotated[
+        Optional[int], Field(None, description='Total Upstream Entity Count')
+    ]
+    entityDownstreamCount: Annotated[
+        Optional[int], Field(None, description='Total Downstream Entity Count')
+    ]

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/personaPreferences.py

@@ -0,0 +1,48 @@
+# generated by datamodel-codegen:
+#   filename:  type/personaPreferences.json
+#   timestamp: 2025-08-08T13:50:05+00:00
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from . import basic
+
+
+class LandingPageSettings(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    headerColor: Annotated[
+        Optional[str],
+        Field(None, description='Custom header background color for the landing page.'),
+    ]
+    headerImage: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='Reference to a custom header background image (reserved for future use).',
+        ),
+    ]
+
+
+class PersonaPreferences(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    personaId: Annotated[
+        basic.Uuid,
+        Field(description='UUID of the persona these preferences belong to.'),
+    ]
+    personaName: Annotated[
+        str, Field(description='Name of the persona for quick reference and linking.')
+    ]
+    landingPageSettings: Annotated[
+        Optional[LandingPageSettings],
+        Field(None, description="User's personal customizations for the landing page."),
+    ]

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-08-08T13:50:05+00:00
 
 from __future__ import annotations
 

metadata/ingestion/api/parser.py

@@ -30,6 +30,10 @@ from metadata.generated.schema.entity.services.databaseService import (
     DatabaseConnection,
     DatabaseServiceType,
 )
+from metadata.generated.schema.entity.services.driveService import (
+    DriveConnection,
+    DriveServiceType,
+)
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
     IngestionPipeline,
 )
@@ -103,6 +107,10 @@ from metadata.generated.schema.metadataIngestion.dbtPipeline import (
     DbtConfigType,
     DbtPipeline,
 )
+from metadata.generated.schema.metadataIngestion.driveServiceMetadataPipeline import (
+    DriveMetadataConfigType,
+    DriveServiceMetadataPipeline,
+)
 from metadata.generated.schema.metadataIngestion.messagingServiceMetadataPipeline import (
     MessagingMetadataConfigType,
     MessagingServiceMetadataPipeline,
@@ -153,6 +161,7 @@ SERVICE_TYPE_MAP = {
     **{service: StorageConnection for service in StorageServiceType.__members__},
     **{service: SearchConnection for service in SearchServiceType.__members__},
     **{service: SecurityConnection for service in SecurityServiceType.__members__},
+    **{service: DriveConnection for service in DriveServiceType.__members__},
 }
 
 SOURCE_CONFIG_CLASS_MAP = {
@@ -167,6 +176,7 @@ SOURCE_CONFIG_CLASS_MAP = {
     StorageMetadataConfigType.StorageMetadata.value: StorageServiceMetadataPipeline,
     SearchMetadataConfigType.SearchMetadata.value: SearchServiceMetadataPipeline,
     SecurityMetadataConfigType.SecurityMetadata.value: SecurityServiceMetadataPipeline,
+    DriveMetadataConfigType.DriveMetadata.value: DriveServiceMetadataPipeline,
     DbtConfigType.DBT.value: DbtPipeline,
 }
 
@@ -200,6 +210,7 @@ def get_service_type(
     Type[MetadataConnection],
     Type[PipelineConnection],
     Type[MlModelConnection],
+    Type[DriveConnection],
 ]:
     """
     Return the service type for a source string
@@ -225,6 +236,7 @@ def get_source_config_class(
     Type[PipelineServiceMetadataPipeline],
     Type[MlModelServiceMetadataPipeline],
     Type[DatabaseServiceMetadataPipeline],
+    Type[DriveServiceMetadataPipeline],
     Type[DbtPipeline],
 ]:
     """
@@ -250,6 +262,7 @@ def get_connection_class(
         Type[MetadataConnection],
         Type[PipelineConnection],
         Type[MlModelConnection],
+        Type[DriveConnection],
     ],
 ) -> Type[T]:
     """

metadata/ingestion/connections/connection.py

@@ -41,12 +41,23 @@ class BaseConnection(ABC, Generic[S, C]):
     """
 
     service_connection: S
+    _client: Optional[C]
 
     def __init__(self, service_connection: S) -> None:
         self.service_connection = service_connection
+        self._client = None
+
+    @property
+    def client(self) -> C:
+        """
+        Return the main client/engine/connection object for this service.
+        """
+        if self._client is None:
+            self._client = self._get_client()
+        return self._client
 
     @abstractmethod
-    def get_client(self) -> C:
+    def _get_client(self) -> C:
         """
         Return the main client/engine/connection object for this service.
         """
@@ -61,3 +72,9 @@ class BaseConnection(ABC, Generic[S, C]):
         """
         Test the connection to the service.
         """
+
+    @abstractmethod
+    def get_connection_dict(self) -> dict:
+        """
+        Return the connection dictionary for this service.
+        """

metadata/ingestion/lineage/sql_lineage.py

@@ -364,6 +364,7 @@ def get_table_entities_from_query(
     database_name: str,
     database_schema: str,
     table_name: str,
+    schema_fallback: bool = False,
 ) -> Optional[List[Table]]:
     """
     Fetch data from API and ES with a fallback strategy.
@@ -394,6 +395,17 @@ def get_table_entities_from_query(
     if table_entities:
         return table_entities
 
+    if schema_fallback:
+        table_entities = search_table_entities(
+            metadata=metadata,
+            service_name=service_name,
+            database=database_query if database_query else database_name,
+            database_schema=None,
+            table=table,
+        )
+        if table_entities:
+            return table_entities
+
     return None
 
 
@@ -516,6 +528,7 @@ def _create_lineage_by_table_name(
     lineage_source: LineageSource = LineageSource.QueryLineage,
     procedure: Optional[EntityReference] = None,
     graph: DiGraph = None,
+    schema_fallback: bool = False,
 ) -> Iterable[Either[AddLineageRequest]]:
     """
     This method is to create a lineage between two tables
@@ -527,6 +540,7 @@ def _create_lineage_by_table_name(
             database_name=database_name,
             database_schema=schema_name,
             table_name=from_table,
+            schema_fallback=schema_fallback,
         )
 
         to_table_entities = get_table_entities_from_query(
@@ -535,6 +549,7 @@ def _create_lineage_by_table_name(
             database_name=database_name,
             database_schema=schema_name,
             table_name=to_table,
+            schema_fallback=schema_fallback,
         )
 
         for table_name, entity in (
@@ -633,6 +648,7 @@ def get_lineage_by_query(
     timeout_seconds: int = LINEAGE_PARSING_TIMEOUT,
     lineage_source: LineageSource = LineageSource.QueryLineage,
     graph: DiGraph = None,
+    schema_fallback: bool = False,
 ) -> Iterable[Either[AddLineageRequest]]:
     """
     This method parses the query to get source, target and intermediate table names to create lineage,
@@ -673,6 +689,7 @@ def get_lineage_by_query(
                         lineage_source=lineage_source,
                         procedure=procedure,
                         graph=graph,
+                        schema_fallback=schema_fallback,
                     )
             for target_table in lineage_parser.target_tables:
                 yield from _create_lineage_by_table_name(
@@ -685,6 +702,7 @@ def get_lineage_by_query(
                     masked_query=masked_query,
                     column_lineage_map=column_lineage,
                     lineage_source=lineage_source,
+                    schema_fallback=schema_fallback,
                 )
         if not lineage_parser.intermediate_tables:
             for target_table in lineage_parser.target_tables:
@@ -711,6 +729,7 @@ def get_lineage_by_query(
                             lineage_source=lineage_source,
                             procedure=procedure,
                             graph=graph,
+                            schema_fallback=schema_fallback,
                         )
         if not lineage_parser.query_parsing_success:
             query_parsing_failures.add(
@@ -740,6 +759,7 @@ def get_lineage_via_table_entity(
     timeout_seconds: int = LINEAGE_PARSING_TIMEOUT,
     lineage_source: LineageSource = LineageSource.QueryLineage,
     graph: DiGraph = None,
+    schema_fallback: bool = False,
 ) -> Iterable[Either[AddLineageRequest]]:
     """Get lineage from table entity"""
     column_lineage = {}
@@ -776,6 +796,7 @@ def get_lineage_via_table_entity(
                     lineage_source=lineage_source,
                     procedure=procedure,
                     graph=graph,
+                    schema_fallback=schema_fallback,
                 ) or []
         if not lineage_parser.query_parsing_success:
             query_parsing_failures.add(

metadata/ingestion/models/custom_properties.py

@@ -28,13 +28,13 @@ class CustomPropertyDataTypes(Enum):
     STRING = "string"
     INTEGER = "integer"
     MARKDOWN = "markdown"
-    DATE = "date"
-    DATETIME = "dateTime"
+    DATE = "date-cp"
+    DATETIME = "dateTime-cp"
     DURATION = "duration"
     EMAIL = "email"
     NUMBER = "number"
     SQLQUERY = "sqlQuery"
-    TIME = "time"
+    TIME = "time-cp"
     TIMEINTERVAL = "timeInterval"
     TIMESTAMP = "timestamp"
     ENUM = "enum"

metadata/ingestion/models/patch_request.py

@@ -342,6 +342,7 @@ def build_patch(
     array_entity_fields: Optional[List] = None,
     remove_change_description: bool = True,
     override_metadata: Optional[bool] = False,
+    skip_on_failure: Optional[bool] = True,
 ) -> Optional[jsonpatch.JsonPatch]:
     """
     Given an Entity type and Source entity and Destination entity,
@@ -352,6 +353,10 @@ def build_patch(
         destination: payload with changes applied to the source.
         allowed_fields: List of field names to filter from source and destination models
         restrict_update_fields: List of field names which will only support add operation
+        array_entity_fields: List of array fields to sort for consistent patching
+        remove_change_description: Whether to remove change description from entities
+        override_metadata: Whether to override existing metadata fields
+        skip_on_failure: Whether to skip the patch operation on failure (default: True)
 
     Returns
         Updated Entity
@@ -421,10 +426,37 @@ def build_patch(
             patch.patch = updated_operations
 
         return patch
-    except Exception:
+    except Exception as exc:
         logger.debug(traceback.format_exc())
-        logger.warning("Couldn't build patch for Entity.")
-        return None
+        if skip_on_failure:
+            entity_info = ""
+            try:
+                if hasattr(source, "fullyQualifiedName"):
+                    entity_info = f" for '{source.fullyQualifiedName.root}'"
+                elif hasattr(source, "name"):
+                    entity_info = f" for '{source.name.root}'"
+            except Exception:
+                pass
+
+            logger.warning(
+                f"Failed to build patch{entity_info}. The patch generation was skipped. "
+                f"Reason: {exc}"
+            )
+            return None
+        else:
+            entity_info = ""
+            try:
+                if hasattr(source, "fullyQualifiedName"):
+                    entity_info = f" for '{source.fullyQualifiedName.root}'"
+                elif hasattr(source, "name"):
+                    entity_info = f" for '{source.name.root}'"
+            except Exception:
+                pass
+
+            raise RuntimeError(
+                f"Failed to build patch{entity_info}. The patch generation failed. "
+                f"Set 'skip_on_failure=True' to skip failed patch operations. Error: {exc}"
+            ) from exc
 
 
 def _get_attribute_name(attr: T) -> str:
@@ -534,7 +566,7 @@ def _remove_change_description(entity: T) -> T:
     We never want to patch that, and we won't have that information
     from the source. It's fully handled in the server.
     """
-    if getattr(entity, "changeDescription"):
+    if hasattr(entity, "changeDescription") and getattr(entity, "changeDescription"):
         entity.changeDescription = None
 
     return entity