openmetadata-ingestion 1.7.0.1__py3-none-any.whl → 1.7.1.1__py3-none-any.whl

metadata/generated/schema/type/dailyCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/dailyCount.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHierarchy.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 
@@ -20,6 +20,7 @@ class LabelType(Enum):
     Propagated = 'Propagated'
     Automated = 'Automated'
     Derived = 'Derived'
+    Generated = 'Generated'
 
 
 class State(Enum):

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-05-15T07:28:23+00:00
+#   timestamp: 2025-05-28T05:11:14+00:00
 
 from __future__ import annotations
 

metadata/ingestion/api/topology_runner.py

@@ -395,7 +395,11 @@ class TopologyRunnerMixin(Generic[C]):
             entity_source_hash = self.cache[stage.type_].get(entity_fqn)
             if entity_source_hash:
                 # if the source hash is present, compare it with new hash
-                if entity_source_hash != create_entity_request_hash:
+                # if overrideMetadata is true, we will always update the entity
+                if (
+                    entity_source_hash != create_entity_request_hash
+                    or self.source_config.overrideMetadata
+                ):
                     # the entity has changed, get the entity from server and make a patch request
                     entity = self.metadata.get_by_name(
                         entity=stage.type_,

metadata/ingestion/models/patch_request.py

@@ -369,6 +369,9 @@ def build_patch(
                 array_entity_fields=array_entity_fields,
             )
 
+        # special handler for tableConstraints
+        _table_constraints_handler(source, destination)
+
         # Get the difference between source and destination
         if allowed_fields:
             patch = jsonpatch.make_patch(
@@ -424,6 +427,69 @@ def build_patch(
         return None
 
 
+def _get_attribute_name(attr: T) -> str:
+    """Get the attribute name from the attribute."""
+    if hasattr(attr, "name"):
+        return model_str(attr.name)
+    return model_str(attr)
+
+
+def rearrange_attributes(final_attributes: List[T], source_attributes: List[T]):
+    source_staging_list = []
+    destination_staging_list = []
+    for attribute in final_attributes or []:
+        if attribute in source_attributes:
+            source_staging_list.append(attribute)
+        else:
+            destination_staging_list.append(attribute)
+    return source_staging_list + destination_staging_list
+
+
+def _table_constraints_handler(source: T, destination: T):
+    """
+    Handle table constraints patching properly.
+    This ensures we only perform allowed operations on constraints and maintain the structure.
+    """
+    if not hasattr(source, "tableConstraints") or not hasattr(
+        destination, "tableConstraints"
+    ):
+        return
+
+    source_table_constraints = getattr(source, "tableConstraints")
+    destination_table_constraints = getattr(destination, "tableConstraints")
+
+    if not source_table_constraints or not destination_table_constraints:
+        return
+
+    # Create a dictionary of source constraints for easy lookup
+    source_constraints_dict = {}
+    for constraint in source_table_constraints:
+        # Create a unique key based on constraintType and columns
+        key = f"{constraint.constraintType}:{','.join(sorted(constraint.columns))}"
+        source_constraints_dict[key] = constraint
+
+    # Rearrange destination constraints to match source order when possible
+    rearranged_constraints = []
+
+    # First add constraints that exist in both source and destination (preserving order from source)
+    for source_constraint in source_table_constraints:
+        key = f"{source_constraint.constraintType}:{','.join(sorted(source_constraint.columns))}"
+        for dest_constraint in destination_table_constraints:
+            dest_key = f"{dest_constraint.constraintType}:{','.join(sorted(dest_constraint.columns))}"
+            if key == dest_key:
+                rearranged_constraints.append(dest_constraint)
+                break
+
+    # Then add new constraints from destination that don't exist in source
+    for dest_constraint in destination_table_constraints:
+        dest_key = f"{dest_constraint.constraintType}:{','.join(sorted(dest_constraint.columns))}"
+        if dest_key not in source_constraints_dict:
+            rearranged_constraints.append(dest_constraint)
+
+    # Update the destination constraints with the rearranged list
+    setattr(destination, "tableConstraints", rearranged_constraints)
+
+
 def _sort_array_entity_fields(
     source: T,
     destination: T,
@@ -439,19 +505,21 @@ def _sort_array_entity_fields(
 
             # Create a dictionary of destination attributes for easy lookup
             destination_dict = {
-                model_str(attr.name): attr for attr in destination_attributes
+                _get_attribute_name(attr): attr for attr in destination_attributes
             }
 
             updated_attributes = []
             for source_attr in source_attributes or []:
                 # Update the destination attribute with the source attribute
-                destination_attr = destination_dict.get(model_str(source_attr.name))
+                destination_attr = destination_dict.get(
+                    _get_attribute_name(source_attr)
+                )
                 if destination_attr:
                     updated_attributes.append(
                         source_attr.model_copy(update=destination_attr.__dict__)
                     )
                     # Remove the updated attribute from the destination dictionary
-                    del destination_dict[model_str(source_attr.name)]
+                    del destination_dict[_get_attribute_name(source_attr)]
                 else:
                     updated_attributes.append(None)
 

metadata/ingestion/ometa/mixins/es_mixin.py

@@ -97,7 +97,7 @@ class ESMixin(Generic[T]):
 
     fqdn_search = (
         "/search/fieldQuery?fieldName={field_name}&fieldValue={field_value}&from={from_}"
-        "&size={size}&index={index}"
+        "&size={size}&index={index}&deleted=false"
     )
 
     # sort_field needs to be unique for the pagination to work, so we can use the FQN
@@ -125,14 +125,20 @@ class ESMixin(Generic[T]):
         if response:
             if fields:
                 fields = fields.split(",")
-            return [
-                self.get_by_name(
+
+            entities = []
+            for hit in response["hits"]["hits"]:
+                entity = self.get_by_name(
                     entity=entity_type,
                     fqn=hit["_source"]["fullyQualifiedName"],
                     fields=fields,
                 )
-                for hit in response["hits"]["hits"]
-            ] or None
+                if entity is None:
+                    continue
+
+                entities.append(entity)
+
+            return entities or None
 
         return None
 

metadata/ingestion/source/api/rest/metadata.py

@@ -197,8 +197,9 @@ class RestSource(ApiServiceSource):
     ) -> Optional[RESTEndpoint]:
         try:
             endpoint = RESTEndpoint(**info)
-            endpoint.name = f"{path}/{method_type}"
-            endpoint.display_name = f"{path}"
+            path_clean_name = clean_uri(path)
+            endpoint.name = f"{path_clean_name}/{method_type}"
+            endpoint.display_name = f"{path_clean_name}"
             endpoint.url = self._generate_endpoint_url(collection, endpoint)
             return endpoint
         except Exception as err:
@@ -279,6 +280,18 @@ class RestSource(ApiServiceSource):
                 .get("schema", {})
                 .get("$ref", {})
             )
+            if not schema_ref:
+                logger.debug("Trying to parse response schema from schema property")
+                schema_ref = (
+                    info.get("responses", {})
+                    .get("200", {})
+                    .get("content", {})
+                    .get("application/json", {})
+                    .get("schema", {})
+                    .get("properties", {})
+                    .get("data", {})
+                    .get("$ref")
+                )
             if not schema_ref:
                 logger.debug("No response schema found for the endpoint")
                 return None