openmetadata-ingestion 1.8.5.0__py3-none-any.whl → 1.8.7.0__py3-none-any.whl

metadata/generated/schema/type/entityHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHierarchy.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-07-17T00:16:43+00:00
+#   timestamp: 2025-07-24T06:26:23+00:00
 
 from __future__ import annotations
 

metadata/great_expectations/action.py

@@ -21,6 +21,7 @@ from datetime import datetime
 from typing import Dict, List, Optional, Union, cast
 
 from great_expectations.checkpoint.actions import ValidationAction
+from great_expectations.core import ExpectationConfiguration
 from great_expectations.core.batch import Batch
 from great_expectations.core.batch_spec import (
     RuntimeDataBatchSpec,
@@ -111,6 +112,7 @@ class OpenMetadataValidationAction(ValidationAction):
         self.schema_name = schema_name  # for database without schema concept
         self.config_file_path = config_file_path
         self.ometa_conn = self._create_ometa_connection()
+        self.expectation_suite = None
 
     def _run(  # pylint: disable=unused-argument
         self,
@@ -133,6 +135,12 @@ class OpenMetadataValidationAction(ValidationAction):
             checkpoint_identifier: identifier for the checkpoint
         """
 
+        if expectation_suite_identifier:
+            expectation_suite_name = expectation_suite_identifier.expectation_suite_name
+            self.expectation_suite = self.data_context.get_expectation_suite(
+                expectation_suite_name
+            )
+
         check_point_spec = self._get_checkpoint_batch_spec(data_asset)
         table_entity = None
         if isinstance(check_point_spec, SqlAlchemyDatasourceBatchSpec):
@@ -314,8 +322,30 @@ class OpenMetadataValidationAction(ValidationAction):
         fqn_ = cast(str, fqn_)
         return fqn_
 
+    def _get_test_case_description(self, result: dict) -> str:
+        """Get test case description from GE test result"""
+        if self.expectation_suite:
+            expectation = self._get_expectation_config(result)
+            if expectation:
+                meta: Optional[Dict] = expectation.get("meta")
+                if meta:
+                    return meta.get("description", "")
+        return ""
+
     def _get_test_case_params_value(self, result: dict) -> List[TestCaseParameterValue]:
         """Build test case parameter value from GE test result"""
+        if self.expectation_suite:
+            expectation = self._get_expectation_config(result)
+            if expectation:
+                return [
+                    TestCaseParameterValue(
+                        name=key,
+                        value=str(value),
+                    )  # type: ignore
+                    for key, value in expectation.kwargs.items()  # type: ignore
+                    if key not in {"column", "batch_id"}
+                ]
+
         if "observed_value" not in result["result"]:
             return [
                 TestCaseParameterValue(
@@ -337,6 +367,17 @@ class OpenMetadataValidationAction(ValidationAction):
         self, result: dict
     ) -> List[TestCaseParameterDefinition]:
         """Build test case parameter definition from GE test result"""
+        if self.expectation_suite:
+            expectation = self._get_expectation_config(result)
+            if expectation:
+                return [
+                    TestCaseParameterDefinition(
+                        name=key,
+                    )  # type: ignore
+                    for key, _ in expectation.kwargs.items()  # type: ignore
+                    if key not in {"column", "batch_id"}
+                ]
+
         if "observed_value" not in result["result"]:
             return [
                 TestCaseParameterDefinition(
@@ -361,19 +402,118 @@ class OpenMetadataValidationAction(ValidationAction):
         Returns:
             TestCaseResult: a test case result object
         """
-        try:
-            test_result_value = TestResultValue(
-                name="observed_value",
-                value=str(result["result"]["observed_value"]),
-            )
-        except KeyError:
-            unexpected_percent_total = result["result"].get("unexpected_percent_total")
-            test_result_value = TestResultValue(
-                name="unexpected_percentage_total",
-                value=str(unexpected_percent_total),
+        test_result_values = []
+        result_data = result.get("result", {})
+
+        numeric_fields = [
+            "unexpected_percent",
+            "unexpected_percent_total",
+            "missing_percent",
+            "unexpected_count",
+            "missing_count",
+            "element_count",
+            "observed_value",
+            "success_rate",
+        ]
+
+        for field in numeric_fields:
+            if field in result_data:
+                value = result_data[field]
+
+                if isinstance(value, (int, float)):
+                    test_result_values.append(
+                        TestResultValue(
+                            name=field,
+                            value=str(value),
+                            predictedValue=None,
+                        )
+                    )
+                elif field == "observed_value":
+                    test_result_values.extend(
+                        self._extract_complex_value_from_observed_value(value)
+                    )
+
+        return test_result_values
+
+    def _extract_complex_value_from_observed_value(
+        self, observed_value
+    ) -> List[TestResultValue]:
+        """Extract complex value from observed value
+
+        Args:
+            observed_value: observed value
+        Returns:
+            str: complex value
+        """
+        if isinstance(observed_value, list):
+            return [
+                TestResultValue(
+                    name="element_count",
+                    value=str(len(observed_value)),
+                    predictedValue=None,
+                )
+            ]
+
+        if isinstance(observed_value, dict):
+            if "quantiles" in observed_value:
+                result_values = []
+                quantiles = observed_value["quantiles"]
+                values = observed_value["values"]
+                for quantile, value in zip(quantiles, values):
+                    result_values.append(
+                        TestResultValue(
+                            name=f"quantile_{str(quantile)}",
+                            value=str(value),
+                            predictedValue=None,
+                        )
+                    )
+                return result_values
+
+            # catch all other cases that are not a quantile
+            for k, v in observed_value.items():
+                if isinstance(v, (int, float)):
+                    return [
+                        TestResultValue(
+                            name=k,
+                            value=str(v),
+                            predictedValue=None,
+                        )
+                    ]
+
+        if isinstance(observed_value, str):
+            return [
+                TestResultValue(
+                    name="observed_value",
+                    value=str(1),
+                    predictedValue=None,
+                )
+            ]
+
+        return []
+
+    def _get_expectation_config(
+        self, result: dict
+    ) -> Optional[ExpectationConfiguration]:
+        """Get expectation config from GE test result
+
+        Args:
+            result: GE test result
+        Returns:
+            Optional[ExpectationConfiguration]: expectation configuration
+        """
+        if self.expectation_suite:
+            name = result["expectation_config"]["expectation_type"]
+            expectation = next(
+                (
+                    expectation
+                    for expectation in self.expectation_suite.expectations
+                    if expectation.expectation_type == name
+                ),
+                None,
             )
 
-        return [test_result_value]
+            return expectation
+        return None
 
     def _handle_test_case(self, result: Dict, table_entity: Table):
         """Handle adding test to table entity based on the test case.
@@ -415,6 +555,7 @@ class OpenMetadataValidationAction(ValidationAction):
                 ),
                 test_definition_fqn=test_definition.fullyQualifiedName.root,
                 test_case_parameter_values=self._get_test_case_params_value(result),
+                description=self._get_test_case_description(result),
             )
 
             self.ometa_conn.add_test_case_results(

metadata/great_expectations/action1xx.py

@@ -319,19 +319,94 @@ class OpenMetadataValidationAction1xx(ValidationAction):
         Returns:
             TestCaseResult: a test case result object
         """
-        try:
-            test_result_value = TestResultValue(
-                name="observed_value",
-                value=str(result["result"]["observed_value"]),
-            )
-        except KeyError:
-            unexpected_percent_total = result["result"].get("unexpected_percent_total")
-            test_result_value = TestResultValue(
-                name="unexpected_percentage_total",
-                value=str(unexpected_percent_total),
-            )
+        test_result_values = []
+        result_data = result.get("result", {})
+
+        numeric_fields = [
+            "unexpected_percent",
+            "unexpected_percent_total",
+            "missing_percent",
+            "unexpected_count",
+            "missing_count",
+            "element_count",
+            "observed_value",
+            "success_rate",
+        ]
+
+        for field in numeric_fields:
+            if field in result_data:
+                value = result_data[field]
+
+                if isinstance(value, (int, float)):
+                    test_result_values.append(
+                        TestResultValue(
+                            name=field,
+                            value=str(value),
+                            predictedValue=None,
+                        )
+                    )
+                elif field == "observed_value":
+                    test_result_values.extend(
+                        self._extract_complex_value_from_observed_value(value)
+                    )
+
+        return test_result_values
+
+    def _extract_complex_value_from_observed_value(
+        self, observed_value
+    ) -> List[TestResultValue]:
+        """Extract complex value from observed value
+
+        Args:
+            observed_value: observed value
+        Returns:
+            str: complex value
+        """
+        if isinstance(observed_value, list):
+            return [
+                TestResultValue(
+                    name="element_count",
+                    value=str(len(observed_value)),
+                    predictedValue=None,
+                )
+            ]
+
+        if isinstance(observed_value, dict):
+            if "quantiles" in observed_value:
+                result_values = []
+                quantiles = observed_value["quantiles"]
+                values = observed_value["values"]
+                for quantile, value in zip(quantiles, values):
+                    result_values.append(
+                        TestResultValue(
+                            name=f"quantile_{str(quantile)}",
+                            value=str(value),
+                            predictedValue=None,
+                        )
+                    )
+                return result_values
+
+            # catch all other cases that are not a quantile
+            for k, v in observed_value.items():
+                if isinstance(v, (int, float)):
+                    return [
+                        TestResultValue(
+                            name=k,
+                            value=str(v),
+                            predictedValue=None,
+                        )
+                    ]
+
+        if isinstance(observed_value, str):
+            return [
+                TestResultValue(
+                    name="observed_value",
+                    value=str(1),
+                    predictedValue=None,
+                )
+            ]
 
-        return [test_result_value]
+        return []
 
     def _handle_test_case(self, result: Dict, table_entity: Table):
         """Handle adding test to table entity based on the test case.

metadata/ingestion/lineage/sql_lineage.py

@@ -14,7 +14,7 @@ Helper functions to handle SQL lineage operations
 import itertools
 import traceback
 from collections import defaultdict
-from typing import Any, Iterable, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 
 import networkx as nx
 from collate_sqllineage.core.models import Column, DataFunction
@@ -894,7 +894,7 @@ def _get_paths_from_subtree(subtree: DiGraph) -> List[List[Any]]:
 
 
 def get_lineage_by_graph(
-    graph: DiGraph,
+    graph: Optional[DiGraph],
     metadata: OpenMetadata,
 ) -> Iterable[Either[AddLineageRequest]]:
     """
@@ -922,3 +922,27 @@ def get_lineage_by_graph(
         subtree = graph.subgraph(component).copy()
         for path in _get_paths_from_subtree(subtree):
             yield from _process_sequence(path, subtree, metadata)
+
+
+def get_lineage_by_procedure_graph(
+    procedure_graph_map: Optional[Dict],
+    metadata: OpenMetadata,
+) -> Iterable[Either[AddLineageRequest]]:
+    """
+    Generate lineage information from a directed graph.
+    """
+    if procedure_graph_map is None:
+        return
+
+    for procedure_and_procedure_graph in procedure_graph_map.values():
+        for either_lineage in get_lineage_by_graph(
+            graph=procedure_and_procedure_graph.graph,
+            metadata=metadata,
+        ):
+            if either_lineage.left is None and either_lineage.right.edge.lineageDetails:
+                either_lineage.right.edge.lineageDetails.pipeline = EntityReference(
+                    id=procedure_and_procedure_graph.procedure.id,
+                    type="storedProcedure",
+                )
+
+            yield either_lineage

metadata/ingestion/ometa/mixins/tests_mixin.py

@@ -169,6 +169,7 @@ class OMetaTestsMixin:
         entity_link: Optional[str] = None,
         test_definition_fqn: Optional[str] = None,
         test_case_parameter_values: Optional[List[TestCaseParameterValue]] = None,
+        description: Optional[str] = None,
     ):
         """Get or create a test case
 
@@ -197,6 +198,7 @@ class OMetaTestsMixin:
                 entityLink=entity_link,
                 testDefinition=test_definition_fqn,
                 parameterValues=test_case_parameter_values,
+                description=description,
             )  # type: ignore
         )
         return test_case

metadata/ingestion/ometa/ometa_api.py

@@ -15,7 +15,7 @@ models from the JSON schemas and provides a typed approach to
 working with OpenMetadata entities.
 """
 import traceback
-from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union
+from typing import Any, Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union
 
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -145,6 +145,7 @@ class OpenMetadata(
         self,
         config: OpenMetadataConnection,
         raw_data: bool = False,
+        additional_client_config_arguments: Optional[Dict[str, Any]] = None,
     ):
         self.config = config
 
@@ -161,14 +162,17 @@ class OpenMetadata(
         extra_headers: Optional[dict[str, str]] = None
         if self.config.extraHeaders:
             extra_headers = self.config.extraHeaders.root
-        client_config: ClientConfig = ClientConfig(
+
+        client_config = ClientConfig(
             base_url=self.config.hostPort,
             api_version=self.config.apiVersion,
             auth_header="Authorization",
             extra_headers=extra_headers,
             auth_token=self._auth_provider.get_access_token,
             verify=get_verify_ssl(self.config.sslConfig),
+            **(additional_client_config_arguments or {}),
         )
+
         self.client = REST(client_config)
         self._use_raw_data = raw_data
         if self.config.enableVersionValidation: