openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl

metadata/data_quality/processor/test_case_runner.py

@@ -28,6 +28,10 @@ from metadata.data_quality.runner.test_suite_source_factory import (
     test_suite_source_factory,
 )
 from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
 )
@@ -35,7 +39,7 @@ from metadata.generated.schema.tests.testCase import TestCase
 from metadata.generated.schema.tests.testDefinition import TestDefinition, TestPlatform
 from metadata.generated.schema.tests.testSuite import TestSuite
 from metadata.generated.schema.type.basic import EntityLink, FullyQualifiedEntityName
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.parser import parse_workflow_config_gracefully
 from metadata.ingestion.api.step import Step
 from metadata.ingestion.api.steps import Processor
@@ -50,7 +54,6 @@ class TestCaseRunner(Processor):
     """Execute the test suite tests and create test cases from the YAML config"""
 
     def __init__(self, config: OpenMetadataWorkflowConfig, metadata: OpenMetadata):
-
         super().__init__()
 
         self.config = config
@@ -62,6 +65,10 @@ class TestCaseRunner(Processor):
             )
         )
 
+    @property
+    def name(self) -> str:
+        return "Data Quality"
+
     def _run(self, record: TableAndTests) -> Either:
         # First, create the executable test suite if it does not exist yet
         # This could happen if the process is executed from YAML and not the UI
@@ -193,8 +200,9 @@ class TestCaseRunner(Processor):
                         ),
                         entityLink=EntityLink(
                             __root__=entity_link.get_entity_link(
-                                table_fqn,
-                                test_case_to_create.columnName,
+                                Table,
+                                fqn=table_fqn,
+                                column_name=test_case_to_create.columnName,
                             )
                         ),
                         testSuite=test_suite_fqn,
@@ -215,7 +223,7 @@ class TestCaseRunner(Processor):
                     StackTraceError(
                         name=table_fqn,
                         error=error,
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -245,8 +253,9 @@ class TestCaseRunner(Processor):
                 updated_test_case = self.metadata.patch_test_case_definition(
                     source=test_case,
                     entity_link=entity_link.get_entity_link(
-                        table_fqn,
-                        test_case_definition.columnName,
+                        Table,
+                        fqn=table_fqn,
+                        column_name=test_case_definition.columnName,
                     ),
                     test_case_parameter_values=test_case_definition.parameterValues,
                 )
@@ -293,7 +302,7 @@ class TestCaseRunner(Processor):
                 StackTraceError(
                     name=test_case.name.__root__,
                     error=error,
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         return None

metadata/data_quality/source/test_suite.py

@@ -19,6 +19,9 @@ from typing import Iterable, List, Optional, cast
 from metadata.data_quality.api.models import TableAndTests
 from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest
 from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.testSuitePipeline import (
     TestSuitePipeline,
 )
@@ -27,7 +30,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 )
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.generated.schema.tests.testSuite import TestSuite
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.parser import parse_workflow_config_gracefully
 from metadata.ingestion.api.step import Step
 from metadata.ingestion.api.steps import Source
@@ -57,6 +60,10 @@ class TestSuiteSource(Source):
 
         self.test_connection()
 
+    @property
+    def name(self) -> str:
+        return "OpenMetadata"
+
     def _get_table_entity(self) -> Optional[Table]:
         """given an entity fqn return the table entity
 
@@ -76,11 +83,11 @@ class TestSuiteSource(Source):
     ) -> Optional[List[TestCase]]:
         """Return test cases if the test suite exists and has them"""
         if test_suite:
-            test_cases = self.metadata.list_entities(
+            test_cases = self.metadata.list_all_entities(
                 entity=TestCase,
                 fields=["testSuite", "entityLink", "testDefinition"],
                 params={"testSuiteId": test_suite.id.__root__},
-            ).entities
+            )
             test_cases = cast(List[TestCase], test_cases)  # satisfy type checker
 
             return test_cases
@@ -142,7 +149,6 @@ class TestSuiteSource(Source):
             )
 
         else:
-
             test_suite_cases = self._get_test_cases_from_test_suite(table.testSuite)
 
             yield Either(

metadata/data_quality/validations/base_test_handler.py

@@ -21,14 +21,11 @@ from datetime import datetime
 from typing import Callable, List, Optional, TypeVar, Union
 
 from metadata.generated.schema.tests.basic import (
-    TestCaseFailureStatus,
-    TestCaseFailureStatusType,
     TestCaseResult,
     TestCaseStatus,
     TestResultValue,
 )
 from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
-from metadata.generated.schema.type.basic import Timestamp
 from metadata.profiler.processor.runner import QueryRunner
 
 T = TypeVar("T", bound=Callable)
@@ -88,12 +85,15 @@ class BaseTestValidator(ABC):
         pre_processed_value = pre_processor(value)
         return type_(pre_processed_value)
 
-    def get_test_case_result_object(
+    def get_test_case_result_object(  # pylint: disable=too-many-arguments
         self,
         execution_date: Union[datetime, float],
         status: TestCaseStatus,
         result: str,
         test_result_value: List[TestResultValue],
+        row_count: Optional[int] = None,
+        failed_rows: Optional[int] = None,
+        passed_rows: Optional[int] = None,
     ) -> TestCaseResult:
         """Returns a TestCaseResult object with the given args
 
@@ -105,26 +105,30 @@ class BaseTestValidator(ABC):
         Returns:
             TestCaseResult:
         """
-        if status == TestCaseStatus.Failed:
-            test_case_failure_status = TestCaseFailureStatus(
-                testCaseFailureStatusType=TestCaseFailureStatusType.New,
-                testCaseFailureReason=None,
-                testCaseFailureComment=None,
-                updatedAt=Timestamp(__root__=int(datetime.utcnow().timestamp() * 1000)),
-                updatedBy=None,
-            )
-        else:
-            test_case_failure_status = None
-
-        return TestCaseResult(
+        test_case_result = TestCaseResult(
             timestamp=execution_date,  # type: ignore
             testCaseStatus=status,
             result=result,
             testResultValue=test_result_value,
             sampleData=None,
-            testCaseFailureStatus=test_case_failure_status,
         )
 
+        if (row_count is not None) and (
+            # we'll need at least one of these to be not None to compute the other
+            (failed_rows is not None)
+            or (passed_rows is not None)
+        ):
+            passed_rows = passed_rows if passed_rows is not None else (row_count - failed_rows)  # type: ignore
+            failed_rows = (
+                failed_rows if failed_rows is not None else (row_count - passed_rows)
+            )
+            test_case_result.passedRows = passed_rows
+            test_case_result.failedRows = failed_rows
+            test_case_result.passedRowsPercentage = (passed_rows / row_count) * 100
+            test_case_result.failedRowsPercentage = (failed_rows / row_count) * 100  # type: ignore
+
+        return test_case_result
+
     def format_column_list(self, status: TestCaseStatus, cols: List):
         """Format column list based on the test status
 

metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py

@@ -15,7 +15,7 @@ Validator for column value length to be between test case
 
 import traceback
 from abc import abstractmethod
-from typing import Union
+from typing import Tuple, Union
 
 from sqlalchemy import Column
 
@@ -65,6 +65,13 @@ class BaseColumnValueLengthsToBeBetweenValidator(BaseTestValidator):
         min_bound = self.get_min_bound("minLength")
         max_bound = self.get_max_bound("maxLength")
 
+        if self.test_case.computePassedFailedRowCount:
+            row_count, failed_rows = self.compute_row_count(
+                column, min_bound, max_bound
+            )
+        else:
+            row_count, failed_rows = None, None
+
         return self.get_test_case_result_object(
             self.execution_date,
             self.get_test_case_status(min_bound <= min_res and max_bound >= max_res),
@@ -74,6 +81,8 @@ class BaseColumnValueLengthsToBeBetweenValidator(BaseTestValidator):
                 TestResultValue(name=MIN, value=str(min_res)),
                 TestResultValue(name=MAX, value=str(max_res)),
             ],
+            row_count=row_count,
+            failed_rows=failed_rows,
         )
 
     @abstractmethod
@@ -83,3 +92,31 @@ class BaseColumnValueLengthsToBeBetweenValidator(BaseTestValidator):
     @abstractmethod
     def _run_results(self, metric: Metrics, column: Union[SQALikeColumn, Column]):
         raise NotImplementedError
+
+    @abstractmethod
+    def compute_row_count(
+        self, column: Union[SQALikeColumn, Column], min_bound, max_bound
+    ):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+            min_bound (_type_): min bound to filter out rows within the bound
+            max_bound (_type_): max bound to filter out rows within the bound
+
+        Raises:
+            NotImplementedError:
+        """
+        raise NotImplementedError
+
+    def get_row_count(self, min_bound, max_bound) -> Tuple[int, int]:
+        """Get row count
+
+        Args:
+            min_bound (_type_): min bound to filter out rows within the bound
+            max_bound (_type_): max bound to filter out rows within the bound
+
+        Returns:
+            Tuple[int, int]:
+        """
+        return self.compute_row_count(self._get_column_name(), min_bound, max_bound)

metadata/data_quality/validations/column/base/columnValuesToBeBetween.py

@@ -16,7 +16,7 @@ Validator for column values to be between test case
 import traceback
 from abc import abstractmethod
 from datetime import date, datetime, time
-from typing import Union
+from typing import Tuple, Union
 
 from sqlalchemy import Column
 
@@ -100,6 +100,11 @@ class BaseColumnValuesToBeBetweenValidator(BaseTestValidator):
             pre_processor=convert_timestamp if is_date_time(column.type) else None,
         )
 
+        if self.test_case.computePassedFailedRowCount:
+            row_count, failed_rows = self.get_row_count(min_bound, max_bound)
+        else:
+            row_count, failed_rows = None, None
+
         return self.get_test_case_result_object(
             self.execution_date,
             self.get_test_case_status(min_res >= min_bound and max_res <= max_bound),
@@ -108,6 +113,8 @@ class BaseColumnValuesToBeBetweenValidator(BaseTestValidator):
                 TestResultValue(name=MIN, value=str(min_res)),
                 TestResultValue(name=MAX, value=str(max_res)),
             ],
+            row_count=row_count,
+            failed_rows=failed_rows,
         )
 
     @abstractmethod
@@ -117,3 +124,31 @@ class BaseColumnValuesToBeBetweenValidator(BaseTestValidator):
     @abstractmethod
     def _run_results(self, metric: Metrics, column: Union[SQALikeColumn, Column]):
         raise NotImplementedError
+
+    @abstractmethod
+    def compute_row_count(
+        self, column: Union[SQALikeColumn, Column], min_bound, max_bound
+    ):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+            min_bound (_type_): min bound to filter out rows within the bound
+            max_bound (_type_): max bound to filter out rows within the bound
+
+        Raises:
+            NotImplementedError:
+        """
+        raise NotImplementedError
+
+    def get_row_count(self, min_bound, max_bound) -> Tuple[int, int]:
+        """Get row count
+
+        Args:
+            min_bound (_type_): min bound to filter out rows within the bound
+            max_bound (_type_): max bound to filter out rows within the bound
+
+        Returns:
+            Tuple[int, int]:
+        """
+        return self.compute_row_count(self._get_column_name(), min_bound, max_bound)

metadata/data_quality/validations/column/base/columnValuesToBeInSet.py

@@ -64,11 +64,18 @@ class BaseColumnValuesToBeInSetValidator(BaseTestValidator):
                 [TestResultValue(name=ALLOWED_VALUE_COUNT, value=None)],
             )
 
+        if self.test_case.computePassedFailedRowCount:
+            row_count = self.get_row_count()
+        else:
+            row_count = None
+
         return self.get_test_case_result_object(
             self.execution_date,
             self.get_test_case_status(res >= 1),
             f"Found countInSet={res}.",
             [TestResultValue(name=ALLOWED_VALUE_COUNT, value=str(res))],
+            row_count=row_count,
+            passed_rows=res,
         )
 
     @abstractmethod
@@ -80,3 +87,23 @@ class BaseColumnValuesToBeInSetValidator(BaseTestValidator):
         self, metric: Metrics, column: Union[SQALikeColumn, Column], **kwargs
     ):
         raise NotImplementedError
+
+    @abstractmethod
+    def compute_row_count(self, column: Union[SQALikeColumn, Column]):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        raise NotImplementedError
+
+    def get_row_count(self) -> int:
+        """Get row count
+
+        Returns:
+            Tuple[int, int]:
+        """
+        return self.compute_row_count(self._get_column_name())

metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py

@@ -70,11 +70,18 @@ class BaseColumnValuesToBeNotInSetValidator(BaseTestValidator):
                 [TestResultValue(name=COUNT_FORBIDDEN_VALUES, value=None)],
             )
 
+        if self.test_case.computePassedFailedRowCount:
+            row_count = self.get_row_count()
+        else:
+            row_count = None
+
         return self.get_test_case_result_object(
             self.execution_date,
             self.get_test_case_status(res == 0),
             f"Found countInSet={res}. It should be 0",
             [TestResultValue(name=COUNT_FORBIDDEN_VALUES, value=str(res))],
+            row_count=row_count,
+            failed_rows=res,
         )
 
     @abstractmethod
@@ -86,3 +93,23 @@ class BaseColumnValuesToBeNotInSetValidator(BaseTestValidator):
         self, metric: Metrics, column: Union[SQALikeColumn, Column], **kwargs
     ):
         raise NotImplementedError
+
+    @abstractmethod
+    def compute_row_count(self, column: Union[SQALikeColumn, Column]):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        raise NotImplementedError
+
+    def get_row_count(self) -> int:
+        """Get row count
+
+        Returns:
+            Tuple[int, int]:
+        """
+        return self.compute_row_count(self._get_column_name())

metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py

@@ -57,11 +57,18 @@ class BaseColumnValuesToBeNotNullValidator(BaseTestValidator):
                 [TestResultValue(name=NULL_COUNT, value=None)],
             )
 
+        if self.test_case.computePassedFailedRowCount:
+            row_count = self.get_row_count()
+        else:
+            row_count = None
+
         return self.get_test_case_result_object(
             self.execution_date,
             self.get_test_case_status(res == 0),
             f"Found nullCount={res}. It should be 0",
             [TestResultValue(name=NULL_COUNT, value=str(res))],
+            row_count=row_count,
+            failed_rows=res,
         )
 
     @abstractmethod
@@ -71,3 +78,23 @@ class BaseColumnValuesToBeNotNullValidator(BaseTestValidator):
     @abstractmethod
     def _run_results(self, metric: Metrics, column: Union[SQALikeColumn, Column]):
         raise NotImplementedError
+
+    @abstractmethod
+    def compute_row_count(self, column: Union[SQALikeColumn, Column]):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        raise NotImplementedError
+
+    def get_row_count(self) -> int:
+        """Get row count
+
+        Returns:
+            Tuple[int, int]:
+        """
+        return self.compute_row_count(self._get_column_name())

metadata/data_quality/validations/column/base/columnValuesToBeUnique.py

@@ -71,6 +71,8 @@ class BaseColumnValuesToBeUniqueValidator(BaseTestValidator):
                 TestResultValue(name=VALUE_COUNT, value=str(count)),
                 TestResultValue(name=UNIQUE_COUNT, value=str(unique_count)),
             ],
+            row_count=count,
+            passed_rows=unique_count,
         )
 
     @abstractmethod
@@ -79,8 +81,21 @@ class BaseColumnValuesToBeUniqueValidator(BaseTestValidator):
 
     @abstractmethod
     def _run_results(self, metric: Metrics, column: Union[SQALikeColumn, Column]):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
         raise NotImplementedError
 
     @abstractmethod
     def _get_unique_count(self, metric: Metrics, column: Union[SQALikeColumn, Column]):
+        """Get row count
+
+        Returns:
+            Tuple[int, int]:
+        """
         raise NotImplementedError

metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py

@@ -65,11 +65,18 @@ class BaseColumnValuesToMatchRegexValidator(BaseTestValidator):
                 [TestResultValue(name=LIKE_COUNT, value=None)],
             )
 
+        if self.test_case.computePassedFailedRowCount:
+            row_count = self.get_row_count()
+        else:
+            row_count = None
+
         return self.get_test_case_result_object(
             self.execution_date,
             self.get_test_case_status(count == match_count),
             f"Found {match_count} value(s) matching regex pattern vs {count} value(s) in the column.",
             [TestResultValue(name=LIKE_COUNT, value=str(match_count))],
+            row_count=row_count,
+            passed_rows=match_count,
         )
 
     @abstractmethod
@@ -81,3 +88,23 @@ class BaseColumnValuesToMatchRegexValidator(BaseTestValidator):
         self, metric: Metrics, column: Union[SQALikeColumn, Column], **kwargs
     ):
         raise NotImplementedError
+
+    @abstractmethod
+    def compute_row_count(self, column: Union[SQALikeColumn, Column]):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        raise NotImplementedError
+
+    def get_row_count(self) -> int:
+        """Get row count
+
+        Returns:
+            Tuple[int, int]:
+        """
+        return self.compute_row_count(self._get_column_name())

metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py

@@ -64,12 +64,19 @@ class BaseColumnValuesToNotMatchRegexValidator(BaseTestValidator):
                 [TestResultValue(name=NOT_LIKE_COUNT, value=None)],
             )
 
+        if self.test_case.computePassedFailedRowCount:
+            row_count = self.get_row_count()
+        else:
+            row_count = None
+
         return self.get_test_case_result_object(
             self.execution_date,
             self.get_test_case_status(not not_match_count),
             f"Found {not_match_count} value(s) matching the forbidden regex pattern vs "
             f"{not_match_count} value(s) in the column.",
             [TestResultValue(name=NOT_LIKE_COUNT, value=str(not_match_count))],
+            row_count=row_count,
+            failed_rows=not_match_count,
         )
 
     @abstractmethod
@@ -81,3 +88,23 @@ class BaseColumnValuesToNotMatchRegexValidator(BaseTestValidator):
         self, metric: Metrics, column: Union[SQALikeColumn, Column], **kwargs
     ):
         raise NotImplementedError
+
+    @abstractmethod
+    def compute_row_count(self, column: Union[SQALikeColumn, Column]):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        raise NotImplementedError
+
+    def get_row_count(self) -> int:
+        """Get row count
+
+        Returns:
+            Tuple[int, int]:
+        """
+        return self.compute_row_count(self._get_column_name())

metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py

@@ -50,3 +50,26 @@ class ColumnValueLengthsToBeBetweenValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column)
+
+    def compute_row_count(self, column: SQALikeColumn, min_bound: int, max_bound: int):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+            min_bound (_type_): min bound to filter out rows within the bound
+            max_bound (_type_): max bound to filter out rows within the bound
+
+        Raises:
+            NotImplementedError:
+        """
+        row_count = self._compute_row_count(self.runner, column)
+        failed_rows = sum(
+            len(
+                runner.query(
+                    f"`{column.name}`.str.len() > {max_bound} or `{column.name}`.str.len() < {min_bound}"
+                )
+            )
+            for runner in self.runner  # type: ignore
+        )
+
+        return row_count, failed_rows

metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py

@@ -49,3 +49,26 @@ class ColumnValuesToBeBetweenValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column)
+
+    def compute_row_count(self, column: SQALikeColumn, min_bound: int, max_bound: int):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+            min_bound (_type_): min bound to filter out rows within the bound
+            max_bound (_type_): max bound to filter out rows within the bound
+
+        Raises:
+            NotImplementedError:
+        """
+        row_count = self._compute_row_count(self.runner, column)
+        failed_rows = sum(
+            len(
+                runner.query(
+                    f"`{column.name}` > {max_bound} or `{column.name}` < {min_bound}"
+                )
+            )
+            for runner in self.runner  # type: ignore
+        )
+
+        return row_count, failed_rows

metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py

@@ -54,3 +54,14 @@ class ColumnValuesToBeInSetValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column, **kwargs)
+
+    def compute_row_count(self, column: SQALikeColumn):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py

@@ -51,3 +51,14 @@ class ColumnValuesToBeNotInSetValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column, **kwargs)
+
+    def compute_row_count(self, column: SQALikeColumn):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py

@@ -49,3 +49,14 @@ class ColumnValuesToBeNotNullValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column)
+
+    def compute_row_count(self, column: SQALikeColumn):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)