openmetadata-ingestion 1.5.11.0__py3-none-any.whl → 1.6.0.0rc1__py3-none-any.whl

metadata/data_quality/validations/column/base/columnValuesToBeInSet.py

@@ -45,6 +45,7 @@ class BaseColumnValuesToBeInSetValidator(BaseTestValidator):
         Returns:
             TestCaseResult:
         """
+        matched = False
         allowed_values = self.get_test_case_param_value(
             self.test_case.parameterValues,  # type: ignore
             "allowedValues",
@@ -58,11 +59,12 @@ class BaseColumnValuesToBeInSetValidator(BaseTestValidator):
         try:
             column: Union[SQALikeColumn, Column] = self._get_column_name()
             res = self._run_results(Metrics.COUNT_IN_SET, column, values=allowed_values)
+            matched = res > 0
             if match_enum:
                 count = self._run_results(
                     Metrics.ROW_COUNT, column, values=allowed_values
                 )
-                res = count - res
+                matched = count - res == 0
         except (ValueError, RuntimeError) as exc:
             msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}"  # type: ignore
             logger.debug(traceback.format_exc())
@@ -81,7 +83,7 @@ class BaseColumnValuesToBeInSetValidator(BaseTestValidator):
 
         return self.get_test_case_result_object(
             self.execution_date,
-            self.get_test_case_status(res == 0 if match_enum else res >= 1),
+            self.get_test_case_status(matched),
             f"Found countInSet={res}.",
             [TestResultValue(name=ALLOWED_VALUE_COUNT, value=str(res))],
             row_count=row_count,

metadata/data_quality/validations/models.py

@@ -4,13 +4,17 @@ from typing import List, Optional
 
 from pydantic import BaseModel
 
-from metadata.generated.schema.entity.data.table import Column
+from metadata.generated.schema.entity.data.table import Column, TableProfilerConfig
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseServiceType,
+)
 
 
 class TableParameter(BaseModel):
     serviceUrl: str
     path: str
     columns: List[Column]
+    database_service_type: DatabaseServiceType
 
 
 class TableDiffRuntimeParameters(BaseModel):
@@ -19,3 +23,4 @@ class TableDiffRuntimeParameters(BaseModel):
     keyColumns: List[str]
     extraColumns: List[str]
     whereClause: Optional[str]
+    table_profile_config: Optional[TableProfilerConfig]

metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py

@@ -77,7 +77,9 @@ class TableDiffParamsSetter(RuntimeParameterSetter):
         key_columns = self.get_key_columns(test_case)
         extra_columns = self.get_extra_columns(key_columns, test_case)
         return TableDiffRuntimeParameters(
+            table_profile_config=self.table_entity.tableProfilerConfig,
             table1=TableParameter(
+                database_service_type=service1.serviceType,
                 path=self.get_data_diff_table_path(
                     self.table_entity.fullyQualifiedName.root
                 ),
@@ -94,6 +96,7 @@ class TableDiffParamsSetter(RuntimeParameterSetter):
                 ),
             ),
             table2=TableParameter(
+                database_service_type=service2.serviceType,
                 path=self.get_data_diff_table_path(table2_fqn),
                 serviceUrl=self.get_data_diff_url(
                     service2,
@@ -118,8 +121,10 @@ class TableDiffParamsSetter(RuntimeParameterSetter):
         param_where_clause = self.get_parameter(test_case, "where", None)
         partition_where_clause = (
             None
-            if not self.sampler._partition_details
-            or not self.sampler._partition_details.enablePartitioning
+            if not (
+                self.sampler._partition_details
+                and self.sampler._partition_details.enablePartitioning
+            )
             else self.sampler.get_partitioned_query().whereclause.compile(
                 compile_kwargs={"literal_binds": True}
             )

metadata/data_quality/validations/table/sqlalchemy/tableDiff.py

@@ -10,8 +10,11 @@
 #  limitations under the License.
 # pylint: disable=missing-module-docstring
 import logging
+import random
+import string
 import traceback
 from decimal import Decimal
+from functools import reduce
 from itertools import islice
 from typing import Dict, Iterable, List, Optional, Tuple, cast
 from urllib.parse import urlparse
@@ -22,23 +25,34 @@ from data_diff.diff_tables import DiffResultWrapper
 from data_diff.errors import DataDiffMismatchingKeyTypesError
 from data_diff.utils import ArithAlphanumeric, CaseInsensitiveDict
 from sqlalchemy import Column as SAColumn
+from sqlalchemy import create_engine, literal, select
 
 from metadata.data_quality.validations import utils
 from metadata.data_quality.validations.base_test_handler import BaseTestValidator
 from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
     SQAValidatorMixin,
 )
-from metadata.data_quality.validations.models import TableDiffRuntimeParameters
-from metadata.generated.schema.entity.data.table import Column
+from metadata.data_quality.validations.models import (
+    TableDiffRuntimeParameters,
+    TableParameter,
+)
+from metadata.generated.schema.entity.data.table import Column, ProfileSampleType
 from metadata.generated.schema.entity.services.connections.database.sapHanaConnection import (
     SapHanaScheme,
 )
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseServiceType,
+)
 from metadata.generated.schema.tests.basic import (
     TestCaseResult,
     TestCaseStatus,
     TestResultValue,
 )
-from metadata.profiler.orm.registry import Dialects
+from metadata.profiler.orm.converter.base import build_orm_col
+from metadata.profiler.orm.functions.md5 import MD5
+from metadata.profiler.orm.functions.substr import Substr
+from metadata.profiler.orm.registry import Dialects, PythonDialects
+from metadata.utils.collections import CaseInsensitiveList
 from metadata.utils.logger import test_suite_logger
 
 logger = test_suite_logger()
@@ -57,6 +71,73 @@ SUPPORTED_DIALECTS = [
 ]
 
 
+def build_sample_where_clause(
+    table: TableParameter, key_columns: List[str], salt: str, hex_nounce: str
+) -> str:
+    sql_alchemy_columns = [
+        build_orm_col(i, c, table.database_service_type)
+        for i, c in enumerate(table.columns)
+        if c.name.root in key_columns
+    ]
+    reduced_concat = reduce(
+        lambda c1, c2: c1.concat(c2), sql_alchemy_columns + [literal(salt)]
+    )
+    sqa_dialect = create_engine(
+        f"{PythonDialects[table.database_service_type.name].value}://"
+    ).dialect
+    return str(
+        select()
+        .filter(
+            Substr(
+                MD5(reduced_concat),
+                1,
+                8,
+            )
+            < hex_nounce
+        )
+        .whereclause.compile(
+            dialect=sqa_dialect,
+            compile_kwargs={"literal_binds": True},
+        )
+    )
+
+
+def compile_and_clauses(elements) -> str:
+    """Compile a list of elements into a string with 'and' clauses.
+
+    Args:
+        elements: A string or a list of strings or lists
+
+    Returns:
+        A string with 'and' clauses
+
+    Raises:
+        ValueError: If the input is not a string or a list
+
+    Examples:
+        >>> compile_and_clauses("a")
+        'a'
+        >>> compile_and_clauses(["a", "b"])
+        'a and b'
+        >>> compile_and_clauses([["a", "b"], "c"])
+        '(a and b) and c'
+    """
+    if isinstance(elements, str):
+        return elements
+    if isinstance(elements, list):
+        if len(elements) == 1:
+            return compile_and_clauses(elements[0])
+        return " and ".join(
+            (
+                f"({compile_and_clauses(e)})"
+                if isinstance(e, list)
+                else compile_and_clauses(e)
+            )
+            for e in elements
+        )
+    raise ValueError("Input must be a string or a list")
+
+
 class UnsupportedDialectError(Exception):
     def __init__(self, param: str, dialect: str):
         super().__init__(f"Unsupported dialect in param {param}: {dialect}")
@@ -241,17 +322,20 @@ class TableDiffValidator(BaseTestValidator, SQAValidatorMixin):
 
     def get_table_diff(self) -> DiffResultWrapper:
         """Calls data_diff.diff_tables with the parameters from the test case."""
+        left_where, right_where = self.sample_where_clause()
         table1 = data_diff.connect_to_table(
             self.runtime_params.table1.serviceUrl,
             self.runtime_params.table1.path,
             self.runtime_params.keyColumns,  # type: ignore
             case_sensitive=self.get_case_sensitive(),
+            where=left_where,
         )
         table2 = data_diff.connect_to_table(
             self.runtime_params.table2.serviceUrl,
             self.runtime_params.table2.path,
             self.runtime_params.keyColumns,  # type: ignore
             case_sensitive=self.get_case_sensitive(),
+            where=right_where,
         )
         data_diff_kwargs = {
             "key_columns": self.runtime_params.keyColumns,
@@ -270,6 +354,90 @@ class TableDiffValidator(BaseTestValidator, SQAValidatorMixin):
         """Returns the where clause from the test case parameters or None if it is a blank string."""
         return self.runtime_params.whereClause or None
 
+    def sample_where_clause(self) -> Tuple[Optional[str], Optional[str]]:
+        """We use a where clause to sample the data for the diff. This is useful because with data diff
+        we do not have access to the underlying 'SELECT' statement. This method generates a where clause
+        that selects a random sample of the data based on the profile sample configuration.
+        The method uses the md5 hash of the key columns and a random salt to select a random sample of the data.
+        This ensures that the same data is selected from the two tables of the comparison.
+
+        Example:
+            -- Table 1 --  | -- Table 2 --
+            id | name      | id | name
+            1  | Alice     | 1  | Alice
+            2  | Bob       | 2  | Bob
+            3  | Charlie   | 3  | Charlie
+            4  | David     | 4  | Edward
+            5  | Edward    | 6  | Frank
+
+        If we want a sample of 20% of the data, the where clause will intend to select one of the rows
+        on Table 1 and the hash will ensure that the same row is selected on Table 2. We want to avoid selecting rows
+        with different ids because the comparison will not be sensible.
+        """
+        if (
+            # no sample configuration
+            self.runtime_params.table_profile_config is None
+            or self.runtime_params.table_profile_config.profileSample is None
+            # sample is 100% or in other words no sample is required
+            or (
+                self.runtime_params.table_profile_config.profileSampleType
+                == ProfileSampleType.PERCENTAGE
+                and self.runtime_params.table_profile_config.profileSample == 100
+            )
+        ):
+            return None, None
+        if DatabaseServiceType.Mssql in [
+            self.runtime_params.table1.database_service_type,
+            self.runtime_params.table2.database_service_type,
+        ]:
+            logger.warning("Sampling not supported in MSSQL. Skipping sampling.")
+            return None, None
+        nounce = self.calculate_nounce()
+        # SQL MD5 returns a 32 character hex string even with leading zeros so we need to
+        # pad the nounce to 8 characters in preserve lexical order.
+        # example: SELECT md5('j(R1wzR*y[^GxWJ5B>L{-HLETRD');
+        hex_nounce = hex(nounce)[2:].rjust(8, "0")
+        # TODO: using strings for this is sub-optimal. But using bytes buffers requires a by-database
+        # implementaiton. We can use this as default and add database specific implementations as the
+        # need arises.
+        salt = "".join(
+            random.choices(string.ascii_letters + string.digits, k=5)
+        )  # 1 / ~62^5 should be enough entropy. Use letters and digits to avoid messing with SQL syntax
+        key_columns = (
+            CaseInsensitiveList(self.runtime_params.keyColumns)
+            if not self.get_case_sensitive()
+            else self.runtime_params.keyColumns
+        )
+        return tuple(
+            build_sample_where_clause(table, key_columns, salt, hex_nounce)
+            for table in [self.runtime_params.table1, self.runtime_params.table2]
+        )
+
+    def calculate_nounce(self, max_nounce=2**32 - 1) -> int:
+        """Calculate the nounce based on the profile sample configuration. The nounce is
+        the sample fraction projected to a number on a scale of 0 to max_nounce"""
+        if (
+            self.runtime_params.table_profile_config.profileSampleType
+            == ProfileSampleType.PERCENTAGE
+        ):
+            return int(
+                max_nounce
+                * self.runtime_params.table_profile_config.profileSample
+                / 100
+            )
+        if (
+            self.runtime_params.table_profile_config.profileSampleType
+            == ProfileSampleType.ROWS
+        ):
+            row_count = self.get_row_count()
+            if row_count is None:
+                raise ValueError("Row count is required for ROWS profile sample type")
+            return int(
+                max_nounce
+                * (self.runtime_params.table_profile_config.profileSample / row_count)
+            )
+        raise ValueError("Invalid profile sample type")
+
     def get_runtime_params(self) -> TableDiffRuntimeParameters:
         raw = self.get_test_case_param_value(
             self.test_case.parameterValues, "runtimeParams", str
@@ -470,3 +638,7 @@ class TableDiffValidator(BaseTestValidator, SQAValidatorMixin):
         return utils.get_bool_test_case_param(
             self.test_case.parameterValues, "caseSensitiveColumns"
         )
+
+    def get_row_count(self) -> Optional[int]:
+        self.runner._sample = None  # pylint: disable=protected-access
+        return self._compute_row_count(self.runner, None)

metadata/examples/workflows/datafactory.yaml

@@ -0,0 +1,29 @@
+source:
+  type: datafactory
+  serviceName: local_datafactory
+  serviceConnection:
+    config:
+      type: DataFactory 
+      configSource: 
+        clientId: client_id
+        clientSecret: client_secret
+        tenantId: tenant_id
+        accountName: account_name
+      subscription_id: subscription_id
+      resource_group_name: resource_group_name
+      factory_name: factory_name
+      run_filter_days : 7
+  sourceConfig:
+    config:
+      type: PipelineMetadata
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: INFO # DEBUG, INFO, WARN or ERROR
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"      
+  

metadata/examples/workflows/matillion.yaml

@@ -0,0 +1,31 @@
+source:
+  type: Matillion
+  serviceName: local_Matillion_123
+  serviceConnection:
+    config:
+      type: Matillion
+      connection:
+        type: MatillionETL
+        hostPort: hostport
+        username: username
+        password: password
+        sslConfig:
+          caCertificate: |
+            -----BEGIN CERTIFICATE-----
+            sample certificate
+            -----END CERTIFICATE-----
+  sourceConfig:
+    config:
+      type: PipelineMetadata
+      includeLineage: true
+   
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG # DEBUG, INFO, WARN or ERROR
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"      

metadata/examples/workflows/rest.yaml

@@ -0,0 +1,20 @@
+source:
+  type: rest
+  serviceName: openapi_rest
+  serviceConnection:
+    config:
+      type: Rest
+      openAPISchemaURL: https://docs.open-metadata.org/swagger.json
+      # token: <jwt_token>
+  sourceConfig:
+    config:
+      type: ApiMetadata
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

metadata/examples/workflows/sigma.yaml

@@ -0,0 +1,25 @@
+source:
+  type: sigma
+  serviceName: local_sigma
+  serviceConnection:
+    config:
+      type: Sigma
+      hostPort: https://api.sigmacomputing.com
+      clientId: client_id
+      clientSecret: client_secret
+      apiVersion: v2
+  sourceConfig:
+    config:
+      type: DashboardMetadata
+      lineageInformation:
+        dbServiceNames: [db_service_name]
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG # DEBUG, INFO, WARN or ERROR
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

metadata/examples/workflows/trino_profiler.yaml

@@ -0,0 +1,20 @@
+source:
+  type: trino
+  serviceName: local_trino
+  sourceConfig:
+    config:
+      type: Profiler
+      # useStatistics: true # Optional, use statistics to profile tables, default is false
+processor:
+  type: "orm-profiler"
+  config: {}
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+#  loggerLevel: INFO # DEBUG, INFO, WARN or ERROR
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"