openmetadata-ingestion 1.2.5.1__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py

@@ -50,3 +50,26 @@ class ColumnValueLengthsToBeBetweenValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column)
+
+    def compute_row_count(self, column: SQALikeColumn, min_bound: int, max_bound: int):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+            min_bound (_type_): min bound to filter out rows within the bound
+            max_bound (_type_): max bound to filter out rows within the bound
+
+        Raises:
+            NotImplementedError:
+        """
+        row_count = self._compute_row_count(self.runner, column)
+        failed_rows = sum(
+            len(
+                runner.query(
+                    f"`{column.name}`.str.len() > {max_bound} or `{column.name}`.str.len() < {min_bound}"
+                )
+            )
+            for runner in self.runner  # type: ignore
+        )
+
+        return row_count, failed_rows

metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py

@@ -49,3 +49,26 @@ class ColumnValuesToBeBetweenValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column)
+
+    def compute_row_count(self, column: SQALikeColumn, min_bound: int, max_bound: int):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+            min_bound (_type_): min bound to filter out rows within the bound
+            max_bound (_type_): max bound to filter out rows within the bound
+
+        Raises:
+            NotImplementedError:
+        """
+        row_count = self._compute_row_count(self.runner, column)
+        failed_rows = sum(
+            len(
+                runner.query(
+                    f"`{column.name}` > {max_bound} or `{column.name}` < {min_bound}"
+                )
+            )
+            for runner in self.runner  # type: ignore
+        )
+
+        return row_count, failed_rows

metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py

@@ -54,3 +54,14 @@ class ColumnValuesToBeInSetValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column, **kwargs)
+
+    def compute_row_count(self, column: SQALikeColumn):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py

@@ -51,3 +51,14 @@ class ColumnValuesToBeNotInSetValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column, **kwargs)
+
+    def compute_row_count(self, column: SQALikeColumn):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py

@@ -49,3 +49,14 @@ class ColumnValuesToBeNotNullValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column)
+
+    def compute_row_count(self, column: SQALikeColumn):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py

@@ -51,3 +51,14 @@ class ColumnValuesToMatchRegexValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column, **kwargs)
+
+    def compute_row_count(self, column: SQALikeColumn):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py

@@ -51,3 +51,14 @@ class ColumnValuesToNotMatchRegexValidator(
             column: column
         """
         return self.run_dataframe_results(self.runner, metric, column, **kwargs)
+
+    def compute_row_count(self, column: SQALikeColumn):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py

@@ -25,6 +25,7 @@ from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
     SQAValidatorMixin,
 )
 from metadata.profiler.metrics.registry import Metrics
+from metadata.profiler.orm.functions.length import LenFn
 
 
 class ColumnValueLengthsToBeBetweenValidator(
@@ -51,3 +52,29 @@ class ColumnValueLengthsToBeBetweenValidator(
             column: column
         """
         return self.run_query_results(self.runner, metric, column)
+
+    def compute_row_count(self, column: Column, min_bound: int, max_bound: int):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+            min_bound (_type_): min bound to filter out rows within the bound
+            max_bound (_type_): max bound to filter out rows within the bound
+
+        Raises:
+            NotImplementedError:
+        """
+        row_count = self._compute_row_count(self.runner, column)
+        failed_rows = self._compute_row_count_between(
+            self.runner,
+            column,
+            {
+                "filters": [
+                    (LenFn(column), "gt", max_bound),
+                    (LenFn(column), "lt", min_bound),
+                ],
+                "or_filter": True,
+            },
+        )
+
+        return row_count, failed_rows

metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py

@@ -50,3 +50,26 @@ class ColumnValuesToBeBetweenValidator(
             column: column
         """
         return self.run_query_results(self.runner, metric, column)
+
+    def compute_row_count(self, column: Column, min_bound: int, max_bound: int):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+            min_bound (_type_): min bound to filter out rows within the bound
+            max_bound (_type_): max bound to filter out rows within the bound
+
+        Raises:
+            NotImplementedError:
+        """
+        row_count = self._compute_row_count(self.runner, column)
+        failed_rows = self._compute_row_count_between(
+            self.runner,
+            column,
+            {
+                "filters": [(column, "gt", max_bound), (column, "lt", min_bound)],
+                "or_filter": True,
+            },
+        )
+
+        return row_count, failed_rows

metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py

@@ -50,3 +50,14 @@ class ColumnValuesToBeInSetValidator(
             column: column
         """
         return self.run_query_results(self.runner, metric, column, **kwargs)
+
+    def compute_row_count(self, column: Column):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py

@@ -50,3 +50,14 @@ class ColumnValuesToBeNotInSetValidator(
             column: column
         """
         return self.run_query_results(self.runner, metric, column, **kwargs)
+
+    def compute_row_count(self, column: Column):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py

@@ -53,3 +53,14 @@ class ColumnValuesToBeNotNullValidator(
             column: column
         """
         return self.run_query_results(self.runner, metric, column)
+
+    def compute_row_count(self, column: Column):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py

@@ -62,3 +62,14 @@ class ColumnValuesToMatchRegexValidator(
             return self.run_query_results(
                 self.runner, Metrics.LIKE_COUNT, column, **kwargs
             )
+
+    def compute_row_count(self, column: Column):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py

@@ -62,3 +62,14 @@ class ColumnValuesToNotMatchRegexValidator(
             return self.run_query_results(
                 self.runner, Metrics.NOT_LIKE_COUNT, column, **kwargs
             )
+
+    def compute_row_count(self, column: Column):
+        """Compute row count for the given column
+
+        Args:
+            column (Union[SQALikeColumn, Column]): column to compute row count for
+
+        Raises:
+            NotImplementedError:
+        """
+        return self._compute_row_count(self.runner, column)

metadata/data_quality/validations/mixins/pandas_validator_mixin.py

@@ -17,7 +17,7 @@ from typing import Optional
 
 from metadata.profiler.metrics.core import add_props
 from metadata.profiler.metrics.registry import Metrics
-from metadata.utils.datalake.datalake_utils import fetch_col_types
+from metadata.utils.datalake.datalake_utils import GenericDataFrameColumnParser
 from metadata.utils.entity_link import get_decoded_column
 from metadata.utils.sqa_like_column import SQALikeColumn
 
@@ -28,7 +28,9 @@ class PandasValidatorMixin:
     def get_column_name(self, entity_link: str, dfs) -> SQALikeColumn:
         # we'll use the first dataframe chunk to get the column name.
         column = dfs[0][get_decoded_column(entity_link)]
-        _type = fetch_col_types(dfs[0], get_decoded_column(entity_link))
+        _type = GenericDataFrameColumnParser.fetch_col_types(
+            dfs[0], get_decoded_column(entity_link)
+        )
         sqa_like_column = SQALikeColumn(
             name=column.name,
             type=_type,
@@ -59,3 +61,12 @@ class PandasValidatorMixin:
             return metric_fn(runner)
         except Exception as exc:
             raise RuntimeError(exc)
+
+    def _compute_row_count(self, runner, column: SQALikeColumn, **kwargs):
+        """compute row count
+
+        Args:
+            runner (List[DataFrame]): runner to run the test case against)
+            column (SQALikeColumn): column to compute row count for
+        """
+        return self.run_dataframe_results(runner, Metrics.ROW_COUNT, column, **kwargs)

metadata/data_quality/validations/mixins/sqa_validator_mixin.py

@@ -83,3 +83,43 @@ class SQAValidatorMixin:
             )
 
         return res
+
+    def _compute_row_count_between(
+        self,
+        runner: QueryRunner,
+        column: Column,
+        query_filter: dict,
+    ):
+        """compute row count for between tests
+
+        Args:
+            runner (QueryRunner): runner
+            column (Column): column
+            query_filter (dict): filter to apply to the query
+
+        Raises:
+            SQLAlchemyError:
+
+        Returns:
+        """
+        try:
+            value = dict(
+                runner.dispatch_query_select_first(
+                    Metrics.ROW_COUNT(column).fn(),
+                    query_filter_=query_filter,
+                )
+            )
+            res = value.get(Metrics.ROW_COUNT.name)
+        except Exception as exc:
+            raise SQLAlchemyError(exc)
+
+        return res
+
+    def _compute_row_count(self, runner: QueryRunner, column: Column, **kwargs):
+        """compute row count
+
+        Args:
+            runner (QueryRunner): runner to run the test case against)
+            column (SQALikeColumn): column to compute row count for
+        """
+        return self.run_query_results(runner, Metrics.ROW_COUNT, column, **kwargs)

metadata/examples/workflows/databricks.yaml

@@ -8,7 +8,6 @@ source:
       token: <databricks token>
       hostPort: localhost:443
       connectionTimeout: 120
-      useUnityCatalog: true
       connectionArguments:
         http_path: <http path of databricks cluster>
   sourceConfig:

metadata/examples/workflows/mongodb.yaml

@@ -5,8 +5,10 @@ source:
     config:
       type: MongoDB
       databaseName: custom_database_name
-      connectionDetails:
-        connectionURI: mongodb+srv://user:pass@localhost:27017
+      scheme: mongodb+srv
+      username: username
+      password: password
+      hostPort:  localhost:27017
   sourceConfig:
     config:
       type: DatabaseMetadata

metadata/examples/workflows/mstr.yaml

@@ -0,0 +1,24 @@
+source:
+  type: mstr
+  serviceName: test
+  serviceConnection:
+    config:
+      type: Mstr
+      username: username
+      password: password
+      hostPort: http://hostPort
+      projectName: project
+  sourceConfig:
+    config:
+      type: DashboardMetadata
+      dashboardFilterPattern: {}
+      chartFilterPattern: {}
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

metadata/examples/workflows/sas.yaml

@@ -0,0 +1,28 @@
+source:
+  type: sas
+  serviceName: local_sas
+  serviceConnection:
+    config:
+      type: SAS
+      username: username
+      password: password
+      serverHost: http://your-server-host.org
+      datatables: True
+      dataTablesCustomFilter: None
+      reports: False
+      reportsCustomFilter: None
+      dataflows: False
+      dataflowsCustomFilter: None
+  sourceConfig:
+    config:
+      type: DatabaseMetadata
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG
+  openMetadataServerConfig:
+    hostPort:  http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJvcGVuLW1ldGFkYXRhLm9yZyIsInN1YiI6ImluZ2VzdGlvbi1ib3QiLCJlbWFpbCI6ImluZ2VzdGlvbi1ib3RAb3Blbm1ldGFkYXRhLm9yZyIsImlzQm90Ijp0cnVlLCJ0b2tlblR5cGUiOiJCT1QiLCJpYXQiOjE3MDQ3NDY0MzYsImV4cCI6bnVsbH0.qRWByaRw3F1B0bDqdpmxDsHEUx9Npk5VNelpabuvmVERjIG8AY88p1dv5gBME6Y1-kfTtCAMtSxsll0_gTR35D1foVdXgGMRAPyNXH0JHRpENBnT1V3OVO0yRWmeqsp5K7yqiaVa-CeqxitSgYrns58BFRD_vFX5vxMHirFBrFddH7b8af8823a8Oh-wMDCuJJd7Ya61Kv6gUpssE7Y403PwomLK6pE7gsAgZ5YOyjHlQ889C9z8oLQ268BX3ndmTp6t1J7MYgOEqeIzeCoBQZf6aBdyCqYajB3TTwb3SHFz2TEYF5xeLvYlZ77ek3l22m6Ehh5d4t2jB-ZOFDUkqA

metadata/examples/workflows/unity_catalog.yaml

@@ -0,0 +1,27 @@
+source:
+  type: unitycatalog
+  serviceName: local_unitycatalog
+  serviceConnection:
+    config:
+      type: UnityCatalog
+      catalog: hive_metastore
+      databaseSchema: default
+      token: <databricks token>
+      hostPort: localhost:443
+      connectionTimeout: 120
+      connectionArguments:
+        http_path: <http path of databricks cluster>
+
+  sourceConfig:
+    config:
+      type: DatabaseMetadata
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"      

metadata/examples/workflows/unity_catalog_lineage.yaml

@@ -0,0 +1,18 @@
+source:
+  type: unitycatalog-lineage
+  serviceName: local_unitycatalog
+  sourceConfig:
+    config:
+      type: DatabaseLineage
+      queryLogDuration: 1
+      resultLimit: 10000
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"      

metadata/examples/workflows/unity_catalog_usage.yaml

@@ -0,0 +1,35 @@
+source:
+  type: unitycatalog-usage
+  serviceName: local_unitycatalog
+  serviceConnection:
+    config:
+      type: UnityCatalog
+      catalog: hive_metastore
+      databaseSchema: default
+      token: <databricks token>
+      hostPort: localhost:443
+      connectionTimeout: 120
+      connectionArguments:
+        http_path: <http path of databricks cluster>
+  sourceConfig:
+    config:
+      type: DatabaseUsage
+      queryLogDuration: 10
+processor:
+  type: query-parser
+  config: {}
+stage:
+  type: table-usage
+  config:
+    filename: /tmp/databricks_usage
+bulkSink:
+  type: metadata-usage
+  config:
+    filename: /tmp/databricks_usage
+workflowConfig:
+  loggerLevel: DEBUG
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"