openmetadata-ingestion 1.7.0.0rc2__py3-none-any.whl → 1.7.1.0__py3-none-any.whl

metadata/applications/example.py

@@ -0,0 +1,74 @@
+#  Copyright 2025 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Example external application
+"""
+from time import sleep
+from typing import Any
+
+from metadata.generated.schema.entity.applications.configuration.internal.helloPipelinesConfiguration import (
+    HelloPipelinesAppConfiguration,
+)
+from metadata.generated.schema.metadataIngestion.application import (
+    OpenMetadataApplicationConfig,
+)
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.logger import app_logger
+from metadata.workflow.application import AppRunner, InvalidAppConfiguration
+
+logger = app_logger()
+
+
+class HelloPipelines(AppRunner):
+    """
+    Example external application that sleeps for a given time and then echoes a message.
+    You can execute it with `metadata app -c <path-to-yaml>`
+    with a YAML file like:
+
+    sourcePythonClass: metadata.applications.example.HelloPipelines
+    appConfig:
+      type: HelloPipelines
+      sleep: 5
+      echo: this will be echoed
+    workflowConfig:
+      loggerLevel: INFO
+      openMetadataServerConfig:
+        hostPort: http://localhost:8585/api
+        authProvider: openmetadata
+        securityConfig:
+          jwtToken: "..."
+    """
+
+    def __init__(
+        self, config: OpenMetadataApplicationConfig, metadata: OpenMetadata[Any, Any]
+    ):
+        super().__init__(config, metadata)  # pyright: ignore [reportUnknownMemberType]
+        try:
+            self.app_config: HelloPipelinesAppConfiguration = (
+                HelloPipelinesAppConfiguration.model_validate(self.app_config)
+            )
+        except Exception as e:
+            raise InvalidAppConfiguration(
+                f"Hello pipelines received invalid configuration: {e}"
+            )
+
+    @property
+    def name(self) -> str:
+        return "HelloPipelines"
+
+    def run(self) -> None:
+        logger.info(f"sleeping for {self.app_config.sleep}")
+        sleep(self.app_config.sleep)
+        logger.info("echoing")
+        logger.info(self.app_config.echo)
+
+    def close(self) -> None:
+        """Nothing to close"""

metadata/cli/classify.py

@@ -16,6 +16,7 @@ import sys
 import traceback
 from pathlib import Path
 
+from metadata.cli.common import execute_workflow
 from metadata.config.common import load_config_file
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
     PipelineType,
@@ -46,7 +47,4 @@ def run_classification(config_path: Path) -> None:
         )
         sys.exit(1)
 
-    workflow.execute()
-    workflow.stop()
-    workflow.print_status()
-    workflow.raise_from_status()
+    execute_workflow(workflow=workflow, config_dict=config_dict)

metadata/cli/common.py

@@ -0,0 +1,26 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Handle workflow execution
+"""
+from typing import Any, Dict
+
+from metadata.workflow.base import BaseWorkflow
+
+
+def execute_workflow(workflow: BaseWorkflow, config_dict: Dict[str, Any]) -> None:
+    """Execute the workflow and raise if needed"""
+    workflow.execute()
+    workflow.print_status()
+    workflow.stop()
+    if config_dict.get("workflowConfig", {}).get("raiseOnError", True):
+        workflow.raise_from_status()

metadata/cli/dataquality.py

@@ -16,6 +16,7 @@ import sys
 import traceback
 from pathlib import Path
 
+from metadata.cli.common import execute_workflow
 from metadata.config.common import load_config_file
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
     PipelineType,
@@ -48,7 +49,4 @@ def run_test(config_path: Path) -> None:
         )
         sys.exit(1)
 
-    workflow.execute()
-    workflow.stop()
-    workflow.print_status()
-    workflow.raise_from_status()
+    execute_workflow(workflow=workflow, config_dict=workflow_config_dict)

metadata/cli/ingest.py

@@ -16,6 +16,7 @@ import sys
 import traceback
 from pathlib import Path
 
+from metadata.cli.common import execute_workflow
 from metadata.config.common import load_config_file
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
     PipelineType,
@@ -46,7 +47,4 @@ def run_ingest(config_path: Path) -> None:
         )
         sys.exit(1)
 
-    workflow.execute()
-    workflow.stop()
-    workflow.print_status()
-    workflow.raise_from_status()
+    execute_workflow(workflow=workflow, config_dict=config_dict)

metadata/cli/profile.py

@@ -16,6 +16,7 @@ import sys
 import traceback
 from pathlib import Path
 
+from metadata.cli.common import execute_workflow
 from metadata.config.common import load_config_file
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
     PipelineType,
@@ -48,7 +49,4 @@ def run_profiler(config_path: Path) -> None:
         )
         sys.exit(1)
 
-    workflow.execute()
-    workflow.stop()
-    workflow.print_status()
-    workflow.raise_from_status()
+    execute_workflow(workflow=workflow, config_dict=workflow_config_dict)

metadata/cli/usage.py

@@ -16,6 +16,7 @@ import sys
 import traceback
 from pathlib import Path
 
+from metadata.cli.common import execute_workflow
 from metadata.config.common import load_config_file
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
     PipelineType,
@@ -44,7 +45,4 @@ def run_usage(config_path: Path) -> None:
         WorkflowInitErrorHandler.print_init_error(exc, config_dict, PipelineType.usage)
         sys.exit(1)
 
-    workflow.execute()
-    workflow.stop()
-    workflow.print_status()
-    workflow.raise_from_status()
+    execute_workflow(workflow=workflow, config_dict=config_dict)

metadata/data_quality/processor/test_case_runner.py

@@ -16,6 +16,8 @@ import traceback
 from copy import deepcopy
 from typing import List, Optional
 
+from pydantic import RootModel
+
 from metadata.data_quality.api.models import (
     TableAndTests,
     TestCaseDefinition,
@@ -27,7 +29,6 @@ from metadata.data_quality.runner.base_test_suite_source import BaseTestSuiteRun
 from metadata.data_quality.runner.core import DataTestsRunner
 from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
 from metadata.generated.schema.entity.data.table import Table
-from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
@@ -94,9 +95,8 @@ class TestCaseRunner(Processor):
             record.table, openmetadata_test_cases
         )
 
-        test_suite_runner = self.get_test_suite_runner(
-            record.table, record.service_connection
-        )
+        self.config.source.serviceConnection = RootModel(record.service_connection)
+        test_suite_runner = self.get_test_suite_runner(record.table)
 
         logger.debug(
             f"Found {len(openmetadata_test_cases)} test cases for table {record.table.fullyQualifiedName.root}"
@@ -354,9 +354,7 @@ class TestCaseRunner(Processor):
                 result.append(tc)
         return result
 
-    def get_test_suite_runner(
-        self, table: Table, service_connection: DatabaseConnection
-    ):
+    def get_test_suite_runner(self, table: Table):
         return BaseTestSuiteRunner(
-            self.config, self.metadata, table, service_connection
+            self.config, self.metadata, table
         ).get_data_quality_runner()

metadata/data_quality/runner/base_test_suite_source.py

@@ -19,6 +19,9 @@ from metadata.data_quality.builders.validator_builder import ValidatorBuilder
 from metadata.data_quality.interface.test_suite_interface import TestSuiteInterface
 from metadata.data_quality.runner.core import DataTestsRunner
 from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
+    BigQueryConnection,
+)
 from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
 from metadata.generated.schema.entity.services.serviceType import ServiceType
 from metadata.generated.schema.metadataIngestion.testSuitePipeline import (
@@ -31,6 +34,7 @@ from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.sampler.models import SampleConfig
 from metadata.sampler.sampler_interface import SamplerInterface
+from metadata.utils.bigquery_utils import copy_service_config
 from metadata.utils.profiler_utils import get_context_entities
 from metadata.utils.service_spec.service_spec import (
     import_sampler_class,
@@ -46,12 +50,11 @@ class BaseTestSuiteRunner:
         config: OpenMetadataWorkflowConfig,
         ometa_client: OpenMetadata,
         entity: Table,
-        service_connection: DatabaseConnection,
     ):
         self.validator_builder_class = ValidatorBuilder
         self._interface = None
         self.entity = entity
-        self.service_conn_config = self._copy_service_config(service_connection, self.entity.database)  # type: ignore
+        self.service_conn_config = self._copy_service_config(config, self.entity.database)  # type: ignore
         self._interface_type: str = self.service_conn_config.type.value.lower()
 
         self.source_config = TestSuitePipeline.model_validate(
@@ -68,7 +71,7 @@ class BaseTestSuiteRunner:
         self._interface = interface
 
     def _copy_service_config(
-        self, service_connection: DatabaseConnection, database: EntityReference
+        self, config: OpenMetadataWorkflowConfig, database: EntityReference
     ) -> DatabaseConnection:
         """Make a copy of the service config and update the database name
 
@@ -78,7 +81,10 @@ class BaseTestSuiteRunner:
         Returns:
             DatabaseService.__config__
         """
-        config_copy = deepcopy(service_connection.config)  # type: ignore
+        if isinstance(config.source.serviceConnection.root.config, BigQueryConnection):
+            return copy_service_config(config, database.name)
+
+        config_copy = deepcopy(config.source.serviceConnection.root.config)  # type: ignore
         if hasattr(
             config_copy,  # type: ignore
             "supportsDatabase",

metadata/data_quality/source/test_suite.py

@@ -136,7 +136,17 @@ class TestSuiteSource(Source):
                         f"Service with name `{service_name}` does not have a connection. "
                         "If the connection is not stored in OpenMetadata, please provide it in the YAML file."
                     )
-                self.service_connection_map[service_name] = service.connection
+
+                # TODO: Clean after https://github.com/open-metadata/OpenMetadata/issues/21259
+                # We are forcing the secret evaluation to "ignore" null secrets down the line
+                # Remove this when the issue above is fixed and empty secrets migrated
+                source_config_class = type(service.connection)
+                dumped_config = service.connection.model_dump()
+                service_connection_clean = source_config_class.model_validate(
+                    dumped_config
+                )
+
+                self.service_connection_map[service_name] = service_connection_clean
 
             except Exception as exc:
                 logger.debug(traceback.format_exc())

metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py

@@ -15,7 +15,7 @@ Validator for column values to be unique test case
 
 from typing import Optional
 
-from sqlalchemy import Column, inspect
+from sqlalchemy import Column, inspect, literal_column
 from sqlalchemy.exc import SQLAlchemyError
 
 from metadata.data_quality.validations.column.base.columnValuesToBeUnique import (
@@ -25,6 +25,7 @@ from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
     SQAValidatorMixin,
 )
 from metadata.profiler.metrics.registry import Metrics
+from metadata.profiler.orm.registry import Dialects
 
 
 class ColumnValuesToBeUniqueValidator(
@@ -57,7 +58,18 @@ class ColumnValuesToBeUniqueValidator(
         )  # type: ignore
 
         try:
-            self.value = dict(self.runner.dispatch_query_select_first(count, unique_count.scalar_subquery().label("uniqueCount")))  # type: ignore
+            if self.runner.dialect == Dialects.Oracle:
+                query_group_by_ = [literal_column("2")]
+            else:
+                query_group_by_ = None
+
+            self.value = dict(
+                self.runner.dispatch_query_select_first(
+                    count,
+                    unique_count.scalar_subquery().label("uniqueCount"),
+                    query_group_by_=query_group_by_,
+                )
+            )  # type: ignore
             res = self.value.get(Metrics.COUNT.name)
         except Exception as exc:
             raise SQLAlchemyError(exc)

metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py

@@ -76,7 +76,7 @@ class ColumnValuesToMatchRegexValidator(
             res = dict(
                 self.runner.dispatch_query_select_first(
                     Metrics.COUNT(column).fn(),
-                    regex_count,
+                    regex_count_fn,
                 )
             )
 

metadata/data_quality/validations/models.py

@@ -8,6 +8,7 @@ from metadata.generated.schema.entity.data.table import Column, TableProfilerCon
 from metadata.generated.schema.entity.services.databaseService import (
     DatabaseServiceType,
 )
+from metadata.ingestion.models.custom_pydantic import CustomSecretStr
 
 
 class TableParameter(BaseModel):
@@ -15,6 +16,8 @@ class TableParameter(BaseModel):
     path: str
     columns: List[Column]
     database_service_type: DatabaseServiceType
+    privateKey: Optional[CustomSecretStr]
+    passPhrase: Optional[CustomSecretStr]
 
 
 class TableDiffRuntimeParameters(BaseModel):

metadata/data_quality/validations/runtime_param_setter/base_diff_params_setter.py

@@ -0,0 +1,120 @@
+"""Base class for param setter logic for table data diff"""
+
+from typing import List, Optional, Set
+from urllib.parse import urlparse
+
+from metadata.data_quality.validations.models import Column, TableParameter
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.ingestion.source.connections import get_connection
+from metadata.profiler.orm.registry import Dialects
+from metadata.utils import fqn
+from metadata.utils.collections import CaseInsensitiveList
+
+
+class BaseTableParameter:
+    """Base table parameter setter for the table diff test."""
+
+    def get(
+        self,
+        service: DatabaseService,
+        entity: Table,
+        key_columns,
+        extra_columns,
+        case_sensitive_columns,
+        service_url: Optional[str],
+    ) -> TableParameter:
+        """Getter table parameter for the table diff test.
+
+        Returns:
+            TableParameter
+        """
+        return TableParameter(
+            database_service_type=service.serviceType,
+            path=self.get_data_diff_table_path(entity.fullyQualifiedName.root),
+            serviceUrl=self.get_data_diff_url(
+                service,
+                entity.fullyQualifiedName.root,
+                override_url=service_url,
+            ),
+            columns=self.filter_relevant_columns(
+                entity.columns,
+                key_columns,
+                extra_columns,
+                case_sensitive=case_sensitive_columns,
+            ),
+            privateKey=None,
+            passPhrase=None,
+        )
+
+    @staticmethod
+    def get_data_diff_table_path(table_fqn: str) -> str:
+        """Get the data diff table path.
+
+        Args:
+            table_fqn (str): The fully qualified name of the table
+
+        Returns:
+            str
+        """
+        _, _, schema, table = fqn.split(table_fqn)
+        return fqn._build(  # pylint: disable=protected-access
+            "___SERVICE___", "__DATABASE__", schema, table
+        ).replace("___SERVICE___.__DATABASE__.", "")
+
+    @staticmethod
+    def get_data_diff_url(
+        db_service: DatabaseService, table_fqn, override_url: Optional[str] = None
+    ) -> str:
+        """Get the url for the data diff service.
+
+        Args:
+            db_service (DatabaseService): The database service entity
+            table_fqn (str): The fully qualified name of the table
+            override_url (Optional[str], optional): Override the url. Defaults to None.
+
+        Returns:
+            str: The url for the data diff service
+        """
+        source_url = (
+            str(get_connection(db_service.connection.config).url)
+            if not override_url
+            else override_url
+        )
+        url = urlparse(source_url)
+        # remove the driver name from the url because table-diff doesn't support it
+        kwargs = {"scheme": url.scheme.split("+")[0]}
+        service, database, schema, table = fqn.split(  # pylint: disable=unused-variable
+            table_fqn
+        )
+        # path needs to include the database AND schema in some of the connectors
+        if hasattr(db_service.connection.config, "supportsDatabase"):
+            kwargs["path"] = f"/{database}"
+        if kwargs["scheme"] in {Dialects.MSSQL, Dialects.Snowflake, Dialects.Trino}:
+            kwargs["path"] = f"/{database}/{schema}"
+        return url._replace(**kwargs).geturl()
+
+    @staticmethod
+    def filter_relevant_columns(
+        columns: List[Column],
+        key_columns: Set[str],
+        extra_columns: Set[str],
+        case_sensitive: bool,
+    ) -> List[Column]:
+        """Filter relevant columns.
+
+        Args:
+            columns (List[Column]): list of columns
+            key_columns (Set[str]): set of key columns
+            extra_columns (Set[str]): set of extra columns
+            case_sensitive (bool): case sensitive flag
+
+        Returns:
+            List[Column]
+        """
+        validated_columns = (
+            [*key_columns, *extra_columns]
+            if case_sensitive
+            else CaseInsensitiveList([*key_columns, *extra_columns])
+        )
+        return [c for c in columns if c.name.root in validated_columns]

metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py

@@ -14,21 +14,32 @@ from typing import List, Optional, Set
 from urllib.parse import urlparse
 
 from metadata.data_quality.validations import utils
-from metadata.data_quality.validations.models import (
-    Column,
-    TableDiffRuntimeParameters,
-    TableParameter,
-)
+from metadata.data_quality.validations.models import Column, TableDiffRuntimeParameters
 from metadata.data_quality.validations.runtime_param_setter.param_setter import (
     RuntimeParameterSetter,
 )
 from metadata.generated.schema.entity.data.table import Constraint, Table
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.services.serviceType import ServiceType
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.ingestion.source.connections import get_connection
 from metadata.profiler.orm.registry import Dialects
 from metadata.utils import fqn
 from metadata.utils.collections import CaseInsensitiveList
+from metadata.utils.importer import get_module_dir, import_from_module
+
+
+def get_for_source(
+    service_type: ServiceType, source_type: str, from_: str = "ingestion"
+):
+    return import_from_module(
+        "metadata.{}.source.{}.{}.{}.ServiceSpec".format(  # pylint: disable=C0209
+            from_,
+            service_type.name.lower(),
+            get_module_dir(source_type),
+            "service_spec",
+        )
+    )
 
 
 class TableDiffParamsSetter(RuntimeParameterSetter):
@@ -51,67 +62,70 @@ class TableDiffParamsSetter(RuntimeParameterSetter):
         }
 
     def get_parameters(self, test_case) -> TableDiffRuntimeParameters:
+        # Using the specs class method causes circular import as TestSuiteInterface
+        # imports RuntimeParameterSetter
+        cls_path = get_for_source(
+            ServiceType.Database,
+            source_type=self.service_connection_config.type.value.lower(),
+        ).data_diff
+        cls = import_from_module(cls_path)()
+
+        service1: DatabaseService = self.ometa_client.get_by_id(
+            DatabaseService, self.table_entity.service.id, nullable=False
+        )
+
         service1_url = (
             str(get_connection(self.service_connection_config).url)
             if self.service_connection_config
             else None
         )
-        service1: DatabaseService = self.ometa_client.get_by_id(
-            DatabaseService, self.table_entity.service.id, nullable=False
-        )
+
         table2_fqn = self.get_parameter(test_case, "table2")
-        case_sensitive_columns: bool = utils.get_bool_test_case_param(
-            test_case.parameterValues, "caseSensitiveColumns"
-        )
         if table2_fqn is None:
             raise ValueError("table2 not set")
         table2: Table = self.ometa_client.get_by_name(
             Table, fqn=table2_fqn, nullable=False
         )
-        service2_url = (
-            service1_url if table2.service == self.table_entity.service else None
-        )
         service2: DatabaseService = self.ometa_client.get_by_id(
             DatabaseService, table2.service.id, nullable=False
         )
+        service2_url = (
+            self.get_parameter(test_case, "service2Url") or service1_url
+            if table2.service == self.table_entity.service
+            else None
+        )
+
         key_columns = self.get_key_columns(test_case)
-        extra_columns = self.get_extra_columns(
-            key_columns, test_case, self.table_entity.columns, table2.columns
+        extra_columns = (
+            self.get_extra_columns(
+                key_columns, test_case, self.table_entity.columns, table2.columns
+            )
+            or set()
         )
+        case_sensitive_columns: bool = (
+            utils.get_bool_test_case_param(
+                test_case.parameterValues, "caseSensitiveColumns"
+            )
+            or False
+        )
+
         return TableDiffRuntimeParameters(
             table_profile_config=self.table_entity.tableProfilerConfig,
-            table1=TableParameter(
-                database_service_type=service1.serviceType,
-                path=self.get_data_diff_table_path(
-                    self.table_entity.fullyQualifiedName.root
-                ),
-                serviceUrl=self.get_data_diff_url(
-                    service1,
-                    self.table_entity.fullyQualifiedName.root,
-                    override_url=service1_url,
-                ),
-                columns=self.filter_relevant_columns(
-                    self.table_entity.columns,
-                    key_columns,
-                    extra_columns,
-                    case_sensitive=case_sensitive_columns,
-                ),
+            table1=cls.get(
+                service1,
+                self.table_entity,
+                key_columns,
+                extra_columns,
+                case_sensitive_columns,
+                service1_url,
             ),
-            table2=TableParameter(
-                database_service_type=service2.serviceType,
-                path=self.get_data_diff_table_path(table2_fqn),
-                serviceUrl=self.get_data_diff_url(
-                    service2,
-                    table2_fqn,
-                    override_url=self.get_parameter(test_case, "service2Url")
-                    or service2_url,
-                ),
-                columns=self.filter_relevant_columns(
-                    table2.columns,
-                    key_columns,
-                    extra_columns,
-                    case_sensitive=case_sensitive_columns,
-                ),
+            table2=cls.get(
+                service2,
+                table2,
+                key_columns,
+                extra_columns,
+                case_sensitive_columns,
+                service2_url,
             ),
             keyColumns=list(key_columns),
             extraColumns=list(extra_columns),