openmetadata-ingestion 1.7.5.3__py3-none-any.whl → 1.8.1.0__py3-none-any.whl

_openmetadata_testutils/helpers/login_user.py

@@ -0,0 +1,22 @@
+"""Base class for resource tests."""
+
+from locust.contrib.fasthttp import FastHttpSession
+from requests.auth import AuthBase
+
+
+class BearerAuth(AuthBase):
+    def __init__(self, token):
+        self.token = token
+
+    def __call__(self, r):
+        r.headers["authorization"] = "Bearer " + self.token
+        return r
+
+
+def login_user(client: FastHttpSession) -> BearerAuth:
+    resp = client.post(
+        "/api/v1/users/login",
+        json={"email": "admin@open-metadata.org", "password": "YWRtaW4="},
+    )
+    token = resp.json().get("accessToken")
+    return BearerAuth(token)

metadata/__init__.py

@@ -0,0 +1,31 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+OpenMetadata package initialization.
+"""
+from typing import Type
+
+from metadata.profiler.metrics.registry import Metrics
+from metadata.profiler.registry import MetricRegistry
+from metadata.profiler.source.database.base.profiler_resolver import (
+    DefaultProfilerResolver,
+    ProfilerResolver,
+)
+from metadata.utils.dependency_injector.dependency_injector import DependencyContainer
+from metadata.utils.service_spec.service_spec import DefaultSourceLoader, SourceLoader
+
+# Initialize the dependency container
+container = DependencyContainer()
+
+# Register the source loader
+container.register(SourceLoader, DefaultSourceLoader)
+container.register(Type[MetricRegistry], lambda: Metrics)
+container.register(Type[ProfilerResolver], lambda: DefaultProfilerResolver)

metadata/automations/runner.py

@@ -12,6 +12,8 @@
 Run the Automation Workflow for OpenMetadata
 """
 
+from typing import Optional
+
 from metadata.automations.execute_runner import run_workflow
 from metadata.generated.schema.entity.automations.testServiceConnection import (
     TestServiceConnectionRequest,
@@ -41,22 +43,15 @@ def _(
     if ssl_manager:
         request.connection.config = ssl_manager.setup_ssl(request.connection.config)
 
-    # Find the test_connection function in each <source>/connection.py file
-    test_connection_fn = get_test_connection_fn(request.connection.config)
-
     try:
-        connection = get_connection(request.connection.config)
         if hasattr(request.connection.config, "hostPort"):
             host_port_str = str(request.connection.config.hostPort or "")
             if "localhost" in host_port_str:
-                result = test_connection_fn(
-                    metadata, connection, request.connection.config
-                )
+
+                result = _test_connection(metadata, request.connection.config)
                 raise_test_connection_exception(result)
 
-        test_connection_fn(
-            metadata, connection, request.connection.config, automation_workflow
-        )
+        _ = _test_connection(metadata, request.connection.config, automation_workflow)
     except Exception as error:
         host_port_str = str(getattr(request.connection.config, "hostPort", None) or "")
         if not host_port_str or "localhost" not in host_port_str:
@@ -68,10 +63,25 @@ def _(
         )
         request.connection.config.hostPort = host_port_type(docker_host_port_str)
 
-        connection = get_connection(request.connection.config)
-        test_connection_fn(
-            metadata, connection, request.connection.config, automation_workflow
-        )
+        _ = _test_connection(metadata, request.connection.config, automation_workflow)
 
     if ssl_manager:
         ssl_manager.cleanup_temp_files()
+
+
+def _test_connection(
+    metadata: OpenMetadata,
+    config,
+    automation_workflow: Optional[AutomationWorkflow] = None,
+):
+    """
+    Test the connection
+    """
+    test_connection_fn = get_test_connection_fn(config)
+    try:
+        return test_connection_fn(metadata, automation_workflow=automation_workflow)
+    except TypeError:
+        connection = get_connection(config)
+        return test_connection_fn(
+            metadata, connection, config, automation_workflow=automation_workflow
+        )

metadata/cli/ingest_dbt.py

@@ -91,7 +91,7 @@ class OpenMetadataDBTConfig(BaseModel):
             return v
         except (ValueError, TypeError) as e:
             raise ValueError(
-                f"Host port must be a valid URL starting with http:// or https://: {e}"
+                f"Host port must be a valid URL starting with http:// or https://"
             )
 
     def _get_filter_pattern(

metadata/data_quality/processor/test_case_runner.py

@@ -41,14 +41,13 @@ from metadata.generated.schema.tests.testDefinition import (
     TestDefinition,
     TestPlatform,
 )
-from metadata.generated.schema.tests.testSuite import TestSuite
 from metadata.generated.schema.type.basic import EntityLink, FullyQualifiedEntityName
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.parser import parse_workflow_config_gracefully
 from metadata.ingestion.api.step import Step
 from metadata.ingestion.api.steps import Processor
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.utils import entity_link, fqn
+from metadata.utils import entity_link
 from metadata.utils.logger import test_suite_logger
 
 logger = test_suite_logger()
@@ -83,11 +82,6 @@ class TestCaseRunner(Processor):
         # Add the test cases from the YAML file, if any
         test_cases = self.get_test_cases(
             test_cases=record.test_cases,
-            test_suite_fqn=fqn.build(
-                None,
-                TestSuite,
-                table_fqn=record.table.fullyQualifiedName.root,
-            ),
             table_fqn=record.table.fullyQualifiedName.root,
         )
         openmetadata_test_cases = self.filter_for_om_test_cases(test_cases)
@@ -113,7 +107,7 @@ class TestCaseRunner(Processor):
         return Either(right=TestCaseResults(test_results=test_results))
 
     def get_test_cases(
-        self, test_cases: List[TestCase], test_suite_fqn: str, table_fqn: str
+        self, test_cases: List[TestCase], table_fqn: str
     ) -> List[TestCase]:
         """
         Based on the test suite test cases that we already know, pick up
@@ -124,7 +118,6 @@ class TestCaseRunner(Processor):
             return self.compare_and_create_test_cases(
                 cli_test_cases_definitions=cli_test_cases,
                 test_cases=test_cases,
-                test_suite_fqn=test_suite_fqn,
                 table_fqn=table_fqn,
             )
 
@@ -141,7 +134,6 @@ class TestCaseRunner(Processor):
         cli_test_cases_definitions: List[TestCaseDefinition],
         test_cases: List[TestCase],
         table_fqn: str,
-        test_suite_fqn: str,
     ) -> List[TestCase]:
         """
         compare test cases defined in CLI config workflow with test cases
@@ -198,7 +190,6 @@ class TestCaseRunner(Processor):
                                 column_name=test_case_to_create.columnName,
                             )
                         ),
-                        testSuite=test_suite_fqn,
                         parameterValues=(
                             list(test_case_to_create.parameterValues)
                             if test_case_to_create.parameterValues

metadata/data_quality/source/test_suite.py

@@ -289,7 +289,9 @@ class TestSuiteSource(Source):
             test_cases, key=lambda t: entity_link.get_table_fqn(t.entityLink.root)
         )
         for table_fqn, group in grouped_by_table:
-            table_entity: Table = self.metadata.get_by_name(Table, table_fqn)
+            table_entity: Table = self.metadata.get_by_name(
+                Table, table_fqn, fields=["tableProfilerConfig"]
+            )
             if table_entity is None:
                 yield Either(
                     left=StackTraceError(

metadata/data_quality/validations/models.py

@@ -1,6 +1,6 @@
 """Models for the TableDiff test case"""
 
-from typing import List, Optional, Union
+from typing import List, Optional
 
 from pydantic import BaseModel
 
@@ -12,7 +12,7 @@ from metadata.ingestion.models.custom_pydantic import CustomSecretStr
 
 
 class TableParameter(BaseModel):
-    serviceUrl: Union[str, dict]
+    serviceUrl: str
     path: str
     columns: List[Column]
     database_service_type: DatabaseServiceType

metadata/data_quality/validations/runtime_param_setter/base_diff_params_setter.py

@@ -1,6 +1,6 @@
 """Base class for param setter logic for table data diff"""
 
-from typing import List, Optional, Set, Union
+from typing import List, Optional, Set
 from urllib.parse import urlparse
 
 from metadata.data_quality.validations.models import Column, TableParameter
@@ -62,31 +62,10 @@ class BaseTableParameter:
             "___SERVICE___", "__DATABASE__", schema, table
         ).replace("___SERVICE___.__DATABASE__.", "")
 
-    @staticmethod
-    def get_service_connection_config(
-        db_service: DatabaseService,
-    ) -> Optional[Union[str, dict]]:
-        """Get the service connection config.
-
-        Args:
-            db_service (DatabaseService): The database service entity
-        """
-        if db_service.connection.config.type.value.lower() == "trino":
-            from metadata.ingestion.source.database.trino.connection import (  # pylint: disable=import-outside-toplevel
-                get_connection_dict,
-            )
-
-            return get_connection_dict(db_service.connection.config)
-        return (
-            str(get_connection(db_service.connection.config).url)
-            if db_service.connection.config
-            else None
-        )
-
     @staticmethod
     def get_data_diff_url(
         db_service: DatabaseService, table_fqn, override_url: Optional[str] = None
-    ) -> Union[str, dict]:
+    ) -> str:
         """Get the url for the data diff service.
 
         Args:
@@ -97,17 +76,11 @@ class BaseTableParameter:
         Returns:
             str: The url for the data diff service
         """
-
         source_url = (
-            BaseTableParameter.get_service_connection_config(db_service)
+            str(get_connection(db_service.connection.config).url)
             if not override_url
             else override_url
         )
-
-        if isinstance(source_url, dict):
-            source_url["driver"] = source_url["driver"].split("+")[0]
-            return source_url
-
         url = urlparse(source_url)
         # remove the driver name from the url because table-diff doesn't support it
         kwargs = {"scheme": url.scheme.split("+")[0]}

metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py

@@ -10,14 +10,11 @@
 #  limitations under the License.
 """Module that defines the TableDiffParamsSetter class."""
 from ast import literal_eval
-from typing import List, Optional, Set, Union
+from typing import List, Optional, Set
 from urllib.parse import urlparse
 
 from metadata.data_quality.validations import utils
 from metadata.data_quality.validations.models import Column, TableDiffRuntimeParameters
-from metadata.data_quality.validations.runtime_param_setter.base_diff_params_setter import (
-    BaseTableParameter,
-)
 from metadata.data_quality.validations.runtime_param_setter.param_setter import (
     RuntimeParameterSetter,
 )
@@ -25,6 +22,7 @@ from metadata.generated.schema.entity.data.table import Constraint, Table
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.serviceType import ServiceType
 from metadata.generated.schema.tests.testCase import TestCase
+from metadata.ingestion.source.connections import get_connection
 from metadata.profiler.orm.registry import Dialects
 from metadata.utils import fqn
 from metadata.utils.collections import CaseInsensitiveList
@@ -77,7 +75,7 @@ class TableDiffParamsSetter(RuntimeParameterSetter):
         )
 
         service1_url = (
-            BaseTableParameter.get_service_connection_config(service1)
+            str(get_connection(self.service_connection_config).url)
             if self.service_connection_config
             else None
         )
@@ -214,7 +212,7 @@ class TableDiffParamsSetter(RuntimeParameterSetter):
     @staticmethod
     def get_data_diff_url(
         db_service: DatabaseService, table_fqn, override_url: Optional[str] = None
-    ) -> Union[str, dict]:
+    ) -> str:
         """Get the url for the data diff service.
 
         Args:
@@ -226,14 +224,10 @@ class TableDiffParamsSetter(RuntimeParameterSetter):
             str: The url for the data diff service
         """
         source_url = (
-            BaseTableParameter.get_service_connection_config(db_service)
+            str(get_connection(db_service.connection.config).url)
             if not override_url
             else override_url
         )
-        if isinstance(source_url, dict):
-            source_url["driver"] = source_url["driver"].split("+")[0]
-            return source_url
-
         url = urlparse(source_url)
         # remove the driver name from the url because table-diff doesn't support it
         kwargs = {"scheme": url.scheme.split("+")[0]}

metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py

@@ -34,7 +34,7 @@ class TableRowInsertedCountToBeBetweenValidator(
     """Validator for table row inserted count to be between test case"""
 
     @staticmethod
-    def _get_threshold_date(range_type: str, range_interval: int):
+    def get_threshold_date_dt(range_type: str, range_interval: int) -> datetime:
         """returns the threshold datetime in utc to count the numbers of rows inserted
 
         Args:
@@ -55,7 +55,22 @@ class TableRowInsertedCountToBeBetweenValidator(
             threshold_date = threshold_date.replace(
                 hour=0, minute=0, second=0, microsecond=0
             )
-        return threshold_date.strftime("%Y%m%d%H%M%S")
+        return threshold_date
+
+    @staticmethod
+    def _get_threshold_date(
+        range_type: str, range_interval: int, date_format: str = "%Y%m%d%H%M%S"
+    ):
+        """returns the threshold datetime in utc  as string to count the numbers of rows inserted
+
+        Args:
+            range_type (str): type of range (i.e. HOUR, DAY, MONTH, YEAR)
+            range_interval (int): interval of range (i.e. 1, 2, 3, 4)
+            date_format (str): format of the date (i.e. %Y%m%d%H%M%S, %Y-%m-%d %H:%M:%S)
+        """
+        return TableRowInsertedCountToBeBetweenValidator.get_threshold_date_dt(
+            range_type, range_interval
+        ).strftime(date_format)
 
     def _get_column_name(self):
         """returns the column name to be validated"""

metadata/data_quality/validations/table/sqlalchemy/tableDiff.py

@@ -508,10 +508,7 @@ class TableDiffValidator(BaseTestValidator, SQAValidatorMixin):
             ("table1.serviceUrl", self.runtime_params.table1.serviceUrl),
             ("table2.serviceUrl", self.runtime_params.table2.serviceUrl),
         ]:
-            if isinstance(param, dict):
-                dialect = param.get("driver")
-            else:
-                dialect = urlparse(param).scheme
+            dialect = urlparse(param).scheme
             if dialect not in SUPPORTED_DIALECTS:
                 raise UnsupportedDialectError(name, dialect)
 

metadata/generated/schema/analytics/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00

metadata/generated/schema/analytics/basic.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/basic.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportData.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00

metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/aggregatedCostAnalysisReportData.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/entityReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/entityReportData.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/rawCostAnalysisReportData.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/webAnalyticEntityViewReportData.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/webAnalyticUserActivityReportData.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEvent.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEventData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEventData.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEventType/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00

metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEventType/customEvent.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEventType/pageViewEvent.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00

metadata/generated/schema/api/addGlossaryToAssetsRequest.py

@@ -1,32 +1,23 @@
 # generated by datamodel-codegen:
 #   filename:  api/addGlossaryToAssetsRequest.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 
-from enum import Enum
-from typing import List, Optional
+from typing import Optional
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
-from ..type import entityReferenceList, tagLabel
-
-
-class Operation(Enum):
-    AddAssets = 'AddAssets'
-    AddGlossaryTags = 'AddGlossaryTags'
+from ..type import entityReferenceList
 
 
 class AddGlossaryToAssetsRequest(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
     )
-    operation: Annotated[
-        Optional[Operation], Field(None, description='Operation to be performed')
-    ]
     dryRun: Annotated[
         Optional[bool],
         Field(
@@ -34,10 +25,6 @@ class AddGlossaryToAssetsRequest(BaseModel):
             description='If true, the request will be validated but no changes will be made',
         ),
     ]
-    glossaryTags: Annotated[
-        Optional[List[tagLabel.TagLabel]],
-        Field(None, description='Glossary Tags to be added'),
-    ]
     assets: Annotated[
         Optional[entityReferenceList.EntityReferenceList],
         Field(

metadata/generated/schema/api/addTagToAssetsRequest.py

@@ -1,10 +1,9 @@
 # generated by datamodel-codegen:
 #   filename:  api/addTagToAssetsRequest.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 
-from enum import Enum
 from typing import Optional
 
 from pydantic import ConfigDict, Field
@@ -15,18 +14,10 @@ from metadata.ingestion.models.custom_pydantic import BaseModel
 from ..type import entityReferenceList
 
 
-class Operation(Enum):
-    AddAssets = 'AddAssets'
-    AddClassificationTags = 'AddClassificationTags'
-
-
 class AddTagToAssetsRequest(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
     )
-    operation: Annotated[
-        Optional[Operation], Field(None, description='Operation to be performed')
-    ]
     dryRun: Annotated[
         Optional[bool],
         Field(

metadata/generated/schema/api/analytics/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00

metadata/generated/schema/api/analytics/createWebAnalyticEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/analytics/createWebAnalyticEvent.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/automations/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00

metadata/generated/schema/api/automations/createWorkflow.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/automations/createWorkflow.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/bulkAssets.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/bulkAssets.json
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/classification/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-06-27T13:20:42+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00