openmetadata-ingestion 1.6.1.0__py3-none-any.whl → 1.6.2.0__py3-none-any.whl

metadata/utils/constants.py

@@ -12,12 +12,17 @@
 """
 Define constants useful for the metadata ingestion
 """
+from metadata.generated.schema.entity.data.apiCollection import APICollection
+from metadata.generated.schema.entity.data.apiEndpoint import APIEndpoint
 from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.container import Container
 from metadata.generated.schema.entity.data.dashboard import Dashboard
 from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
+from metadata.generated.schema.entity.data.glossary import Glossary
+from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
+from metadata.generated.schema.entity.data.metric import Metric
 from metadata.generated.schema.entity.data.mlmodel import MlModel
 from metadata.generated.schema.entity.data.pipeline import Pipeline
 from metadata.generated.schema.entity.data.searchIndex import SearchIndex
@@ -42,12 +47,6 @@ from metadata.generated.schema.entity.services.connections.database.deltaLakeCon
 from metadata.generated.schema.entity.services.connections.database.domoDatabaseConnection import (
     DomoDatabaseType,
 )
-from metadata.generated.schema.entity.services.connections.database.dorisConnection import (
-    DorisType,
-)
-from metadata.generated.schema.entity.services.connections.database.druidConnection import (
-    DruidType,
-)
 from metadata.generated.schema.entity.services.connections.database.dynamoDBConnection import (
     DynamoDBType,
 )
@@ -69,9 +68,6 @@ from metadata.generated.schema.entity.services.connections.database.sapErpConnec
 from metadata.generated.schema.entity.services.connections.database.sasConnection import (
     SasType,
 )
-from metadata.generated.schema.entity.services.connections.database.unityCatalogConnection import (
-    DatabricksType,
-)
 from metadata.generated.schema.entity.services.dashboardService import DashboardService
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.messagingService import MessagingService
@@ -131,6 +127,8 @@ ENTITY_REFERENCE_CLASS_MAP = {
     "metadataService": MetadataService,
     "searchService": SearchService,
     # Data Asset Entities
+    "apiCollection": APICollection,
+    "apiEndpoint": APIEndpoint,
     "table": Table,
     "storedProcedure": StoredProcedure,
     "database": Database,
@@ -149,6 +147,10 @@ ENTITY_REFERENCE_CLASS_MAP = {
     # Domain
     "domain": Domain,
     "dataProduct": DataProduct,
+    # Governance
+    "metric": Metric,
+    "glossary": Glossary,
+    "glossaryTerm": GlossaryTerm,
 }
 
 ENTITY_REFERENCE_TYPE_MAP = {
@@ -161,11 +163,8 @@ NON_SQA_DATABASE_CONNECTIONS = (
     DatalakeType.Datalake.value,
     BigtableType.BigTable.value,
     CouchbaseType.Couchbase.value,
-    DatabricksType.UnityCatalog.value,
     DeltaLakeType.DeltaLake.value,
     DomoDatabaseType.DomoDatabase.value,
-    DorisType.Doris.value,
-    DruidType.Druid.value,
     DynamoDBType.DynamoDB.value,
     GlueType.Glue.value,
     IcebergType.Iceberg.value,

metadata/utils/elasticsearch.py

@@ -18,6 +18,8 @@ from pydantic import BaseModel
 
 from metadata.generated.schema.analytics.reportData import ReportData
 from metadata.generated.schema.entity.classification.tag import Tag
+from metadata.generated.schema.entity.data.apiCollection import APICollection
+from metadata.generated.schema.entity.data.apiEndpoint import APIEndpoint
 from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.container import Container
 from metadata.generated.schema.entity.data.dashboard import Dashboard
@@ -26,6 +28,7 @@ from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.glossary import Glossary
 from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
+from metadata.generated.schema.entity.data.metric import Metric
 from metadata.generated.schema.entity.data.mlmodel import MlModel
 from metadata.generated.schema.entity.data.pipeline import Pipeline
 from metadata.generated.schema.entity.data.query import Query
@@ -33,6 +36,7 @@ from metadata.generated.schema.entity.data.searchIndex import SearchIndex
 from metadata.generated.schema.entity.data.storedProcedure import StoredProcedure
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.data.topic import Topic
+from metadata.generated.schema.entity.services.apiService import ApiService
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.teams.team import Team
 from metadata.generated.schema.entity.teams.user import User
@@ -42,6 +46,9 @@ logger = utils_logger()
 T = TypeVar("T", bound=BaseModel)
 
 ES_INDEX_MAP = {
+    ApiService.__name__: "api_service_search_index",
+    APICollection.__name__: "api_collection_search_index",
+    APIEndpoint.__name__: "api_endpoint_search_index",
     Table.__name__: "table_search_index",
     Database.__name__: "database_search_index",
     DatabaseSchema.__name__: "database_schema_search_index",
@@ -56,12 +63,13 @@ ES_INDEX_MAP = {
     Topic.__name__: "topic_search_index",
     Pipeline.__name__: "pipeline_search_index",
     Glossary.__name__: "glossary_search_index",
-    GlossaryTerm.__name__: "glossary_search_index",
+    GlossaryTerm.__name__: "glossary_term_search_index",
     MlModel.__name__: "mlmodel_search_index",
     Tag.__name__: "tag_search_index",
     Container.__name__: "container_search_index",
     Query.__name__: "query_search_index",
     ReportData.__name__: "entity_report_data_index",
+    Metric.__name__: "metric_search_index",
     "web_analytic_user_activity_report": "web_analytic_user_activity_report_data_index",
     "web_analytic_entity_view_report": "web_analytic_entity_view_report_data_index",
 }

metadata/workflow/data_quality.py

@@ -11,15 +11,10 @@
 """
 Workflow definition for the Data Quality
 """
-import traceback
 from typing import Optional
 
 from metadata.data_quality.processor.test_case_runner import TestCaseRunner
 from metadata.data_quality.source.test_suite import TestSuiteSource
-from metadata.generated.schema.entity.services.connections.serviceConnection import (
-    ServiceConnection,
-)
-from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.tests.testSuite import ServiceType, TestSuite
 from metadata.ingestion.api.steps import Processor, Sink
 from metadata.ingestion.ometa.utils import model_str
@@ -63,44 +58,11 @@ class TestSuiteWorkflow(IngestionWorkflow):
     def _get_test_runner_processor(self) -> Processor:
         return TestCaseRunner.create(self.config.model_dump(), self.metadata)
 
-    def _retrieve_service_connection_if_needed(self, service_type: ServiceType) -> None:
-        """Get service object from source config `entityFullyQualifiedName`"""
-        if (
-            not self.config.source.serviceConnection
-            and not self.metadata.config.forceEntityOverwriting
-        ):
-            fully_qualified_name = (
-                self.config.source.sourceConfig.config.entityFullyQualifiedName.root
-            )
-            try:
-                service_name = fqn.split(fully_qualified_name)[0]
-            except IndexError as exc:
-                logger.debug(traceback.format_exc())
-                raise IndexError(
-                    f"Could not retrieve service name from entity fully qualified name {fully_qualified_name}: {exc}"
-                )
-            try:
-                service: DatabaseService = self.metadata.get_by_name(
-                    DatabaseService, service_name
-                )
-                if not service:
-                    raise ConnectionError(
-                        f"Could not retrieve service with name `{service_name}`. "
-                        "Typically caused by the `entityFullyQualifiedName` does not exists in OpenMetadata "
-                        "or the JWT Token is invalid."
-                    )
-
-                self.config.source.serviceConnection = ServiceConnection(
-                    service.connection
-                )
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error getting service connection for service name [{service_name}]"
-                    f" using the secrets manager provider [{self.metadata.config.secretsManagerProvider}]: {exc}"
-                )
-                raise exc
+    def _retrieve_service_connection_if_needed(self, _: ServiceType) -> None:
+        """A test suite might require multiple connections (e.g., for logical test suites)
+        We'll skip this step and get the connections at runtime if they are not informed
+        in the YAML already.
+        """
 
     def _get_ingestion_pipeline_service(self) -> Optional[T]:
         """