openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/utils/dispatch.py

@@ -14,7 +14,7 @@ Helper that implements custom dispatcher logic
 """
 
 from collections import namedtuple
-from typing import Type, TypeVar
+from typing import Any, Optional, Type, TypeVar
 
 from pydantic import BaseModel
 
@@ -52,5 +52,8 @@ def class_register():
 
         return inner
 
-    Register = namedtuple("Register", ["add", "registry"])
-    return Register(add, registry)
+    def get(entity_type: Type[T], default: Optional[Any] = None):
+        return registry.get(entity_type.__name__, default)
+
+    Register = namedtuple("Register", ["add", "registry", "get"])
+    return Register(add, registry, get)

metadata/utils/elasticsearch.py

@@ -88,6 +88,9 @@ def get_entity_from_es_result(
     :param entity_list: ES query result
     :return: single entity
     """
+    if entity_list is None:
+        return None
+    entity_list = [e for e in entity_list if e is not None]
     if entity_list and len(entity_list):
         if fetch_multiple_entities:
             return entity_list

metadata/utils/fqn.py

@@ -13,9 +13,12 @@ Handle FQN building and splitting logic.
 Filter information has been taken from the
 ES indexes definitions
 """
+from __future__ import annotations
+
 import hashlib
 import re
-from typing import Dict, List, Optional, Type, TypeVar, Union
+import traceback
+from typing import TYPE_CHECKING, Dict, List, Optional, Type, TypeVar, Union
 
 from antlr4.CommonTokenStream import CommonTokenStream
 from antlr4.error.ErrorStrategy import BailErrorStrategy
@@ -50,9 +53,14 @@ from metadata.generated.schema.entity.teams.team import Team
 from metadata.generated.schema.entity.teams.user import User
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.generated.schema.tests.testSuite import TestSuite
-from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils.dispatch import class_register
 from metadata.utils.elasticsearch import get_entity_from_es_result
+from metadata.utils.logger import utils_logger
+
+if TYPE_CHECKING:
+    from metadata.ingestion.ometa.ometa_api import OpenMetadata
+
+logger = utils_logger()
 
 T = TypeVar("T", bound=BaseModel)
 
@@ -144,12 +152,32 @@ def build(
     :param kwargs: required to build the FQN
     :return: FQN as a string
     """
+    # Transform table_name and column_name if they exist and contain special characters
+    if kwargs.get("table_name") or kwargs.get("column_name"):
+        from metadata.ingestion.models.custom_basemodel_validation import (  # pylint: disable=import-outside-toplevel
+            replace_separators,
+        )
+
+        table_name = kwargs.get("table_name")
+        if table_name and isinstance(table_name, str):
+            kwargs["table_name"] = replace_separators(table_name)
+
+        column_name = kwargs.get("column_name")
+        if column_name and isinstance(column_name, str):
+            kwargs["column_name"] = replace_separators(column_name)
+
     func = fqn_build_registry.registry.get(entity_type.__name__)
-    if not func:
+    try:
+        if not func:
+            raise FQNBuildingException(
+                f"Invalid Entity Type {entity_type.__name__}. FQN builder not implemented."
+            )
+        return func(metadata, **kwargs)
+    except Exception as e:
+        logger.debug(traceback.format_exc())
         raise FQNBuildingException(
-            f"Invalid Entity Type {entity_type.__name__}. FQN builder not implemented."
+            f"Error building FQN for {entity_type.__name__}: {e}"
         )
-    return func(metadata, **kwargs)
 
 
 @fqn_build_registry.add(Table)
@@ -230,15 +258,31 @@ def _(
 
 @fqn_build_registry.add(Database)
 def _(
-    _: Optional[OpenMetadata],  # ES Search not enabled for Databases
+    metadata: Optional[OpenMetadata],
     *,
     service_name: str,
     database_name: str,
-) -> str:
+    skip_es_search: bool = True,
+    fetch_multiple_entities: bool = False,
+) -> Union[Optional[str], Optional[List[str]]]:
+    if not skip_es_search:
+        entity = search_database_from_es(
+            metadata,
+            database_name,
+            service_name,
+            fetch_multiple_entities=fetch_multiple_entities,
+        )
+
+        if entity and fetch_multiple_entities:
+            return [str(database.fullyQualifiedName.root) for database in entity]
+        if entity:
+            return str(entity.fullyQualifiedName.root)
+
     if not service_name or not database_name:
         raise FQNBuildingException(
             f"Args should be informed, but got service=`{service_name}`, db=`{database_name}``"
         )
+
     return _build(service_name, database_name)
 
 
@@ -826,3 +870,56 @@ def get_query_checksum(query: str) -> str:
     The checksum is used as the query's name.
     """
     return hashlib.md5(query.encode()).hexdigest()
+
+
+# Not adding container since children can have recursive slots: service.container1.container2...
+FQN_ENTITY_SLOTS = {
+    Table.__name__: 4,
+    DatabaseSchema.__name__: 3,
+    Database.__name__: 2,
+    Dashboard.__name__: 2,
+    APICollection.__name__: 2,
+    Chart.__name__: 2,
+    MlModel.__name__: 2,
+    Topic.__name__: 2,
+    SearchIndex.__name__: 2,
+    Tag.__name__: 2,
+    DataModel.__name__: 2,
+    StoredProcedure.__name__: 4,
+    Pipeline.__name__: 2,
+}
+
+
+def prefix_entity_for_wildcard_search(entity_type: Type[T], fqn: str) -> str:
+    """
+    Given an entity type and an FQN, return the FQN prefixed with wildcards
+    to match any parent hierarchy leading to that entity.
+
+    For example, for a Topic with FQN "potato", return "*.potato" to match
+    the topic in any service. For a Table with FQN "schema.table", return
+    "*.*.schema.table" to match the table in any service and database.
+
+    Args:
+        entity_type: The entity type to match.
+        fqn: The FQN to prefix.
+
+    Returns:
+        The prefixed FQN with wildcards for missing parent levels.
+    """
+    slots = FQN_ENTITY_SLOTS.get(entity_type.__name__)
+    if not slots:
+        raise FQNBuildingException(
+            f"Entity type {entity_type.__name__} not supported for wildcard search"
+        )
+
+    parts = split(fqn)
+    if len(parts) > slots:
+        raise FQNBuildingException(
+            f"FQN {fqn} has too many parts ({len(parts)})"
+            f"for entity type {entity_type.__name__} (expected {slots} or fewer)"
+        )
+
+    # Add wildcards for missing parent levels
+    wildcards_needed = slots - len(parts)
+    prefixed_parts = ["*"] * wildcards_needed + parts
+    return _build(*prefixed_parts, quote=True)

metadata/utils/helpers.py

@@ -105,6 +105,11 @@ om_chart_type_dict = {
     "pie": ChartType.Pie,
     "text": ChartType.Text,
     "scatter": ChartType.Scatter,
+    "gauge": ChartType.Gauge,
+    "map": ChartType.Map,
+    "graph": ChartType.Graph,
+    "heatmap": ChartType.Heatmap,
+    "timeline": ChartType.Timeline,
 }
 
 
@@ -183,7 +188,7 @@ def replace_special_with(raw: str, replacement: str) -> str:
     return re.sub(r"[^a-zA-Z0-9]", replacement, raw)
 
 
-def get_standard_chart_type(raw_chart_type: str) -> ChartType.Other:
+def get_standard_chart_type(raw_chart_type: str) -> ChartType:
     """
     Get standard chart type supported by OpenMetadata based on raw chart type input
     :param raw_chart_type: raw chart type to be standardize
@@ -202,6 +207,7 @@ def find_in_iter(element: Any, container: Iterable[Any]) -> Optional[Any]:
     :param container: container with element
     :return: element or None
     """
+    logger.debug(f"[find_in_iter] Searching for element '{element}'")
     return next((elem for elem in container if elem == element), None)
 
 
@@ -576,3 +582,14 @@ def pprint_format_object(data: Any) -> str:
     Pretty print an object in a format that is easy to read
     """
     return pprint.pformat(data, width=150)
+
+
+def can_spawn_child_process() -> bool:
+    """
+    Check if the current process can spawn a child process
+    """
+    # pylint: disable=import-outside-toplevel
+    from multiprocessing import Process
+
+    process = Process(target=lambda: None)
+    return not process.daemon

metadata/utils/importer.py

@@ -122,7 +122,7 @@ def get_class_name_root(type_: str) -> str:
     )
 
 
-def import_from_module(key: str) -> Type[Any]:
+def import_from_module(key: str, log_traceback: bool = True) -> Type[Any]:
     """
     Dynamically import an object from a module path
     """
@@ -132,7 +132,8 @@ def import_from_module(key: str) -> Type[Any]:
         obj = getattr(importlib.import_module(module_name), obj_name)
         return obj
     except (ModuleNotFoundError, ImportError) as err:
-        logger.debug(traceback.format_exc())
+        if log_traceback:
+            logger.debug(traceback.format_exc())
         raise DynamicImportException(module=module_name, key=obj_name, cause=err)
 
 

metadata/utils/logger.py

@@ -36,6 +36,7 @@ from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.models.patch_request import PatchRequest
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
+from metadata.ingestion.models.user import OMetaUserProfile
 
 METADATA_LOGGER = "metadata"
 BASE_LOGGING_FORMAT = (
@@ -212,6 +213,8 @@ def get_log_name(record: Entity) -> Optional[str]:
     try:
         if hasattr(record, "name"):
             return f"{type(record).__name__} [{getattr(record, 'name').root}]"
+        if hasattr(record, "table") and hasattr(record.table, "name"):
+            return f"{type(record).__name__} [{record.table.name.root}]"
         return f"{type(record).__name__} [{record.entity.name.root}]"
     except Exception:
         return str(record)
@@ -310,6 +313,15 @@ def _(record: DataContractResult) -> str:
     return f"DataContractResult for [{record.dataContractFQN.root}]; status: {record.contractExecutionStatus.value}]"
 
 
+@get_log_name.register
+def _(record: OMetaUserProfile) -> str:
+    """Get the log of the new entity"""
+    return (
+        f"User Profile: {get_log_name(record.user)},"
+        f"Teams: {record.teams if record.teams else 'None'}, \nRoles: {record.roles if record.roles else 'None'}"
+    )
+
+
 def redacted_config(config: Dict[str, Union[str, dict]]) -> Dict[str, Union[str, dict]]:
     config_copy = deepcopy(config)
 

metadata/utils/sqa_utils.py

@@ -274,5 +274,8 @@ def update_mssql_ischema_names(ischema_names):
             "sql_variant": create_sqlalchemy_type("SQL_VARIANT"),
             "uniqueidentifier": create_sqlalchemy_type("UUID"),
             "xml": create_sqlalchemy_type("XML"),
+            "hierarchyid": create_sqlalchemy_type("HIERARCHYID"),
+            "geography": create_sqlalchemy_type("GEOGRAPHY"),
+            "geometry": create_sqlalchemy_type("GEOMETRY"),
         }
     )

metadata/utils/ssl_manager.py

@@ -37,9 +37,15 @@ from metadata.generated.schema.entity.services.connections.database.dorisConnect
 from metadata.generated.schema.entity.services.connections.database.greenplumConnection import (
     GreenplumConnection,
 )
+from metadata.generated.schema.entity.services.connections.database.hiveConnection import (
+    HiveConnection,
+)
 from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import (
     MongoDBConnection,
 )
+from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
+    MssqlConnection,
+)
 from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
     MysqlConnection,
 )
@@ -221,11 +227,17 @@ class SSLManager:
                 "ssl.key.location": getattr(self, "key_consumer_config", None),
                 "ssl.certificate.location": getattr(self, "cert_consumer_config", None),
             }
-        connection.schemaRegistryConfig["ssl.ca.location"] = self.ca_file_path
-        connection.schemaRegistryConfig["ssl.key.location"] = self.key_file_path
-        connection.schemaRegistryConfig[
-            "ssl.certificate.location"
-        ] = self.cert_file_path
+        if connection.schemaRegistrySSL:
+            connection.schemaRegistryConfig["ssl.ca.location"] = getattr(
+                self, "ca_schema_registry", None
+            )
+
+            connection.schemaRegistryConfig["ssl.key.location"] = getattr(
+                self, "key_schema_registry", None
+            )
+            connection.schemaRegistryConfig["ssl.certificate.location"] = getattr(
+                self, "cert_schema_registry", None
+            )
         return connection
 
     @setup_ssl.register(CassandraConnection)
@@ -247,6 +259,48 @@ class SSLManager:
         connection.connectionArguments.root["ssl_context"] = ssl_context
         return connection
 
+    @setup_ssl.register(HiveConnection)
+    def _(self, connection):
+        connection = cast(HiveConnection, connection)
+
+        if not connection.connectionArguments:
+            connection.connectionArguments = init_empty_connection_arguments()
+
+        # Add certificate paths if available (following MySQL pattern)
+        ssl_args = connection.connectionArguments.root.get("ssl", {})
+        if self.ca_file_path:
+            ssl_args["ssl_ca"] = self.ca_file_path
+        if self.cert_file_path:
+            ssl_args["ssl_cert"] = self.cert_file_path
+        if self.key_file_path:
+            ssl_args["ssl_key"] = self.key_file_path
+        connection.connectionArguments.root["ssl"] = ssl_args
+
+        return connection
+
+    @setup_ssl.register(MssqlConnection)
+    def _(self, connection):
+        connection = cast(MssqlConnection, connection)
+
+        if not connection.connectionArguments:
+            connection.connectionArguments = init_empty_connection_arguments()
+
+        # Handle driver-specific SSL configuration
+        if connection.scheme.value == "mssql+pyodbc":
+            # ODBC Driver SSL parameters
+            if connection.encrypt:
+                connection.connectionArguments.root["Encrypt"] = "yes"
+
+            if connection.trustServerCertificate:
+                connection.connectionArguments.root["TrustServerCertificate"] = "yes"
+
+        elif connection.scheme.value == "mssql+pytds":
+            # pytds driver SSL parameters
+            if self.ca_file_path:
+                connection.connectionArguments.root["cafile"] = self.ca_file_path
+
+        return connection
+
 
 @singledispatch
 def check_ssl_and_init(
@@ -299,6 +353,21 @@ def _(connection):
     return None
 
 
+@check_ssl_and_init.register(MssqlConnection)
+def _(connection):
+    service_connection = cast(MssqlConnection, connection)
+    ssl: Optional[
+        verifySSLConfig.SslConfig
+    ] = service_connection.sslConfig or verifySSLConfig.SslConfig(
+        **{"caCertificate": None}
+    )
+    return SSLManager(
+        ca=ssl.root.caCertificate,
+        cert=ssl.root.sslCertificate,
+        key=ssl.root.sslKey,
+    )
+
+
 @check_ssl_and_init.register(MongoDBConnection)
 def _(connection):
     service_connection = cast(Union[MysqlConnection, DorisConnection], connection)
@@ -375,6 +444,25 @@ def _(connection):
     return None
 
 
+@check_ssl_and_init.register(HiveConnection)
+def _(connection):
+    service_connection = cast(HiveConnection, connection)
+    if hasattr(service_connection, "useSSL") and service_connection.useSSL:
+        # Check if SSL config is provided in sslConfig (following MySQL pattern)
+        if hasattr(service_connection, "sslConfig") and service_connection.sslConfig:
+            if (
+                service_connection.sslConfig.root.caCertificate
+                or service_connection.sslConfig.root.sslCertificate
+                or service_connection.sslConfig.root.sslKey
+            ):
+                return SSLManager(
+                    ca=service_connection.sslConfig.root.caCertificate,
+                    cert=service_connection.sslConfig.root.sslCertificate,
+                    key=service_connection.sslConfig.root.sslKey,
+                )
+    return None
+
+
 def get_ssl_connection(service_config):
     try:
         # To be cleaned up as part of https://github.com/open-metadata/OpenMetadata/issues/15913