openmetadata-ingestion 1.9.1.0__py3-none-any.whl → 1.9.2.1__py3-none-any.whl

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-08-14T07:03:44+00:00
+#   timestamp: 2025-08-22T15:30:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-08-14T07:03:44+00:00
+#   timestamp: 2025-08-22T15:30:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-08-14T07:03:44+00:00
+#   timestamp: 2025-08-22T15:30:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-08-14T07:03:44+00:00
+#   timestamp: 2025-08-22T15:30:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-08-14T07:03:44+00:00
+#   timestamp: 2025-08-22T15:30:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-08-14T07:03:44+00:00
+#   timestamp: 2025-08-22T15:30:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-08-14T07:03:44+00:00
+#   timestamp: 2025-08-22T15:30:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-08-14T07:03:44+00:00
+#   timestamp: 2025-08-22T15:30:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-08-14T07:03:44+00:00
+#   timestamp: 2025-08-22T15:30:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-08-14T07:03:44+00:00
+#   timestamp: 2025-08-22T15:30:10+00:00
 
 from __future__ import annotations
 

metadata/ingestion/models/custom_basemodel_validation.py

@@ -13,19 +13,103 @@ Validation logic for Custom Pydantic BaseModel
 """
 
 import logging
+from enum import Enum
+from typing import Any, Callable, Dict, Optional
 
 logger = logging.getLogger("metadata")
 
-
 RESTRICTED_KEYWORDS = ["::", ">"]
 RESERVED_COLON_KEYWORD = "__reserved__colon__"
 RESERVED_ARROW_KEYWORD = "__reserved__arrow__"
 RESERVED_QUOTE_KEYWORD = "__reserved__quote__"
 
-CREATE_ADJACENT_MODELS = {"ProfilerResponse", "SampleData"}
-NAME_FIELDS = {"EntityName", "str", "ColumnName", "TableData"}
-FETCH_MODELS = {"Table", "CustomColumnName", "DashboardDataModel"}
-FIELD_NAMES = {"name", "columns", "root"}
+
+class TransformDirection(Enum):
+    """Direction of name transformation"""
+
+    ENCODE = "encode"  # For storage (Create operations) - replace separators
+    DECODE = "decode"  # For display (Fetch operations) - revert separators
+
+
+def is_service_level_create_model(model_name: str) -> bool:
+    """
+    Check if a model is a Service-level Create model that should NOT be transformed.
+    Service-level models follow the pattern: Create*ServiceRequest where * is the service name
+    This is scalable and requires no maintenance for new services.
+    """
+    if not model_name.startswith("Create") or not model_name.endswith("ServiceRequest"):
+        return False
+
+    # Extract the middle part (service name) - must not be empty
+    # "CreateServiceRequest" -> middle = "" (invalid)
+    # "CreateDatabaseServiceRequest" -> middle = "Database" (valid)
+    middle = model_name[
+        6:-14
+    ]  # Remove "Create" (6 chars) and "ServiceRequest" (14 chars)
+    return len(middle) > 0
+
+
+# Explicit configuration for entity name transformations
+# This dictionary will be populated lazily to avoid circular imports
+TRANSFORMABLE_ENTITIES: Dict[Any, Dict[str, Any]] = {}
+
+
+def _initialize_transformable_entities():
+    """Initialize the transformable entities dictionary lazily to avoid circular imports"""
+    # Import all model classes here to avoid circular dependency at module load time
+    from metadata.generated.schema.api.data.createDashboardDataModel import (
+        CreateDashboardDataModelRequest,
+    )
+    from metadata.generated.schema.api.data.createTable import CreateTableRequest
+    from metadata.generated.schema.entity.data.dashboardDataModel import (
+        DashboardDataModel,
+    )
+    from metadata.generated.schema.entity.data.table import (
+        ColumnName,
+        ColumnProfile,
+        Table,
+        TableData,
+    )
+    from metadata.profiler.api.models import ProfilerResponse
+    from metadata.utils.entity_link import CustomColumnName
+
+    # Now populate the dictionary with the imported classes
+    TRANSFORMABLE_ENTITIES.update(
+        {
+            # Fetch models - decode reserved keywords back to original characters
+            Table: {
+                "fields": {"name", "columns", "children", "tableConstraints"},
+                "direction": TransformDirection.DECODE,
+            },
+            DashboardDataModel: {
+                "fields": {"name", "columns", "children"},
+                "direction": TransformDirection.DECODE,
+            },
+            CustomColumnName: {
+                "fields": {"root"},
+                "direction": TransformDirection.DECODE,
+            },
+            # Create/Store models - encode special characters to reserved keywords
+            ProfilerResponse: {
+                "fields": {"name", "profile"},
+                "direction": TransformDirection.ENCODE,
+            },
+            TableData: {"fields": {"columns"}, "direction": TransformDirection.ENCODE},
+            ColumnName: {"fields": {"root"}, "direction": TransformDirection.ENCODE},
+            CreateTableRequest: {
+                "fields": {"name", "columns", "children", "tableConstraints"},
+                "direction": TransformDirection.ENCODE,
+            },
+            CreateDashboardDataModelRequest: {
+                "fields": {"name", "columns", "children"},
+                "direction": TransformDirection.ENCODE,
+            },
+            ColumnProfile: {
+                "fields": {"name"},
+                "direction": TransformDirection.ENCODE,
+            },
+        }
+    )
 
 
 def revert_separators(value):
@@ -44,42 +128,84 @@ def replace_separators(value):
     )
 
 
-def validate_name_and_transform(values, modification_method, field_name: str = None):
-    """
-    Validate the name and transform it if needed.
-    """
-    if isinstance(values, str) and field_name in FIELD_NAMES:
-        values = modification_method(values)
-    elif (
-        hasattr(values, "root")
-        and isinstance(values.root, str)
-        and field_name in FIELD_NAMES
+def get_entity_config(model: Optional[Any]) -> Optional[Dict[str, Any]]:
+    """Get transformation configuration for entity"""
+    _initialize_transformable_entities()  # Ensure entities are loaded
+    return TRANSFORMABLE_ENTITIES.get(model)
+
+
+def get_transformer(model: Optional[Any]) -> Optional[Callable]:
+    """Get the appropriate transformer function for model"""
+    config = get_entity_config(model)
+    if not config:
+        return None
+
+    direction = config.get("direction")
+    if direction == TransformDirection.ENCODE:
+        return replace_separators
+    elif direction == TransformDirection.DECODE:
+        return revert_separators
+    return None
+
+
+def transform_all_names(obj, transformer):
+    """Transform all name fields recursively"""
+    if not obj:
+        return
+
+    # Transform name field if it exists (supports both obj.name.root and obj.root)
+    name = getattr(obj, "name", None)
+    if name and hasattr(name, "root") and name.root is not None:
+        name.root = transformer(name.root)
+    elif hasattr(obj, "root") and obj.root is not None:
+        obj.root = transformer(obj.root)
+
+    # Transform nested collections in a single loop each
+    for attr_name in ["columns", "children"]:
+        if hasattr(obj, attr_name):
+            attr_value = getattr(obj, attr_name)
+            if attr_value is not None:
+                for item in attr_value:
+                    transform_all_names(item, transformer)
+
+    # Transform table constraints
+    if hasattr(obj, "tableConstraints"):
+        table_constraints = getattr(obj, "tableConstraints")
+        if table_constraints is not None:
+            for constraint in table_constraints:
+                if hasattr(constraint, "columns"):
+                    constraint.columns = [
+                        transformer(col) for col in constraint.columns
+                    ]
+
+    if transformer == replace_separators and type(name) == str:
+        obj.name = transformer(name)
+
+
+def transform_entity_names(entity: Any, model: Optional[Any]) -> Any:
+    """Transform entity names"""
+    model_name = model.__name__
+    if not entity or (
+        model_name.startswith("Create") and is_service_level_create_model(model_name)
     ):
-        values.root = modification_method(values.root)
-    elif hasattr(type(values), "model_fields"):
-        for key in type(values).model_fields.keys():
-            if getattr(values, key):
-                if getattr(values, key).__class__.__name__ in NAME_FIELDS:
-                    setattr(
-                        values,
-                        key,
-                        validate_name_and_transform(
-                            getattr(values, key),
-                            modification_method=modification_method,
-                            field_name=key,
-                        ),
-                    )
-                elif isinstance(getattr(values, key), list):
-                    setattr(
-                        values,
-                        key,
-                        [
-                            validate_name_and_transform(
-                                item,
-                                modification_method=modification_method,
-                                field_name=key,
-                            )
-                            for item in getattr(values, key)
-                        ],
-                    )
-    return values
+        return entity
+
+    # Root attribute handling
+    if hasattr(entity, "root") and entity.root is not None:
+        entity.root = (
+            replace_separators(entity.root)
+            if model_name.startswith("Create")
+            else revert_separators(entity.root)
+        )
+        return entity
+
+    # Get model-specific transformer
+    transformer = get_transformer(model)
+    if not transformer:
+        # Fallback to original logic for backward compatibility
+        transformer = (
+            replace_separators if model_name.startswith("Create") else revert_separators
+        )
+
+    transform_all_names(entity, transformer)
+    return entity

metadata/ingestion/models/custom_pydantic.py

@@ -26,13 +26,7 @@ from pydantic.types import SecretStr
 from pydantic_core.core_schema import SerializationInfo
 from typing_extensions import Annotated
 
-from metadata.ingestion.models.custom_basemodel_validation import (
-    CREATE_ADJACENT_MODELS,
-    FETCH_MODELS,
-    replace_separators,
-    revert_separators,
-    validate_name_and_transform,
-)
+from metadata.ingestion.models.custom_basemodel_validation import transform_entity_names
 
 logger = logging.getLogger("metadata")
 
@@ -75,25 +69,18 @@ class BaseModel(PydanticBaseModel):
     @classmethod
     def parse_name(cls, values):  # pylint: disable=inconsistent-return-statements
         """
-        Primary entry point to process values based on their class.
+        Transform entity names using hybrid configuration system.
         """
 
         if not values:
-            return
+            return values
 
         try:
-
-            if cls.__name__ in CREATE_ADJACENT_MODELS or cls.__name__.startswith(
-                "Create"
-            ):
-                values = validate_name_and_transform(values, replace_separators)
-            elif cls.__name__ in FETCH_MODELS:
-                values = validate_name_and_transform(values, revert_separators)
-
+            # Try new hybrid system first
+            return transform_entity_names(entity=values, model=cls)
         except Exception as exc:
             logger.warning("Exception while parsing Basemodel: %s", exc)
-            raise exc
-        return values
+            return values
 
     def model_dump_json(  # pylint: disable=too-many-arguments
         self,

metadata/ingestion/source/database/databricks/connection.py

@@ -56,11 +56,17 @@ class DatabricksEngineWrapper:
         self.inspector = inspect(engine)
         self.schemas = None
         self.first_schema = None
+        self.first_catalog = None
 
-    def get_schemas(self):
+    def get_schemas(self, schema_name: Optional[str] = None):
         """Get schemas and cache them"""
+        if schema_name is not None:
+            with self.engine.connect() as connection:
+                connection.execute(f"USE CATALOG `{self.first_catalog}`")
+            self.first_schema = schema_name
+            return [schema_name]
         if self.schemas is None:
-            self.schemas = self.inspector.get_schema_names()
+            self.schemas = self.inspector.get_schema_names(database=self.first_catalog)
             if self.schemas:
                 # Find the first schema that's not a system schema
                 for schema in self.schemas:
@@ -81,7 +87,11 @@ class DatabricksEngineWrapper:
         if self.first_schema is None:
             self.get_schemas()  # This will set first_schema
         if self.first_schema:
-            return self.inspector.get_table_names(self.first_schema)
+            with self.engine.connect() as connection:
+                tables = connection.execute(
+                    f"SHOW TABLES IN `{self.first_catalog}`.`{self.first_schema}`"
+                )
+            return tables
         return []
 
     def get_views(self):
@@ -89,9 +99,27 @@ class DatabricksEngineWrapper:
         if self.first_schema is None:
             self.get_schemas()  # This will set first_schema
         if self.first_schema:
-            return self.inspector.get_view_names(self.first_schema)
+            with self.engine.connect() as connection:
+                views = connection.execute(
+                    f"SHOW VIEWS IN `{self.first_catalog}`.`{self.first_schema}`"
+                )
+            return views
         return []
 
+    def get_catalogs(self, catalog_name: Optional[str] = None):
+        """Get catalogs"""
+        catalogs = []
+        if catalog_name is not None:
+            self.first_catalog = catalog_name
+            return [catalog_name]
+        with self.engine.connect() as connection:
+            catalogs = connection.execute(DATABRICKS_GET_CATALOGS).fetchall()
+            for catalog in catalogs:
+                if catalog[0] != "__databricks_internal":
+                    self.first_catalog = catalog[0]
+                    break
+        return catalogs
+
 
 def get_connection_url(connection: DatabricksConnection) -> str:
     url = f"{connection.scheme.value}://token:{connection.token.get_secret_value()}@{connection.hostPort}"
@@ -144,13 +172,13 @@ def test_connection(
 
     test_fn = {
         "CheckAccess": partial(test_connection_engine_step, connection),
-        "GetSchemas": engine_wrapper.get_schemas,
+        "GetSchemas": partial(
+            engine_wrapper.get_schemas, schema_name=service_connection.databaseSchema
+        ),
         "GetTables": engine_wrapper.get_tables,
         "GetViews": engine_wrapper.get_views,
         "GetDatabases": partial(
-            test_database_query,
-            engine=connection,
-            statement=DATABRICKS_GET_CATALOGS,
+            engine_wrapper.get_catalogs, catalog_name=service_connection.catalog
         ),
         "GetQueries": partial(
             test_database_query,

metadata/ingestion/source/database/exasol/connection.py

@@ -19,6 +19,7 @@ from metadata.ingestion.connections.builders import (
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.exasol.queries import EXASOL_TEST_GET_QUERIES
 from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
@@ -89,10 +90,15 @@ def test_connection(
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
+
+    queries = {
+        "GetQueries": EXASOL_TEST_GET_QUERIES,
+    }
     return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        queries=queries,
         timeout_seconds=timeout_seconds,
     )

metadata/ingestion/source/database/exasol/lineage.py

@@ -0,0 +1,24 @@
+from metadata.ingestion.lineage.models import Dialect
+from metadata.ingestion.source.database.exasol.queries import EXASOL_SQL_STATEMENT
+from metadata.ingestion.source.database.exasol.query_parser import (
+    ExasolQueryParserSource,
+)
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class ExasolLineageSource(ExasolQueryParserSource, LineageSource):
+    """
+    Exasol class for Lineage
+    """
+
+    dialect = Dialect.EXASOL
+    sql_stmt = EXASOL_SQL_STATEMENT
+    filters = """
+        AND (
+            s.command_name IN ('MERGE', 'UPDATE', 'CREATE TABLE AS', 'CREATE VIEW')
+            OR (s.command_name = 'INSERT' AND LOWER(s.sql_text) LIKE '%insert%into%select%from%')
+    )
+    """

metadata/ingestion/source/database/exasol/metadata.py

@@ -1,5 +1,7 @@
 from typing import Optional, cast
 
+from sqlalchemy.engine.reflection import Inspector
+
 from metadata.generated.schema.entity.services.connections.database.exasolConnection import (
     ExasolConnection,
 )
@@ -9,6 +11,10 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
+from metadata.utils.sqlalchemy_utils import get_all_table_ddls, get_table_ddl
+
+Inspector.get_all_table_ddls = get_all_table_ddls
+Inspector.get_table_ddl = get_table_ddl
 
 
 class ExasolSource(CommonDbSourceService):

metadata/ingestion/source/database/exasol/queries.py

@@ -1,3 +1,25 @@
+import textwrap
+
+EXASOL_SQL_STATEMENT = textwrap.dedent(
+    """
+    SELECT
+      s.sql_text "query_text",
+      s.command_name "query_type",
+      se.user_name "user_name",
+      s.start_time "start_time",
+      s.stop_time "end_time",
+      s.duration "duration"
+    FROM EXA_DBA_AUDIT_SQL s
+    JOIN EXA_DBA_AUDIT_SESSIONS se
+    ON s.SESSION_ID = se.SESSION_ID
+    WHERE s.sql_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' 
+    AND s.sql_text NOT LIKE '/* {{"app": "dbt", %%}} */%%'
+    AND start_time between TO_TIMESTAMP('{start_time}') and TO_TIMESTAMP('{end_time}')
+    {filters}
+    LIMIT {result_limit}
+    """
+)
+
 EXASOL_TEST_GET_QUERIES = """
     SELECT
       s.sql_text,

metadata/ingestion/source/database/exasol/query_parser.py

@@ -0,0 +1,33 @@
+from abc import ABC
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.exasolConnection import (
+    ExasolConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.query_parser_source import QueryParserSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class ExasolQueryParserSource(QueryParserSource, ABC):
+    """
+    Exasol base for Usage and Lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: ExasolConnection = config.serviceConnection.root.config
+        if not isinstance(connection, ExasolConnection):
+            raise InvalidSourceException(
+                f"Expected ExasolConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/exasol/service_spec.py

@@ -1,4 +1,8 @@
+from metadata.ingestion.source.database.exasol.lineage import ExasolLineageSource
 from metadata.ingestion.source.database.exasol.metadata import ExasolSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(metadata_source_class=ExasolSource)
+ServiceSpec = DefaultDatabaseSpec(
+    metadata_source_class=ExasolSource,
+    lineage_source_class=ExasolLineageSource,
+)

metadata/ingestion/source/database/iceberg/models.py

@@ -51,24 +51,33 @@ class IcebergTable(BaseModel):
         """Responsible for parsing the needed information from a PyIceberg Table."""
         iceberg_columns = table.schema().fields
 
+        partition_columns = []
+        for partition in table.spec().fields:
+            partition_column_name = get_column_from_partition(
+                iceberg_columns, partition
+            )
+            column_partition_type = get_column_partition_type(
+                iceberg_columns, partition
+            )
+
+            if not (partition_column_name and column_partition_type):
+                continue
+
+            partition_columns.append(
+                PartitionColumnDetails(
+                    columnName=partition_column_name,
+                    intervalType=column_partition_type,
+                    interval=None,
+                )
+            )
+
         return IcebergTable(
             name=name,
             tableType=table_type,
             description=table.properties.get("comment"),
             owners=owners,
             columns=[IcebergColumnParser.parse(column) for column in iceberg_columns],
-            tablePartition=TablePartition(
-                columns=[
-                    PartitionColumnDetails(
-                        columnName=get_column_from_partition(
-                            iceberg_columns, partition
-                        ),
-                        intervalType=get_column_partition_type(
-                            iceberg_columns, partition
-                        ),
-                        interval=None,
-                    )
-                    for partition in table.spec().fields
-                ]
+            tablePartition=(
+                TablePartition(columns=partition_columns) if partition_columns else None
             ),
         )