openmetadata-ingestion 1.9.11.2__py3-none-any.whl → 1.10.0.0__py3-none-any.whl

metadata/sdk/examples/builder_end_to_end.py

@@ -0,0 +1,387 @@
+"""
+Builder-style end-to-end example (Python)
+
+This keeps a builder-only style by wrapping the SDK Create*Request
+objects with tiny chainable builders in this example file, while the
+SDK performs the actual operations.
+
+Run:
+  python -m metadata.sdk.examples.builder_end_to_end
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
+from metadata.generated.schema.api.data.createDatabaseSchema import (
+    CreateDatabaseSchemaRequest,
+)
+from metadata.generated.schema.api.data.createTable import CreateTableRequest
+from metadata.generated.schema.api.services.createDatabaseService import (
+    CreateDatabaseServiceRequest,
+)
+from metadata.generated.schema.entity.data.table import Column, ColumnName
+from metadata.generated.schema.entity.data.table import DataType as ColumnDataType
+from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
+    MysqlConnection,
+    MySQLScheme,
+    MySQLType,
+)
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseConnection,
+    DatabaseServiceType,
+)
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+)
+from metadata.sdk import OpenMetadata, OpenMetadataConfig
+from metadata.sdk.entities.database_services import DatabaseServices
+from metadata.sdk.entities.databases import Databases
+from metadata.sdk.entities.databaseschemas import DatabaseSchemas
+from metadata.sdk.entities.glossaries import Glossaries
+from metadata.sdk.entities.tables import Tables
+
+# ------------------------
+# Example-local builders
+# ------------------------
+
+
+@dataclass
+class DatabaseServiceBuilderPy:
+    name_val: Optional[str] = None
+    description_val: Optional[str] = None
+    type_val: Optional[DatabaseServiceType] = None
+    connection_val: Optional[DatabaseConnection] = None
+
+    def name(self, name: str) -> "DatabaseServiceBuilderPy":
+        self.name_val = name
+        return self
+
+    def description(self, desc: str) -> "DatabaseServiceBuilderPy":
+        self.description_val = desc
+        return self
+
+    def service_type(self, st: DatabaseServiceType) -> "DatabaseServiceBuilderPy":
+        self.type_val = st
+        return self
+
+    def mysql_connection(
+        self, host_port: str, username: str, database: Optional[str] = None
+    ) -> "DatabaseServiceBuilderPy":
+        conn = DatabaseConnection(
+            config=MysqlConnection(
+                type=MySQLType.Mysql,
+                scheme=MySQLScheme.mysql_pymysql,
+                username=username,
+                authType=None,
+                hostPort=host_port,
+                databaseName=database,
+                databaseSchema=None,
+                sslConfig=None,
+                connectionOptions=None,
+                connectionArguments=None,
+                schemaFilterPattern=None,
+                tableFilterPattern=None,
+                databaseFilterPattern=None,
+                supportsMetadataExtraction=None,
+                supportsDBTExtraction=None,
+                supportsProfiler=None,
+                supportsQueryComment=None,
+                sampleDataStorageConfig=None,
+                supportsDataDiff=None,
+                supportsUsageExtraction=None,
+                supportsLineageExtraction=None,
+                useSlowLogs=False,
+            )
+        )
+        self.connection_val = conn
+        self.type_val = DatabaseServiceType.Mysql
+        return self
+
+    def build(self) -> CreateDatabaseServiceRequest:
+        if not self.name_val:
+            raise ValueError("Service name is required")
+        if not self.type_val:
+            raise ValueError("Service type is required")
+        return CreateDatabaseServiceRequest(
+            name=EntityName(self.name_val),
+            description=Markdown(self.description_val)
+            if self.description_val
+            else None,
+            serviceType=self.type_val,
+            connection=self.connection_val,
+            displayName=None,
+            tags=None,
+            owners=None,
+            dataProducts=None,
+            domains=None,
+            ingestionRunner=None,
+        )
+
+    def create(self):
+        return DatabaseServices.create(self.build())
+
+
+@dataclass
+class DatabaseBuilderPy:
+    name_val: Optional[str] = None
+    description_val: Optional[str] = None
+    service_fqn_val: Optional[str] = None
+
+    def name(self, name: str) -> "DatabaseBuilderPy":
+        self.name_val = name
+        return self
+
+    def description(self, desc: str) -> "DatabaseBuilderPy":
+        self.description_val = desc
+        return self
+
+    def in_service(self, service_fqn: str) -> "DatabaseBuilderPy":
+        self.service_fqn_val = service_fqn
+        return self
+
+    def build(self) -> CreateDatabaseRequest:
+        if not self.name_val:
+            raise ValueError("Database name is required")
+        if not self.service_fqn_val:
+            raise ValueError("Database service FQN is required")
+        return CreateDatabaseRequest(
+            name=EntityName(self.name_val),
+            description=Markdown(self.description_val)
+            if self.description_val
+            else None,
+            service=FullyQualifiedEntityName(self.service_fqn_val),
+            displayName=None,
+            tags=None,
+            owners=None,
+            dataProducts=None,
+            default=False,
+            retentionPeriod=None,
+            extension=None,
+            sourceUrl=None,
+            domains=None,
+            lifeCycle=None,
+            sourceHash=None,
+        )
+
+    def create(self):
+        return Databases.create(self.build())
+
+
+@dataclass
+class SchemaBuilderPy:
+    name_val: Optional[str] = None
+    description_val: Optional[str] = None
+    database_fqn_val: Optional[str] = None
+
+    def name(self, name: str) -> "SchemaBuilderPy":
+        self.name_val = name
+        return self
+
+    def description(self, desc: str) -> "SchemaBuilderPy":
+        self.description_val = desc
+        return self
+
+    def in_database(self, database_fqn: str) -> "SchemaBuilderPy":
+        self.database_fqn_val = database_fqn
+        return self
+
+    def build(self) -> CreateDatabaseSchemaRequest:
+        if not self.name_val:
+            raise ValueError("Schema name is required")
+        if not self.database_fqn_val:
+            raise ValueError("Database FQN is required")
+        return CreateDatabaseSchemaRequest(
+            name=EntityName(self.name_val),
+            description=Markdown(self.description_val)
+            if self.description_val
+            else None,
+            database=FullyQualifiedEntityName(self.database_fqn_val),
+            displayName=None,
+            owners=None,
+            dataProducts=None,
+            tags=None,
+            retentionPeriod=None,
+            extension=None,
+            sourceUrl=None,
+            domains=None,
+            lifeCycle=None,
+            sourceHash=None,
+        )
+
+    def create(self):
+        return DatabaseSchemas.create(self.build())
+
+
+@dataclass
+class TableBuilderPy:
+    name_val: Optional[str] = None
+    description_val: Optional[str] = None
+    schema_fqn_val: Optional[str] = None
+    columns_val: List[Column] = field(default_factory=list)
+
+    def name(self, name: str) -> "TableBuilderPy":
+        self.name_val = name
+        return self
+
+    def description(self, desc: str) -> "TableBuilderPy":
+        self.description_val = desc
+        return self
+
+    def in_schema(self, schema_fqn: str) -> "TableBuilderPy":
+        self.schema_fqn_val = schema_fqn
+        return self
+
+    def add_column(
+        self, name: str, dtype: ColumnDataType, *, length: Optional[int] = None
+    ) -> "TableBuilderPy":
+        col = Column(
+            name=ColumnName(name),
+            displayName=None,
+            dataType=dtype,
+            arrayDataType=None,
+            dataLength=length if (dtype == ColumnDataType.VARCHAR and length) else None,
+            precision=None,
+            scale=None,
+            dataTypeDisplay=None,
+            description=None,
+            fullyQualifiedName=None,
+            tags=None,
+            constraint=None,
+            ordinalPosition=None,
+            jsonSchema=None,
+            children=None,
+            profile=None,
+            customMetrics=None,
+        )
+        self.columns_val.append(col)
+        return self
+
+    def build(self) -> CreateTableRequest:
+        if not self.name_val:
+            raise ValueError("Table name is required")
+        if not self.schema_fqn_val:
+            raise ValueError("Schema FQN is required")
+        if not self.columns_val:
+            raise ValueError("At least one column is required")
+        return CreateTableRequest(
+            name=EntityName(self.name_val),
+            description=Markdown(self.description_val)
+            if self.description_val
+            else None,
+            databaseSchema=FullyQualifiedEntityName(self.schema_fqn_val),
+            columns=self.columns_val,
+            displayName=None,
+            tableType=None,
+            dataModel=None,
+            locationPath=None,
+            tableConstraints=None,
+            tablePartition=None,
+            tableProfilerConfig=None,
+            owners=None,
+            tags=None,
+            schemaDefinition=None,
+            retentionPeriod=None,
+            extension=None,
+            sourceUrl=None,
+            domains=None,
+            dataProducts=None,
+            fileFormat=None,
+            lifeCycle=None,
+            sourceHash=None,
+        )
+
+    def create(self):
+        return Tables.create(self.build())
+
+
+def main() -> None:
+    config = OpenMetadataConfig(
+        server_url="http://localhost:8585",
+        jwt_token="YOUR_JWT_OR_API_KEY",
+        verify_ssl=False,
+    )
+    _ = OpenMetadata.initialize(config)
+
+    # 1) Service (builder)
+    service = (
+        DatabaseServiceBuilderPy()
+        .name("mysql_prod")
+        .description("Production MySQL")
+        .mysql_connection(
+            host_port="localhost:3306", username="om_user", database="prod"
+        )
+        .create()
+    )
+    service_fqn = (
+        service.fullyQualifiedName.root
+        if service.fullyQualifiedName
+        else str(service.name.root)
+    )
+
+    # 2) Database (builder)
+    database = (
+        DatabaseBuilderPy()
+        .name("sales")
+        .description("Sales database")
+        .in_service(service_fqn)
+        .create()
+    )
+    database_fqn = (
+        database.fullyQualifiedName.root
+        if database.fullyQualifiedName
+        else str(database.name.root)
+    )
+
+    # 3) Schema (builder)
+    schema = (
+        SchemaBuilderPy()
+        .name("public")
+        .description("Default schema")
+        .in_database(database_fqn)
+        .create()
+    )
+    schema_fqn = (
+        schema.fullyQualifiedName.root
+        if schema.fullyQualifiedName
+        else str(schema.name.root)
+    )
+
+    # 4) Table (builder)
+    table = (
+        TableBuilderPy()
+        .name("customers")
+        .description("Customer master table")
+        .in_schema(schema_fqn)
+        .add_column("id", ColumnDataType.BIGINT)
+        .add_column("email", ColumnDataType.VARCHAR, length=255)
+        .create()
+    )
+    # 5) Update description (builder-like: reuse entity, call SDK update)
+    table.description = Markdown(root="Updated description: includes PII columns")
+    table = Tables.update(table)
+
+    # Add tag via helper (still chained by intent)
+    table = Tables.add_tag(table.id.root, "PII.Sensitive")
+
+    # 6) Glossary export/import using SDK's CSV operations
+    glossary_name = "BusinessGlossary"  # adjust to your glossary
+    csv_text = Glossaries.export_csv(glossary_name).execute()
+    # dry run
+    _ = (
+        Glossaries.import_csv(glossary_name)
+        .set_dry_run(True)
+        .with_data(csv_text)
+        .execute()
+    )
+    # apply
+    _ = Glossaries.import_csv(glossary_name).with_data(csv_text).execute()
+
+    print("Completed builder-based example successfully.")
+
+
+if __name__ == "__main__":
+    main()

metadata/sdk/types.py

@@ -0,0 +1,15 @@
+"""Common type aliases for the OpenMetadata SDK."""
+from __future__ import annotations
+
+from typing import Any, TypeAlias
+from uuid import UUID
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+from metadata.ingestion.ometa.ometa_api import OpenMetadata as _OMeta
+
+JsonDict: TypeAlias = dict[str, Any]
+UuidLike: TypeAlias = str | UUID
+
+OMetaClient: TypeAlias = _OMeta[BaseModel, BaseModel]
+
+__all__ = ["JsonDict", "UuidLike", "OMetaClient"]

metadata/utils/datalake/datalake_utils.py

@@ -542,8 +542,129 @@ class JsonDataFrameColumnParser(GenericDataFrameColumnParser):
         """
         if self.raw_data:
             try:
+                # First, check if this is an Iceberg/Delta Lake metadata file
+                data = json.loads(self.raw_data)
+                if self._is_iceberg_delta_metadata(data):
+                    return self._parse_iceberg_delta_schema(data)
+                # Otherwise, try to parse as standard JSON Schema
                 return parse_json_schema(schema_text=self.raw_data, cls=Column)
             except Exception as exc:
                 logger.warning(f"Unable to parse the json schema: {exc}")
                 logger.debug(traceback.format_exc())
         return self._get_columns(self.data_frame)
+
+    def _is_iceberg_delta_metadata(self, data: dict) -> bool:
+        """
+        Check if the JSON data is an Iceberg or Delta Lake metadata file.
+        These files have a specific structure with 'schema' containing 'fields'.
+        """
+        return (
+            isinstance(data, dict)
+            and "schema" in data
+            and isinstance(data["schema"], dict)
+            and "fields" in data["schema"]
+            and isinstance(data["schema"]["fields"], list)
+        )
+
+    def _parse_iceberg_delta_schema(self, data: dict) -> List[Column]:
+        """
+        Parse Iceberg/Delta Lake metadata file schema to extract columns.
+        These files have structure: {"schema": {"fields": [{"id": ..., "name": ..., "type": ..., "required": ...}, ...]}}
+        """
+        columns = []
+        schema = data.get("schema", {})
+        fields = schema.get("fields", [])
+
+        for field in fields:
+            try:
+                column_name = field.get("name", "")
+                column_type = field.get("type", "string")
+
+                # Get the type string from dict if needed
+                type_str = column_type
+                if isinstance(column_type, dict):
+                    type_str = column_type.get("type", "string")
+
+                # Use DataType enum directly - it will handle the conversion
+                try:
+                    data_type = (
+                        DataType(type_str.upper())
+                        if isinstance(type_str, str)
+                        else DataType.STRING
+                    )
+                except (ValueError, AttributeError):
+                    # If the type is not recognized, default to STRING
+                    data_type = DataType.STRING
+
+                column = Column(
+                    name=truncate_column_name(column_name),
+                    displayName=column_name,
+                    dataType=data_type,
+                    dataTypeDisplay=column_type
+                    if isinstance(column_type, str)
+                    else str(column_type),
+                )
+
+                # Handle nested struct types
+                if (
+                    isinstance(column_type, dict)
+                    and column_type.get("type") == "struct"
+                ):
+                    column.children = self._parse_struct_fields(
+                        column_type.get("fields", [])
+                    )
+                    column.dataType = DataType.STRUCT
+
+                columns.append(column)
+            except Exception as exc:
+                logger.warning(f"Unable to parse field {field}: {exc}")
+                logger.debug(traceback.format_exc())
+
+        return columns
+
+    def _parse_struct_fields(self, fields: list) -> List[dict]:
+        """
+        Parse nested struct fields in Iceberg/Delta Lake metadata.
+        """
+        children = []
+        for field in fields:
+            try:
+                child_name = field.get("name", "")
+                child_type = field.get("type", "string")
+
+                # Get the type string from dict if needed
+                type_str = child_type
+                if isinstance(child_type, dict):
+                    type_str = child_type.get("type", "string")
+
+                # Use DataType enum directly
+                try:
+                    data_type = (
+                        DataType(type_str.upper())
+                        if isinstance(type_str, str)
+                        else DataType.STRING
+                    )
+                except (ValueError, AttributeError):
+                    data_type = DataType.STRING
+
+                child = {
+                    "name": truncate_column_name(child_name),
+                    "displayName": child_name,
+                    "dataType": data_type.value,
+                    "dataTypeDisplay": child_type
+                    if isinstance(child_type, str)
+                    else str(child_type),
+                }
+
+                # Recursively handle nested structs
+                if isinstance(child_type, dict) and child_type.get("type") == "struct":
+                    child["children"] = self._parse_struct_fields(
+                        child_type.get("fields", [])
+                    )
+
+                children.append(child)
+            except Exception as exc:
+                logger.warning(f"Unable to parse nested field {field}: {exc}")
+                logger.debug(traceback.format_exc())
+
+        return children

metadata/utils/elasticsearch.py

@@ -88,6 +88,9 @@ def get_entity_from_es_result(
     :param entity_list: ES query result
     :return: single entity
     """
+    if entity_list is None:
+        return None
+    entity_list = [e for e in entity_list if e is not None]
     if entity_list and len(entity_list):
         if fetch_multiple_entities:
             return entity_list

metadata/utils/fqn.py

@@ -254,15 +254,31 @@ def _(
 
 @fqn_build_registry.add(Database)
 def _(
-    _: Optional[OpenMetadata],  # ES Search not enabled for Databases
+    metadata: Optional[OpenMetadata],
     *,
     service_name: str,
     database_name: str,
-) -> str:
+    skip_es_search: bool = True,
+    fetch_multiple_entities: bool = False,
+) -> Union[Optional[str], Optional[List[str]]]:
+    if not skip_es_search:
+        entity = search_database_from_es(
+            metadata,
+            database_name,
+            service_name,
+            fetch_multiple_entities=fetch_multiple_entities,
+        )
+
+        if entity and fetch_multiple_entities:
+            return [str(database.fullyQualifiedName.root) for database in entity]
+        if entity:
+            return str(entity.fullyQualifiedName.root)
+
     if not service_name or not database_name:
         raise FQNBuildingException(
             f"Args should be informed, but got service=`{service_name}`, db=`{database_name}``"
         )
+
     return _build(service_name, database_name)
 
 

metadata/utils/importer.py

@@ -122,7 +122,7 @@ def get_class_name_root(type_: str) -> str:
     )
 
 
-def import_from_module(key: str) -> Type[Any]:
+def import_from_module(key: str, log_traceback: bool = True) -> Type[Any]:
     """
     Dynamically import an object from a module path
     """
@@ -132,7 +132,8 @@ def import_from_module(key: str) -> Type[Any]:
         obj = getattr(importlib.import_module(module_name), obj_name)
         return obj
     except (ModuleNotFoundError, ImportError) as err:
-        logger.debug(traceback.format_exc())
+        if log_traceback:
+            logger.debug(traceback.format_exc())
         raise DynamicImportException(module=module_name, key=obj_name, cause=err)
 
 

metadata/utils/logger.py

@@ -36,6 +36,7 @@ from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.models.patch_request import PatchRequest
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
+from metadata.ingestion.models.user import OMetaUserProfile
 
 METADATA_LOGGER = "metadata"
 BASE_LOGGING_FORMAT = (
@@ -312,6 +313,15 @@ def _(record: DataContractResult) -> str:
     return f"DataContractResult for [{record.dataContractFQN.root}]; status: {record.contractExecutionStatus.value}]"
 
 
+@get_log_name.register
+def _(record: OMetaUserProfile) -> str:
+    """Get the log of the new entity"""
+    return (
+        f"User Profile: {get_log_name(record.user)},"
+        f"Teams: {record.teams if record.teams else 'None'}, \nRoles: {record.roles if record.roles else 'None'}"
+    )
+
+
 def redacted_config(config: Dict[str, Union[str, dict]]) -> Dict[str, Union[str, dict]]:
     config_copy = deepcopy(config)
 

metadata/utils/sqa_utils.py

@@ -274,5 +274,8 @@ def update_mssql_ischema_names(ischema_names):
             "sql_variant": create_sqlalchemy_type("SQL_VARIANT"),
             "uniqueidentifier": create_sqlalchemy_type("UUID"),
             "xml": create_sqlalchemy_type("XML"),
+            "hierarchyid": create_sqlalchemy_type("HIERARCHYID"),
+            "geography": create_sqlalchemy_type("GEOGRAPHY"),
+            "geometry": create_sqlalchemy_type("GEOMETRY"),
         }
     )