openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/sdk/examples/builder_end_to_end.py

@@ -0,0 +1,387 @@
+"""
+Builder-style end-to-end example (Python)
+
+This keeps a builder-only style by wrapping the SDK Create*Request
+objects with tiny chainable builders in this example file, while the
+SDK performs the actual operations.
+
+Run:
+  python -m metadata.sdk.examples.builder_end_to_end
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
+from metadata.generated.schema.api.data.createDatabaseSchema import (
+    CreateDatabaseSchemaRequest,
+)
+from metadata.generated.schema.api.data.createTable import CreateTableRequest
+from metadata.generated.schema.api.services.createDatabaseService import (
+    CreateDatabaseServiceRequest,
+)
+from metadata.generated.schema.entity.data.table import Column, ColumnName
+from metadata.generated.schema.entity.data.table import DataType as ColumnDataType
+from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
+    MysqlConnection,
+    MySQLScheme,
+    MySQLType,
+)
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseConnection,
+    DatabaseServiceType,
+)
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+)
+from metadata.sdk import OpenMetadata, OpenMetadataConfig
+from metadata.sdk.entities.database_services import DatabaseServices
+from metadata.sdk.entities.databases import Databases
+from metadata.sdk.entities.databaseschemas import DatabaseSchemas
+from metadata.sdk.entities.glossaries import Glossaries
+from metadata.sdk.entities.tables import Tables
+
+# ------------------------
+# Example-local builders
+# ------------------------
+
+
+@dataclass
+class DatabaseServiceBuilderPy:
+    name_val: Optional[str] = None
+    description_val: Optional[str] = None
+    type_val: Optional[DatabaseServiceType] = None
+    connection_val: Optional[DatabaseConnection] = None
+
+    def name(self, name: str) -> "DatabaseServiceBuilderPy":
+        self.name_val = name
+        return self
+
+    def description(self, desc: str) -> "DatabaseServiceBuilderPy":
+        self.description_val = desc
+        return self
+
+    def service_type(self, st: DatabaseServiceType) -> "DatabaseServiceBuilderPy":
+        self.type_val = st
+        return self
+
+    def mysql_connection(
+        self, host_port: str, username: str, database: Optional[str] = None
+    ) -> "DatabaseServiceBuilderPy":
+        conn = DatabaseConnection(
+            config=MysqlConnection(
+                type=MySQLType.Mysql,
+                scheme=MySQLScheme.mysql_pymysql,
+                username=username,
+                authType=None,
+                hostPort=host_port,
+                databaseName=database,
+                databaseSchema=None,
+                sslConfig=None,
+                connectionOptions=None,
+                connectionArguments=None,
+                schemaFilterPattern=None,
+                tableFilterPattern=None,
+                databaseFilterPattern=None,
+                supportsMetadataExtraction=None,
+                supportsDBTExtraction=None,
+                supportsProfiler=None,
+                supportsQueryComment=None,
+                sampleDataStorageConfig=None,
+                supportsDataDiff=None,
+                supportsUsageExtraction=None,
+                supportsLineageExtraction=None,
+                useSlowLogs=False,
+            )
+        )
+        self.connection_val = conn
+        self.type_val = DatabaseServiceType.Mysql
+        return self
+
+    def build(self) -> CreateDatabaseServiceRequest:
+        if not self.name_val:
+            raise ValueError("Service name is required")
+        if not self.type_val:
+            raise ValueError("Service type is required")
+        return CreateDatabaseServiceRequest(
+            name=EntityName(self.name_val),
+            description=Markdown(self.description_val)
+            if self.description_val
+            else None,
+            serviceType=self.type_val,
+            connection=self.connection_val,
+            displayName=None,
+            tags=None,
+            owners=None,
+            dataProducts=None,
+            domains=None,
+            ingestionRunner=None,
+        )
+
+    def create(self):
+        return DatabaseServices.create(self.build())
+
+
+@dataclass
+class DatabaseBuilderPy:
+    name_val: Optional[str] = None
+    description_val: Optional[str] = None
+    service_fqn_val: Optional[str] = None
+
+    def name(self, name: str) -> "DatabaseBuilderPy":
+        self.name_val = name
+        return self
+
+    def description(self, desc: str) -> "DatabaseBuilderPy":
+        self.description_val = desc
+        return self
+
+    def in_service(self, service_fqn: str) -> "DatabaseBuilderPy":
+        self.service_fqn_val = service_fqn
+        return self
+
+    def build(self) -> CreateDatabaseRequest:
+        if not self.name_val:
+            raise ValueError("Database name is required")
+        if not self.service_fqn_val:
+            raise ValueError("Database service FQN is required")
+        return CreateDatabaseRequest(
+            name=EntityName(self.name_val),
+            description=Markdown(self.description_val)
+            if self.description_val
+            else None,
+            service=FullyQualifiedEntityName(self.service_fqn_val),
+            displayName=None,
+            tags=None,
+            owners=None,
+            dataProducts=None,
+            default=False,
+            retentionPeriod=None,
+            extension=None,
+            sourceUrl=None,
+            domains=None,
+            lifeCycle=None,
+            sourceHash=None,
+        )
+
+    def create(self):
+        return Databases.create(self.build())
+
+
+@dataclass
+class SchemaBuilderPy:
+    name_val: Optional[str] = None
+    description_val: Optional[str] = None
+    database_fqn_val: Optional[str] = None
+
+    def name(self, name: str) -> "SchemaBuilderPy":
+        self.name_val = name
+        return self
+
+    def description(self, desc: str) -> "SchemaBuilderPy":
+        self.description_val = desc
+        return self
+
+    def in_database(self, database_fqn: str) -> "SchemaBuilderPy":
+        self.database_fqn_val = database_fqn
+        return self
+
+    def build(self) -> CreateDatabaseSchemaRequest:
+        if not self.name_val:
+            raise ValueError("Schema name is required")
+        if not self.database_fqn_val:
+            raise ValueError("Database FQN is required")
+        return CreateDatabaseSchemaRequest(
+            name=EntityName(self.name_val),
+            description=Markdown(self.description_val)
+            if self.description_val
+            else None,
+            database=FullyQualifiedEntityName(self.database_fqn_val),
+            displayName=None,
+            owners=None,
+            dataProducts=None,
+            tags=None,
+            retentionPeriod=None,
+            extension=None,
+            sourceUrl=None,
+            domains=None,
+            lifeCycle=None,
+            sourceHash=None,
+        )
+
+    def create(self):
+        return DatabaseSchemas.create(self.build())
+
+
+@dataclass
+class TableBuilderPy:
+    name_val: Optional[str] = None
+    description_val: Optional[str] = None
+    schema_fqn_val: Optional[str] = None
+    columns_val: List[Column] = field(default_factory=list)
+
+    def name(self, name: str) -> "TableBuilderPy":
+        self.name_val = name
+        return self
+
+    def description(self, desc: str) -> "TableBuilderPy":
+        self.description_val = desc
+        return self
+
+    def in_schema(self, schema_fqn: str) -> "TableBuilderPy":
+        self.schema_fqn_val = schema_fqn
+        return self
+
+    def add_column(
+        self, name: str, dtype: ColumnDataType, *, length: Optional[int] = None
+    ) -> "TableBuilderPy":
+        col = Column(
+            name=ColumnName(name),
+            displayName=None,
+            dataType=dtype,
+            arrayDataType=None,
+            dataLength=length if (dtype == ColumnDataType.VARCHAR and length) else None,
+            precision=None,
+            scale=None,
+            dataTypeDisplay=None,
+            description=None,
+            fullyQualifiedName=None,
+            tags=None,
+            constraint=None,
+            ordinalPosition=None,
+            jsonSchema=None,
+            children=None,
+            profile=None,
+            customMetrics=None,
+        )
+        self.columns_val.append(col)
+        return self
+
+    def build(self) -> CreateTableRequest:
+        if not self.name_val:
+            raise ValueError("Table name is required")
+        if not self.schema_fqn_val:
+            raise ValueError("Schema FQN is required")
+        if not self.columns_val:
+            raise ValueError("At least one column is required")
+        return CreateTableRequest(
+            name=EntityName(self.name_val),
+            description=Markdown(self.description_val)
+            if self.description_val
+            else None,
+            databaseSchema=FullyQualifiedEntityName(self.schema_fqn_val),
+            columns=self.columns_val,
+            displayName=None,
+            tableType=None,
+            dataModel=None,
+            locationPath=None,
+            tableConstraints=None,
+            tablePartition=None,
+            tableProfilerConfig=None,
+            owners=None,
+            tags=None,
+            schemaDefinition=None,
+            retentionPeriod=None,
+            extension=None,
+            sourceUrl=None,
+            domains=None,
+            dataProducts=None,
+            fileFormat=None,
+            lifeCycle=None,
+            sourceHash=None,
+        )
+
+    def create(self):
+        return Tables.create(self.build())
+
+
+def main() -> None:
+    config = OpenMetadataConfig(
+        server_url="http://localhost:8585",
+        jwt_token="YOUR_JWT_OR_API_KEY",
+        verify_ssl=False,
+    )
+    _ = OpenMetadata.initialize(config)
+
+    # 1) Service (builder)
+    service = (
+        DatabaseServiceBuilderPy()
+        .name("mysql_prod")
+        .description("Production MySQL")
+        .mysql_connection(
+            host_port="localhost:3306", username="om_user", database="prod"
+        )
+        .create()
+    )
+    service_fqn = (
+        service.fullyQualifiedName.root
+        if service.fullyQualifiedName
+        else str(service.name.root)
+    )
+
+    # 2) Database (builder)
+    database = (
+        DatabaseBuilderPy()
+        .name("sales")
+        .description("Sales database")
+        .in_service(service_fqn)
+        .create()
+    )
+    database_fqn = (
+        database.fullyQualifiedName.root
+        if database.fullyQualifiedName
+        else str(database.name.root)
+    )
+
+    # 3) Schema (builder)
+    schema = (
+        SchemaBuilderPy()
+        .name("public")
+        .description("Default schema")
+        .in_database(database_fqn)
+        .create()
+    )
+    schema_fqn = (
+        schema.fullyQualifiedName.root
+        if schema.fullyQualifiedName
+        else str(schema.name.root)
+    )
+
+    # 4) Table (builder)
+    table = (
+        TableBuilderPy()
+        .name("customers")
+        .description("Customer master table")
+        .in_schema(schema_fqn)
+        .add_column("id", ColumnDataType.BIGINT)
+        .add_column("email", ColumnDataType.VARCHAR, length=255)
+        .create()
+    )
+    # 5) Update description (builder-like: reuse entity, call SDK update)
+    table.description = Markdown(root="Updated description: includes PII columns")
+    table = Tables.update(table)
+
+    # Add tag via helper (still chained by intent)
+    table = Tables.add_tag(table.id.root, "PII.Sensitive")
+
+    # 6) Glossary export/import using SDK's CSV operations
+    glossary_name = "BusinessGlossary"  # adjust to your glossary
+    csv_text = Glossaries.export_csv(glossary_name).execute()
+    # dry run
+    _ = (
+        Glossaries.import_csv(glossary_name)
+        .set_dry_run(True)
+        .with_data(csv_text)
+        .execute()
+    )
+    # apply
+    _ = Glossaries.import_csv(glossary_name).with_data(csv_text).execute()
+
+    print("Completed builder-based example successfully.")
+
+
+if __name__ == "__main__":
+    main()

metadata/sdk/types.py

@@ -0,0 +1,15 @@
+"""Common type aliases for the OpenMetadata SDK."""
+from __future__ import annotations
+
+from typing import Any, TypeAlias
+from uuid import UUID
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+from metadata.ingestion.ometa.ometa_api import OpenMetadata as _OMeta
+
+JsonDict: TypeAlias = dict[str, Any]
+UuidLike: TypeAlias = str | UUID
+
+OMetaClient: TypeAlias = _OMeta[BaseModel, BaseModel]
+
+__all__ = ["JsonDict", "UuidLike", "OMetaClient"]

metadata/utils/bigquery_utils.py

@@ -14,9 +14,7 @@ Utils module of BigQuery
 """
 
 from copy import deepcopy
-from typing import List, Optional
-
-from google.cloud import bigquery
+from typing import TYPE_CHECKING, List, Optional
 
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
@@ -34,6 +32,9 @@ from metadata.utils.credentials import (
     get_gcp_impersonate_credentials,
 )
 
+if TYPE_CHECKING:
+    from google.cloud import bigquery
+
 
 def get_bigquery_client(
     project_id: Optional[str] = None,
@@ -42,7 +43,7 @@ def get_bigquery_client(
     quota_project_id: Optional[str] = None,
     scopes: Optional[List[str]] = None,
     lifetime: Optional[int] = 3600,
-) -> bigquery.Client:
+) -> "bigquery.Client":
     """Get a BigQuery client
 
     Args:
@@ -65,6 +66,8 @@ def get_bigquery_client(
             scopes=scopes,
             lifetime=lifetime,
         )
+    from google.cloud import bigquery  # pylint: disable=import-outside-toplevel
+
     return bigquery.Client(
         credentials=credentials, project=project_id, location=location
     )

metadata/utils/datalake/datalake_utils.py

@@ -542,8 +542,129 @@ class JsonDataFrameColumnParser(GenericDataFrameColumnParser):
         """
         if self.raw_data:
             try:
+                # First, check if this is an Iceberg/Delta Lake metadata file
+                data = json.loads(self.raw_data)
+                if self._is_iceberg_delta_metadata(data):
+                    return self._parse_iceberg_delta_schema(data)
+                # Otherwise, try to parse as standard JSON Schema
                 return parse_json_schema(schema_text=self.raw_data, cls=Column)
             except Exception as exc:
                 logger.warning(f"Unable to parse the json schema: {exc}")
                 logger.debug(traceback.format_exc())
         return self._get_columns(self.data_frame)
+
+    def _is_iceberg_delta_metadata(self, data: dict) -> bool:
+        """
+        Check if the JSON data is an Iceberg or Delta Lake metadata file.
+        These files have a specific structure with 'schema' containing 'fields'.
+        """
+        return (
+            isinstance(data, dict)
+            and "schema" in data
+            and isinstance(data["schema"], dict)
+            and "fields" in data["schema"]
+            and isinstance(data["schema"]["fields"], list)
+        )
+
+    def _parse_iceberg_delta_schema(self, data: dict) -> List[Column]:
+        """
+        Parse Iceberg/Delta Lake metadata file schema to extract columns.
+        These files have structure: {"schema": {"fields": [{"id": ..., "name": ..., "type": ..., "required": ...}, ...]}}
+        """
+        columns = []
+        schema = data.get("schema", {})
+        fields = schema.get("fields", [])
+
+        for field in fields:
+            try:
+                column_name = field.get("name", "")
+                column_type = field.get("type", "string")
+
+                # Get the type string from dict if needed
+                type_str = column_type
+                if isinstance(column_type, dict):
+                    type_str = column_type.get("type", "string")
+
+                # Use DataType enum directly - it will handle the conversion
+                try:
+                    data_type = (
+                        DataType(type_str.upper())
+                        if isinstance(type_str, str)
+                        else DataType.STRING
+                    )
+                except (ValueError, AttributeError):
+                    # If the type is not recognized, default to STRING
+                    data_type = DataType.STRING
+
+                column = Column(
+                    name=truncate_column_name(column_name),
+                    displayName=column_name,
+                    dataType=data_type,
+                    dataTypeDisplay=column_type
+                    if isinstance(column_type, str)
+                    else str(column_type),
+                )
+
+                # Handle nested struct types
+                if (
+                    isinstance(column_type, dict)
+                    and column_type.get("type") == "struct"
+                ):
+                    column.children = self._parse_struct_fields(
+                        column_type.get("fields", [])
+                    )
+                    column.dataType = DataType.STRUCT
+
+                columns.append(column)
+            except Exception as exc:
+                logger.warning(f"Unable to parse field {field}: {exc}")
+                logger.debug(traceback.format_exc())
+
+        return columns
+
+    def _parse_struct_fields(self, fields: list) -> List[dict]:
+        """
+        Parse nested struct fields in Iceberg/Delta Lake metadata.
+        """
+        children = []
+        for field in fields:
+            try:
+                child_name = field.get("name", "")
+                child_type = field.get("type", "string")
+
+                # Get the type string from dict if needed
+                type_str = child_type
+                if isinstance(child_type, dict):
+                    type_str = child_type.get("type", "string")
+
+                # Use DataType enum directly
+                try:
+                    data_type = (
+                        DataType(type_str.upper())
+                        if isinstance(type_str, str)
+                        else DataType.STRING
+                    )
+                except (ValueError, AttributeError):
+                    data_type = DataType.STRING
+
+                child = {
+                    "name": truncate_column_name(child_name),
+                    "displayName": child_name,
+                    "dataType": data_type.value,
+                    "dataTypeDisplay": child_type
+                    if isinstance(child_type, str)
+                    else str(child_type),
+                }
+
+                # Recursively handle nested structs
+                if isinstance(child_type, dict) and child_type.get("type") == "struct":
+                    child["children"] = self._parse_struct_fields(
+                        child_type.get("fields", [])
+                    )
+
+                children.append(child)
+            except Exception as exc:
+                logger.warning(f"Unable to parse nested field {field}: {exc}")
+                logger.debug(traceback.format_exc())
+
+        return children

metadata/utils/db_utils.py

@@ -12,9 +12,9 @@
 """
 Helpers module for db sources
 """
-
+import time
 import traceback
-from typing import Iterable
+from typing import Iterable, List, Union
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.table import Table
@@ -32,6 +32,7 @@ from metadata.ingestion.lineage.sql_lineage import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.models import TableView
 from metadata.utils import fqn
+from metadata.utils.execution_time_tracker import calculate_execution_time_generator
 from metadata.utils.logger import utils_logger
 
 logger = utils_logger()
@@ -47,16 +48,21 @@ def get_host_from_host_port(uri: str) -> str:
     return uri.split(":")[0]
 
 
+#  pylint: disable=too-many-locals
+@calculate_execution_time_generator()
 def get_view_lineage(
     view: TableView,
     metadata: OpenMetadata,
-    service_name: str,
+    service_names: Union[str, List[str]],
     connection_type: str,
     timeout_seconds: int = LINEAGE_PARSING_TIMEOUT,
 ) -> Iterable[Either[AddLineageRequest]]:
     """
     Method to generate view lineage
+    Now supports cross-database lineage by accepting a list of service names.
     """
+    if isinstance(service_names, str):
+        service_names = [service_names]
     table_name = view.table_name
     schema_name = view.schema_name
     db_name = view.db_name
@@ -65,7 +71,7 @@ def get_view_lineage(
     table_fqn = fqn.build(
         metadata,
         entity_type=Table,
-        service_name=service_name,
+        service_name=service_names[0],  # Use first service for table entity lookup
         database_name=db_name,
         schema_name=schema_name,
         table_name=table_name,
@@ -82,6 +88,8 @@ def get_view_lineage(
     try:
         connection_type = str(connection_type)
         dialect = ConnectionTypeDialectMapper.dialect_of(connection_type)
+        start_time = time.time()
+        logger.debug(f"Processing view lineage for: {table_fqn}")
         lineage_parser = LineageParser(
             view_definition, dialect, timeout_seconds=timeout_seconds
         )
@@ -91,16 +99,21 @@ def get_view_lineage(
             schema_name = PUBLIC_SCHEMA
             schema_fallback = True
 
+        end_time = time.time()
+        logger.debug(
+            f"Time taken to parse view lineage for: {table_fqn} is {end_time - start_time} seconds"
+        )
         if lineage_parser.source_tables and lineage_parser.target_tables:
             yield from get_lineage_by_query(
                 metadata,
                 query=view_definition,
-                service_name=service_name,
+                service_names=service_names,
                 database_name=db_name,
                 schema_name=schema_name,
                 dialect=dialect,
                 timeout_seconds=timeout_seconds,
                 lineage_source=LineageSource.ViewLineage,
+                lineage_parser=lineage_parser,
                 schema_fallback=schema_fallback,
             ) or []
 
@@ -108,13 +121,14 @@ def get_view_lineage(
             yield from get_lineage_via_table_entity(
                 metadata,
                 table_entity=table_entity,
-                service_name=service_name,
+                service_names=service_names,
                 database_name=db_name,
                 schema_name=schema_name,
                 query=view_definition,
                 dialect=dialect,
                 timeout_seconds=timeout_seconds,
                 lineage_source=LineageSource.ViewLineage,
+                lineage_parser=lineage_parser,
                 schema_fallback=schema_fallback,
             ) or []
     except Exception as exc:

metadata/utils/deprecation.py

@@ -11,9 +11,11 @@
 """
 Announce method deprecation
 """
-import warnings
+import logging
 from functools import wraps
 
+from metadata.utils.logger import METADATA_LOGGER
+
 
 def deprecated(message: str, release: str):
     """Decorator factory to accept specific messages for each function"""
@@ -21,12 +23,13 @@ def deprecated(message: str, release: str):
     def _deprecated(fn):
         @wraps(fn)
         def inner(*args, **kwargs):
-            warnings.simplefilter("always", DeprecationWarning)
-            warnings.warn(
-                f"[{fn.__name__}] will be deprecated in the release [{release}]: {message}",
-                category=DeprecationWarning,
+            # Get the metadata logger
+            logger = logging.getLogger(METADATA_LOGGER)
+            # Log deprecation warning using the logging system
+            # This will respect the loggerLevel configuration
+            logger.warning(
+                f"[{fn.__name__}] will be deprecated in the release [{release}]: {message}"
             )
-            warnings.simplefilter("default", DeprecationWarning)
 
             return fn(*args, **kwargs)