openmetadata-ingestion 1.8.9.0__py3-none-any.whl → 1.8.11.0__py3-none-any.whl

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-08-01T13:07:02+00:00
+#   timestamp: 2025-10-21T16:05:17+00:00
 
 from __future__ import annotations
 

metadata/ingestion/ometa/mixins/domain_mixin.py

@@ -9,11 +9,15 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """Domain and Data Product specific operations"""
+import traceback
 from typing import Dict, List
 
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.models.custom_pydantic import BaseModel
 from metadata.ingestion.ometa.client import REST
+from metadata.utils.logger import ometa_logger
+
+logger = ometa_logger()
 
 
 class AssetsRequest(BaseModel):
@@ -57,6 +61,50 @@ class OMetaDomainMixin:
         """
         return self._handle_data_product_assets(name, assets, "remove")
 
+    def get_data_product_assets(
+        self, name: str, limit: int = 10, offset: int = 0
+    ) -> Dict:
+        """
+        Get paginated list of assets for a data product
+
+        Args:
+            name: Name of the data product
+            limit: Maximum number of assets to return (default 10, max 1000)
+            offset: Offset from which to start returning results (default 0)
+
+        Returns:
+            API response as a dictionary containing paginated assets
+        """
+        try:
+            path = f"/dataProducts/name/{name}/assets"
+            params = {"limit": limit, "offset": offset}
+            return self.client.get(path, params)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not get data product assets due to {exc}")
+            return {}
+
+    def get_domain_assets(self, name: str, limit: int = 10, offset: int = 0) -> Dict:
+        """
+        Get paginated list of assets for a domain
+
+        Args:
+            name: Name of the domain
+            limit: Maximum number of assets to return (default 10, max 1000)
+            offset: Offset from which to start returning results (default 0)
+
+        Returns:
+            API response as a dictionary containing paginated assets
+        """
+        try:
+            path = f"/domains/name/{name}/assets"
+            params = {"limit": limit, "offset": offset}
+            return self.client.get(path, params)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not get domain assets due to {exc}")
+            return {}
+
     def _handle_data_product_assets(
         self,
         name: str,

metadata/ingestion/ometa/mixins/patch_mixin.py

@@ -558,7 +558,16 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
         """Patch domain data for an Entity"""
         try:
             destination: Entity = entity.model_copy(deep=True)
-            destination.domain = EntityReference(id=domain.id, type="domain")
+            destination.domain = EntityReference(
+                id=domain.id,
+                type="domain",
+                name=domain.name.root,
+                fullyQualifiedName=domain.fullyQualifiedName.root,
+                displayName=domain.displayName
+                if domain.displayName
+                else domain.name.root,
+                description=domain.description.root if domain.description else None,
+            )
             return self.patch(
                 entity=type(entity), source=entity, destination=destination
             )

metadata/ingestion/ometa/mixins/tag_glossary_mixin.py

@@ -0,0 +1,68 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Tag and Glossary Term specific operations"""
+import traceback
+from typing import Dict
+
+from metadata.ingestion.ometa.client import REST
+from metadata.utils.logger import ometa_logger
+
+logger = ometa_logger()
+
+
+class OMetaTagGlossaryMixin:
+    """Mixin class containing Tag and Glossary Term specific methods"""
+
+    client: REST
+
+    def get_tag_assets(self, fqn: str, limit: int = 10, offset: int = 0) -> Dict:
+        """
+        Get paginated list of assets for a tag
+
+        Args:
+            fqn: Fully qualified name of the tag
+            limit: Maximum number of assets to return (default 10, max 1000)
+            offset: Offset from which to start returning results (default 0)
+
+        Returns:
+            API response as a dictionary containing paginated assets
+        """
+        try:
+            path = f"/tags/name/{fqn}/assets"
+            params = {"limit": limit, "offset": offset}
+            return self.client.get(path, params)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not get tag assets due to {exc}")
+            return {}
+
+    def get_glossary_term_assets(
+        self, fqn: str, limit: int = 10, offset: int = 0
+    ) -> Dict:
+        """
+        Get paginated list of assets for a glossary term
+
+        Args:
+            fqn: Fully qualified name of the glossary term
+            limit: Maximum number of assets to return (default 10, max 1000)
+            offset: Offset from which to start returning results (default 0)
+
+        Returns:
+            API response as a dictionary containing paginated assets
+        """
+        try:
+            path = f"/glossaryTerms/name/{fqn}/assets"
+            params = {"limit": limit, "offset": offset}
+            return self.client.get(path, params)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not get glossary term assets due to {exc}")
+            return {}

metadata/ingestion/ometa/mixins/user_mixin.py

@@ -14,6 +14,7 @@ Mixin class containing User specific methods
 To be used by OpenMetadata class
 """
 import json
+import traceback
 from functools import lru_cache
 from typing import Optional, Type
 
@@ -215,3 +216,45 @@ class OMetaUserMixin:
             )
 
         return None
+
+    def get_user_assets(self, name: str, limit: int = 10, offset: int = 0) -> dict:
+        """
+        Get paginated list of assets for a user
+
+        Args:
+            name: Name of the user
+            limit: Maximum number of assets to return (default 10, max 1000)
+            offset: Offset from which to start returning results (default 0)
+
+        Returns:
+            API response as a dictionary containing paginated assets
+        """
+        try:
+            path = f"/users/name/{name}/assets"
+            params = {"limit": limit, "offset": offset}
+            return self.client.get(path, params)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not get user assets due to {exc}")
+            return {}
+
+    def get_team_assets(self, name: str, limit: int = 10, offset: int = 0) -> dict:
+        """
+        Get paginated list of assets for a team
+
+        Args:
+            name: Name of the team
+            limit: Maximum number of assets to return (default 10, max 1000)
+            offset: Offset from which to start returning results (default 0)
+
+        Returns:
+            API response as a dictionary containing paginated assets
+        """
+        try:
+            path = f"/teams/name/{name}/assets"
+            params = {"limit": limit, "offset": offset}
+            return self.client.get(path, params)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not get team assets due to {exc}")
+            return {}

metadata/ingestion/ometa/ometa_api.py

@@ -53,6 +53,7 @@ from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin
 from metadata.ingestion.ometa.mixins.service_mixin import OMetaServiceMixin
 from metadata.ingestion.ometa.mixins.suggestions_mixin import OMetaSuggestionsMixin
 from metadata.ingestion.ometa.mixins.table_mixin import OMetaTableMixin
+from metadata.ingestion.ometa.mixins.tag_glossary_mixin import OMetaTagGlossaryMixin
 from metadata.ingestion.ometa.mixins.tests_mixin import OMetaTestsMixin
 from metadata.ingestion.ometa.mixins.topic_mixin import OMetaTopicMixin
 from metadata.ingestion.ometa.mixins.user_mixin import OMetaUserMixin
@@ -127,6 +128,7 @@ class OpenMetadata(
     OMetaCustomPropertyMixin,
     OMetaSuggestionsMixin,
     OMetaDomainMixin,
+    OMetaTagGlossaryMixin,
     Generic[T, C],
 ):
     """

metadata/ingestion/source/database/bigquery/lineage.py

@@ -42,6 +42,7 @@ class BigqueryLineageSource(
         AND (
             statement_type IN ("MERGE", "CREATE_TABLE_AS_SELECT", "UPDATE") 
             OR (statement_type = "INSERT" and UPPER(query) like '%%INSERT%%INTO%%SELECT%%')
+            OR (statement_type = "CREATE_TABLE" and UPPER(query) like '%%CLONE%%')
         )
     """
 

metadata/readers/dataframe/mf4.py

@@ -0,0 +1,75 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+MF4 DataFrame reader for processing MF4 (Measurement Data Format) files
+"""
+
+import io
+
+import pandas as pd
+
+from metadata.readers.dataframe.base import DataFrameReader
+from metadata.readers.dataframe.common import dataframe_to_chunks
+from metadata.readers.dataframe.models import DatalakeColumnWrapper
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class MF4DataFrameReader(DataFrameReader):
+    """
+    MF4 file reader implementation for extracting schema and data
+    from MF4 (Measurement Data Format) files.
+    """
+
+    @staticmethod
+    def extract_schema_from_header(mf4_bytes: bytes) -> DatalakeColumnWrapper:
+        """
+        Extract schema from MF4 header common properties.
+        This method uses the MDF header metadata instead of actual data extraction.
+        """
+        from asammdf import MDF
+
+        file_obj = io.BytesIO(mf4_bytes)
+
+        mdf = MDF(file_obj)
+
+        if hasattr(mdf, "header") and hasattr(mdf.header, "_common_properties"):
+            common_props = mdf.header._common_properties
+
+            schema_dict = {}
+
+            for key, value in common_props.items():
+                schema_dict[key] = pd.Series(value)
+
+            if schema_dict:
+                schema_df = pd.DataFrame(schema_dict, index=[0])
+                logger.info(f"Extracted {len(schema_dict)} properties from MF4 header")
+                return DatalakeColumnWrapper(
+                    dataframes=dataframe_to_chunks(schema_df), raw_data=common_props
+                )
+
+        logger.debug("No _common_properties found in header.")
+
+    @staticmethod
+    def read_from_mf4(mf4_bytes: bytes) -> DatalakeColumnWrapper:
+        """
+        Convert MF4 file content to DatalakeColumnWrapper using header schema extraction.
+        """
+        return MF4DataFrameReader.extract_schema_from_header(mf4_bytes)
+
+    def _read(self, *, key: str, bucket_name: str, **__) -> DatalakeColumnWrapper:
+        """
+        Read MF4 file and extract schema information.
+        """
+        file_content = self.reader.read(key, bucket_name=bucket_name)
+        return self.read_from_mf4(file_content)

metadata/readers/dataframe/reader_factory.py

@@ -27,6 +27,7 @@ from metadata.readers.dataframe.dsv import (
     get_dsv_reader_by_separator,
 )
 from metadata.readers.dataframe.json import JSONDataFrameReader
+from metadata.readers.dataframe.mf4 import MF4DataFrameReader
 from metadata.readers.dataframe.parquet import ParquetDataFrameReader
 from metadata.readers.models import ConfigSource
 from metadata.utils.logger import utils_logger
@@ -50,6 +51,7 @@ class SupportedTypes(Enum):
     JSONL = "jsonl"
     JSONLGZ = "jsonl.gz"
     JSONLZIP = "jsonl.zip"
+    MF4 = "MF4"
 
 
 DF_READER_MAP = {
@@ -68,6 +70,7 @@ DF_READER_MAP = {
     SupportedTypes.JSONL.value: JSONDataFrameReader,
     SupportedTypes.JSONLGZ.value: JSONDataFrameReader,
     SupportedTypes.JSONLZIP.value: JSONDataFrameReader,
+    SupportedTypes.MF4.value: MF4DataFrameReader,
 }
 
 

metadata/utils/datalake/datalake_utils.py

@@ -542,8 +542,129 @@ class JsonDataFrameColumnParser(GenericDataFrameColumnParser):
         """
         if self.raw_data:
             try:
+                # First, check if this is an Iceberg/Delta Lake metadata file
+                data = json.loads(self.raw_data)
+                if self._is_iceberg_delta_metadata(data):
+                    return self._parse_iceberg_delta_schema(data)
+                # Otherwise, try to parse as standard JSON Schema
                 return parse_json_schema(schema_text=self.raw_data, cls=Column)
             except Exception as exc:
                 logger.warning(f"Unable to parse the json schema: {exc}")
                 logger.debug(traceback.format_exc())
         return self._get_columns(self.data_frame)
+
+    def _is_iceberg_delta_metadata(self, data: dict) -> bool:
+        """
+        Check if the JSON data is an Iceberg or Delta Lake metadata file.
+        These files have a specific structure with 'schema' containing 'fields'.
+        """
+        return (
+            isinstance(data, dict)
+            and "schema" in data
+            and isinstance(data["schema"], dict)
+            and "fields" in data["schema"]
+            and isinstance(data["schema"]["fields"], list)
+        )
+
+    def _parse_iceberg_delta_schema(self, data: dict) -> List[Column]:
+        """
+        Parse Iceberg/Delta Lake metadata file schema to extract columns.
+        These files have structure: {"schema": {"fields": [{"id": ..., "name": ..., "type": ..., "required": ...}, ...]}}
+        """
+        columns = []
+        schema = data.get("schema", {})
+        fields = schema.get("fields", [])
+
+        for field in fields:
+            try:
+                column_name = field.get("name", "")
+                column_type = field.get("type", "string")
+
+                # Get the type string from dict if needed
+                type_str = column_type
+                if isinstance(column_type, dict):
+                    type_str = column_type.get("type", "string")
+
+                # Use DataType enum directly - it will handle the conversion
+                try:
+                    data_type = (
+                        DataType(type_str.upper())
+                        if isinstance(type_str, str)
+                        else DataType.STRING
+                    )
+                except (ValueError, AttributeError):
+                    # If the type is not recognized, default to STRING
+                    data_type = DataType.STRING
+
+                column = Column(
+                    name=truncate_column_name(column_name),
+                    displayName=column_name,
+                    dataType=data_type,
+                    dataTypeDisplay=column_type
+                    if isinstance(column_type, str)
+                    else str(column_type),
+                )
+
+                # Handle nested struct types
+                if (
+                    isinstance(column_type, dict)
+                    and column_type.get("type") == "struct"
+                ):
+                    column.children = self._parse_struct_fields(
+                        column_type.get("fields", [])
+                    )
+                    column.dataType = DataType.STRUCT
+
+                columns.append(column)
+            except Exception as exc:
+                logger.warning(f"Unable to parse field {field}: {exc}")
+                logger.debug(traceback.format_exc())
+
+        return columns
+
+    def _parse_struct_fields(self, fields: list) -> List[dict]:
+        """
+        Parse nested struct fields in Iceberg/Delta Lake metadata.
+        """
+        children = []
+        for field in fields:
+            try:
+                child_name = field.get("name", "")
+                child_type = field.get("type", "string")
+
+                # Get the type string from dict if needed
+                type_str = child_type
+                if isinstance(child_type, dict):
+                    type_str = child_type.get("type", "string")
+
+                # Use DataType enum directly
+                try:
+                    data_type = (
+                        DataType(type_str.upper())
+                        if isinstance(type_str, str)
+                        else DataType.STRING
+                    )
+                except (ValueError, AttributeError):
+                    data_type = DataType.STRING
+
+                child = {
+                    "name": truncate_column_name(child_name),
+                    "displayName": child_name,
+                    "dataType": data_type.value,
+                    "dataTypeDisplay": child_type
+                    if isinstance(child_type, str)
+                    else str(child_type),
+                }
+
+                # Recursively handle nested structs
+                if isinstance(child_type, dict) and child_type.get("type") == "struct":
+                    child["children"] = self._parse_struct_fields(
+                        child_type.get("fields", [])
+                    )
+
+                children.append(child)
+            except Exception as exc:
+                logger.warning(f"Unable to parse nested field {field}: {exc}")
+                logger.debug(traceback.format_exc())
+
+        return children