openmetadata-ingestion 1.9.11.2__py3-none-any.whl → 1.10.0.0__py3-none-any.whl

metadata/profiler/metrics/hybrid/cardinality_distribution.py

@@ -0,0 +1,232 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Cardinality Distribution Metric definition
+"""
+from typing import Any, Dict, Optional
+
+from sqlalchemy import case, column, func, or_
+from sqlalchemy.orm import DeclarativeMeta, Session
+
+from metadata.profiler.metrics.core import HybridMetric
+from metadata.profiler.metrics.static.count import Count
+from metadata.profiler.metrics.static.distinct_count import DistinctCount
+from metadata.profiler.orm.registry import is_concatenable, is_enum
+from metadata.utils.logger import profiler_logger
+
+logger = profiler_logger()
+
+
+class CardinalityDistribution(HybridMetric):
+    """
+    CARDINALITY_DISTRIBUTION Metric
+
+    Given a column, return the cardinality distribution showing top categories
+    with an "Others" bucket. Only works for concatenable types (strings, enums).
+    """
+
+    threshold_percentage: float = 0.02  # 2% threshold for "Others" bucket
+    min_buckets: int = 5  # Minimum number of top categories to show
+
+    @classmethod
+    def name(cls):
+        return "cardinalityDistribution"
+
+    @property
+    def metric_type(self):
+        return dict
+
+    def fn(
+        self,
+        sample: Optional[DeclarativeMeta],
+        res: Dict[str, Any],
+        session: Optional[Session] = None,
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Build the Cardinality Distribution metric query
+        """
+        if not session or self.col is None:
+            raise AttributeError(
+                f"We are missing the {'session' if not session else 'column'} attribute to compute the CardinalityDistribution."
+            )
+
+        if not (is_concatenable(self.col.type) or is_enum(self.col.type)):
+            logger.debug(
+                f"CardinalityDistribution not applicable for {self.col.name} because type {self.col.type} is not supported."
+            )
+            return None
+
+        # Get total count from other metrics if available, otherwise compute it
+        total_count = res.get(Count.name())
+        distinct_count = res.get(DistinctCount.name())
+
+        if not total_count or total_count == 0:
+            return None
+
+        if total_count == distinct_count:
+            logger.debug(
+                f"CardinalityDistribution not applicable for {self.col.name} because all values are distinct."
+            )
+            return None
+
+        col = column(self.col.name, self.col.type)
+        threshold = self.threshold_percentage * total_count
+
+        # Build a cross-database compatible query using CTEs
+        # Step 1: Get value counts
+        value_counts_cte = (
+            session.query(  # type: ignore
+                col.label("category"), func.count(col).label("category_count")
+            )
+            .select_from(sample)
+            .group_by(col)
+            .cte("value_counts")
+        )
+
+        # Step 2: Get top categories
+        top_categories_cte = (
+            session.query(value_counts_cte.c.category)  # type: ignore
+            .select_from(value_counts_cte)
+            .order_by(value_counts_cte.c.category_count.desc())
+            .limit(self.min_buckets)
+            .cte("top_categories")
+        )
+
+        # Step 3: Create categorization CTE to avoid complex GROUP BY
+        categorized_cte = (
+            session.query(  # type: ignore
+                case(
+                    (
+                        or_(
+                            value_counts_cte.c.category.in_(
+                                session.query(top_categories_cte.c.category)  # type: ignore
+                            ),
+                            value_counts_cte.c.category_count >= threshold,
+                        ),
+                        value_counts_cte.c.category,
+                    ),
+                    else_="Others",
+                ).label("category_group"),
+                value_counts_cte.c.category_count,
+            )
+            .select_from(value_counts_cte)
+            .cte("categorized")
+        )
+
+        # Step 4: Final aggregation with simple GROUP BY
+        query = (
+            session.query(  # type: ignore
+                categorized_cte.c.category_group,
+                func.sum(categorized_cte.c.category_count).label("total_count"),
+            )
+            .select_from(categorized_cte)
+            .group_by(categorized_cte.c.category_group)
+            .order_by(func.sum(categorized_cte.c.category_count).desc())
+        )
+
+        rows = query.all()
+
+        if rows:
+            return {
+                "categories": [row[0] for row in rows],
+                "counts": [int(row[1]) for row in rows],
+                "percentages": [round((row[1] / total_count) * 100, 2) for row in rows],
+            }
+        return None
+
+    def df_fn(self, res: Dict[str, Any], dfs=None):
+        """
+        Pandas implementation for dataframes
+        """
+        # pylint: disable=import-outside-toplevel
+        import pandas as pd
+
+        if self.col is None:
+            return None
+
+        if not (is_concatenable(self.col.type) or is_enum(self.col.type)):
+            logger.debug(
+                f"CardinalityDistribution not applicable for {self.col.name} because type {self.col.type} is not supported."
+            )
+            return None
+
+        # Get total count and distinct count from other metrics
+        total_count = res.get(Count.name())
+        distinct_count = res.get(DistinctCount.name())
+
+        if not total_count or total_count == 0:
+            return None
+
+        if total_count == distinct_count:
+            logger.debug(
+                f"CardinalityDistribution not applicable for {self.col.name} because all values are distinct."
+            )
+            return None
+
+        try:
+            if dfs is None:
+                return None
+
+            # Calculate threshold
+            threshold = self.threshold_percentage * total_count
+
+            # Use pandas value_counts() for efficient processing - much faster for large datasets
+            combined_value_counts = pd.Series(dtype="object")
+
+            for df in dfs:
+                # Use value_counts() directly on the column - much more memory efficient
+                df_value_counts = df[self.col.name].value_counts()
+                combined_value_counts = combined_value_counts.add(
+                    df_value_counts, fill_value=0
+                )
+
+            # Get top categories that meet the threshold OR are in the top N
+            top_categories = {}
+            others_count = 0
+
+            # Get the top N categories by count
+            top_n_categories = set(combined_value_counts.head(self.min_buckets).index)
+
+            # Process in descending order of frequency
+            for category, count in combined_value_counts.items():
+                # First check if it's in top N, then check threshold
+                if category in top_n_categories or count >= threshold:
+                    top_categories[category] = count
+                else:
+                    others_count += count
+
+            # Build result
+            categories = []
+            counts = []
+            percentages = []
+
+            for category, count in top_categories.items():
+                categories.append(str(category))
+                counts.append(int(count))
+                percentages.append(round((count / total_count) * 100, 2))
+
+            if others_count > 0:
+                categories.append("Others")
+                counts.append(int(others_count))
+                percentages.append(round((others_count / total_count) * 100, 2))
+
+            return {
+                "categories": categories,
+                "counts": counts,
+                "percentages": percentages,
+            }
+
+        except Exception as err:
+            logger.debug(
+                f"Error computing CardinalityDistribution for {self.col.name}: {err}"
+            )
+            return None

metadata/profiler/metrics/hybrid/histogram.py

@@ -151,7 +151,7 @@ class Histogram(HybridMetric):
                 "We are missing the session attribute to compute the Histogram."
             )
 
-        if not (is_quantifiable(self.col.type) or is_concatenable(self.col.type)) or (
+        if not is_quantifiable(self.col.type) or (
             is_value_non_numeric(res.get(Min.name()))
             or is_value_non_numeric(res.get(Max.name()))
         ):

metadata/profiler/metrics/registry.py

@@ -26,6 +26,9 @@ from metadata.profiler.metrics.composed.like_ratio import LikeRatio
 from metadata.profiler.metrics.composed.non_parametric_skew import NonParametricSkew
 from metadata.profiler.metrics.composed.null_ratio import NullRatio
 from metadata.profiler.metrics.composed.unique_ratio import UniqueRatio
+from metadata.profiler.metrics.hybrid.cardinality_distribution import (
+    CardinalityDistribution,
+)
 from metadata.profiler.metrics.hybrid.histogram import Histogram
 from metadata.profiler.metrics.static.column_count import ColumnCount
 from metadata.profiler.metrics.static.column_names import ColumnNames
@@ -104,6 +107,7 @@ class Metrics(MetricRegistry):
 
     # Hybrid Metrics
     HISTOGRAM = Histogram
+    CARDINALITY_DISTRIBUTION = CardinalityDistribution
 
     # Missing Count
     NULL_MISSING_COUNT = NullMissingCount

metadata/profiler/orm/converter/converter_registry.py

@@ -24,6 +24,7 @@ from metadata.profiler.orm.converter.mariadb.converter import MariaDBMapTypes
 from metadata.profiler.orm.converter.mssql.converter import MssqlMapTypes
 from metadata.profiler.orm.converter.redshift.converter import RedshiftMapTypes
 from metadata.profiler.orm.converter.snowflake.converter import SnowflakeMapTypes
+from metadata.profiler.orm.converter.trino import TrinoMapTypes
 
 converter_registry = defaultdict(lambda: CommonMapTypes)
 converter_registry[DatabaseServiceType.BigQuery] = BigqueryMapTypes
@@ -32,3 +33,4 @@ converter_registry[DatabaseServiceType.Redshift] = RedshiftMapTypes
 converter_registry[DatabaseServiceType.Mssql] = MssqlMapTypes
 converter_registry[DatabaseServiceType.AzureSQL] = AzureSqlMapTypes
 converter_registry[DatabaseServiceType.MariaDB] = MariaDBMapTypes
+converter_registry[DatabaseServiceType.Trino] = TrinoMapTypes

metadata/profiler/orm/converter/trino/__init__.py

@@ -0,0 +1,14 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from .converter import TrinoMapTypes
+
+__all__ = ("TrinoMapTypes",)

metadata/profiler/orm/converter/trino/converter.py

@@ -0,0 +1,39 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import Dict, Set
+
+from sqlalchemy.sql.type_api import TypeEngine
+
+from metadata.generated.schema.entity.data.table import DataType
+from metadata.profiler.orm.converter.common import CommonMapTypes
+from metadata.profiler.orm.types.trino import TrinoArray, TrinoMap, TrinoStruct
+
+
+class TrinoMapTypes(CommonMapTypes):
+    _TYPE_MAP_OVERRIDE = {
+        DataType.ARRAY: TrinoArray,
+        DataType.MAP: TrinoMap,
+        DataType.STRUCT: TrinoStruct,
+    }
+    _TYPE_MAP = {
+        **CommonMapTypes._TYPE_MAP,
+        **_TYPE_MAP_OVERRIDE,
+    }
+
+    @classmethod
+    def map_sqa_to_om_types(cls) -> Dict[TypeEngine, Set[DataType]]:
+        """returns an ORM type"""
+        # pylint: disable=import-outside-toplevel
+
+        return {
+            **CommonMapTypes.map_sqa_to_om_types(),
+            **{v: {k} for k, v in cls._TYPE_MAP_OVERRIDE.items()},
+        }

metadata/profiler/orm/functions/conn_test.py

@@ -46,6 +46,11 @@ def _(*_, **__):
     return "SELECT SESSION_USER()"
 
 
+@compiles(ConnTestFn, Dialects.Databricks)
+def _(*_, **__):
+    return "SELECT '42'"
+
+
 @compiles(ConnTestFn, Dialects.Db2)
 @compiles(ConnTestFn, Dialects.IbmDbSa)
 @compiles(ConnTestFn, Dialects.Ibmi)

metadata/profiler/orm/registry.py

@@ -215,3 +215,9 @@ def is_value_non_numeric(value) -> bool:
         return False
     except Exception:
         return False
+
+
+def is_enum(_type) -> bool:
+    if isinstance(_type, DataType):
+        return _type.value == DataType.ENUM.value
+    return issubclass(_type.__class__, Enum)

metadata/profiler/orm/types/__init__.py

@@ -0,0 +1,10 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

metadata/profiler/orm/types/trino.py

@@ -0,0 +1,70 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Type adapter for Trino to handle NamedRowTuple serialization
+"""
+from typing import Any
+
+from sqlalchemy import ARRAY
+from sqlalchemy.engine import Dialect
+from sqlalchemy.sql.sqltypes import String, TypeDecorator
+
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class TrinoTypesMixin:
+    def process_result_value(self, value: Any, dialect: Dialect) -> Any:
+        # pylint: disable=import-outside-toplevel
+        from trino.types import NamedRowTuple
+
+        def _convert_value(obj: Any) -> Any:
+            if isinstance(obj, NamedRowTuple):
+                return {
+                    k: _convert_value(getattr(obj, k))
+                    for k in obj.__annotations__["names"]
+                }
+            elif isinstance(obj, (list, tuple)):
+                return type(obj)(_convert_value(v) for v in obj)
+            elif isinstance(obj, dict):
+                return {k: _convert_value(v) for k, v in obj.items()}
+            return obj
+
+        return _convert_value(value)
+
+
+class TrinoArray(TrinoTypesMixin, TypeDecorator):
+    impl = ARRAY
+    cache_ok = True
+
+    @property
+    def python_type(self):
+        return list
+
+
+class TrinoMap(TrinoTypesMixin, TypeDecorator):
+    impl = String
+    cache_ok = True
+
+    @property
+    def python_type(self):
+        return dict
+
+
+class TrinoStruct(TrinoTypesMixin, TypeDecorator):
+    impl = String
+    cache_ok = True
+
+    @property
+    def python_type(self):
+        return dict

metadata/profiler/processor/default.py

@@ -59,6 +59,7 @@ def get_default_metrics(
         metrics_registry.UNIQUE_RATIO.value,
         metrics_registry.IQR.value,
         metrics_registry.HISTOGRAM.value,
+        metrics_registry.CARDINALITY_DISTRIBUTION.value,
         metrics_registry.NON_PARAMETRIC_SKEW.value,
     ]
 

metadata/readers/dataframe/mf4.py

@@ -0,0 +1,75 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+MF4 DataFrame reader for processing MF4 (Measurement Data Format) files
+"""
+
+import io
+
+import pandas as pd
+
+from metadata.readers.dataframe.base import DataFrameReader
+from metadata.readers.dataframe.common import dataframe_to_chunks
+from metadata.readers.dataframe.models import DatalakeColumnWrapper
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class MF4DataFrameReader(DataFrameReader):
+    """
+    MF4 file reader implementation for extracting schema and data
+    from MF4 (Measurement Data Format) files.
+    """
+
+    @staticmethod
+    def extract_schema_from_header(mf4_bytes: bytes) -> DatalakeColumnWrapper:
+        """
+        Extract schema from MF4 header common properties.
+        This method uses the MDF header metadata instead of actual data extraction.
+        """
+        from asammdf import MDF
+
+        file_obj = io.BytesIO(mf4_bytes)
+
+        mdf = MDF(file_obj)
+
+        if hasattr(mdf, "header") and hasattr(mdf.header, "_common_properties"):
+            common_props = mdf.header._common_properties
+
+            schema_dict = {}
+
+            for key, value in common_props.items():
+                schema_dict[key] = pd.Series(value)
+
+            if schema_dict:
+                schema_df = pd.DataFrame(schema_dict, index=[0])
+                logger.info(f"Extracted {len(schema_dict)} properties from MF4 header")
+                return DatalakeColumnWrapper(
+                    dataframes=dataframe_to_chunks(schema_df), raw_data=common_props
+                )
+
+        logger.debug("No _common_properties found in header.")
+
+    @staticmethod
+    def read_from_mf4(mf4_bytes: bytes) -> DatalakeColumnWrapper:
+        """
+        Convert MF4 file content to DatalakeColumnWrapper using header schema extraction.
+        """
+        return MF4DataFrameReader.extract_schema_from_header(mf4_bytes)
+
+    def _read(self, *, key: str, bucket_name: str, **__) -> DatalakeColumnWrapper:
+        """
+        Read MF4 file and extract schema information.
+        """
+        file_content = self.reader.read(key, bucket_name=bucket_name)
+        return self.read_from_mf4(file_content)

metadata/readers/dataframe/reader_factory.py

@@ -27,6 +27,7 @@ from metadata.readers.dataframe.dsv import (
     get_dsv_reader_by_separator,
 )
 from metadata.readers.dataframe.json import JSONDataFrameReader
+from metadata.readers.dataframe.mf4 import MF4DataFrameReader
 from metadata.readers.dataframe.parquet import ParquetDataFrameReader
 from metadata.readers.models import ConfigSource
 from metadata.utils.logger import utils_logger
@@ -50,6 +51,7 @@ class SupportedTypes(Enum):
     JSONL = "jsonl"
     JSONLGZ = "jsonl.gz"
     JSONLZIP = "jsonl.zip"
+    MF4 = "MF4"
 
 
 DF_READER_MAP = {
@@ -68,6 +70,7 @@ DF_READER_MAP = {
     SupportedTypes.JSONL.value: JSONDataFrameReader,
     SupportedTypes.JSONLGZ.value: JSONDataFrameReader,
     SupportedTypes.JSONLZIP.value: JSONDataFrameReader,
+    SupportedTypes.MF4.value: MF4DataFrameReader,
 }