openmetadata-ingestion 1.9.12.0__py3-none-any.whl → 1.10.0.0__py3-none-any.whl

metadata/profiler/metrics/hybrid/cardinality_distribution.py

@@ -0,0 +1,232 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Cardinality Distribution Metric definition
+"""
+from typing import Any, Dict, Optional
+
+from sqlalchemy import case, column, func, or_
+from sqlalchemy.orm import DeclarativeMeta, Session
+
+from metadata.profiler.metrics.core import HybridMetric
+from metadata.profiler.metrics.static.count import Count
+from metadata.profiler.metrics.static.distinct_count import DistinctCount
+from metadata.profiler.orm.registry import is_concatenable, is_enum
+from metadata.utils.logger import profiler_logger
+
+logger = profiler_logger()
+
+
+class CardinalityDistribution(HybridMetric):
+    """
+    CARDINALITY_DISTRIBUTION Metric
+
+    Given a column, return the cardinality distribution showing top categories
+    with an "Others" bucket. Only works for concatenable types (strings, enums).
+    """
+
+    threshold_percentage: float = 0.02  # 2% threshold for "Others" bucket
+    min_buckets: int = 5  # Minimum number of top categories to show
+
+    @classmethod
+    def name(cls):
+        return "cardinalityDistribution"
+
+    @property
+    def metric_type(self):
+        return dict
+
+    def fn(
+        self,
+        sample: Optional[DeclarativeMeta],
+        res: Dict[str, Any],
+        session: Optional[Session] = None,
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Build the Cardinality Distribution metric query
+        """
+        if not session or self.col is None:
+            raise AttributeError(
+                f"We are missing the {'session' if not session else 'column'} attribute to compute the CardinalityDistribution."
+            )
+
+        if not (is_concatenable(self.col.type) or is_enum(self.col.type)):
+            logger.debug(
+                f"CardinalityDistribution not applicable for {self.col.name} because type {self.col.type} is not supported."
+            )
+            return None
+
+        # Get total count from other metrics if available, otherwise compute it
+        total_count = res.get(Count.name())
+        distinct_count = res.get(DistinctCount.name())
+
+        if not total_count or total_count == 0:
+            return None
+
+        if total_count == distinct_count:
+            logger.debug(
+                f"CardinalityDistribution not applicable for {self.col.name} because all values are distinct."
+            )
+            return None
+
+        col = column(self.col.name, self.col.type)
+        threshold = self.threshold_percentage * total_count
+
+        # Build a cross-database compatible query using CTEs
+        # Step 1: Get value counts
+        value_counts_cte = (
+            session.query(  # type: ignore
+                col.label("category"), func.count(col).label("category_count")
+            )
+            .select_from(sample)
+            .group_by(col)
+            .cte("value_counts")
+        )
+
+        # Step 2: Get top categories
+        top_categories_cte = (
+            session.query(value_counts_cte.c.category)  # type: ignore
+            .select_from(value_counts_cte)
+            .order_by(value_counts_cte.c.category_count.desc())
+            .limit(self.min_buckets)
+            .cte("top_categories")
+        )
+
+        # Step 3: Create categorization CTE to avoid complex GROUP BY
+        categorized_cte = (
+            session.query(  # type: ignore
+                case(
+                    (
+                        or_(
+                            value_counts_cte.c.category.in_(
+                                session.query(top_categories_cte.c.category)  # type: ignore
+                            ),
+                            value_counts_cte.c.category_count >= threshold,
+                        ),
+                        value_counts_cte.c.category,
+                    ),
+                    else_="Others",
+                ).label("category_group"),
+                value_counts_cte.c.category_count,
+            )
+            .select_from(value_counts_cte)
+            .cte("categorized")
+        )
+
+        # Step 4: Final aggregation with simple GROUP BY
+        query = (
+            session.query(  # type: ignore
+                categorized_cte.c.category_group,
+                func.sum(categorized_cte.c.category_count).label("total_count"),
+            )
+            .select_from(categorized_cte)
+            .group_by(categorized_cte.c.category_group)
+            .order_by(func.sum(categorized_cte.c.category_count).desc())
+        )
+
+        rows = query.all()
+
+        if rows:
+            return {
+                "categories": [row[0] for row in rows],
+                "counts": [int(row[1]) for row in rows],
+                "percentages": [round((row[1] / total_count) * 100, 2) for row in rows],
+            }
+        return None
+
+    def df_fn(self, res: Dict[str, Any], dfs=None):
+        """
+        Pandas implementation for dataframes
+        """
+        # pylint: disable=import-outside-toplevel
+        import pandas as pd
+
+        if self.col is None:
+            return None
+
+        if not (is_concatenable(self.col.type) or is_enum(self.col.type)):
+            logger.debug(
+                f"CardinalityDistribution not applicable for {self.col.name} because type {self.col.type} is not supported."
+            )
+            return None
+
+        # Get total count and distinct count from other metrics
+        total_count = res.get(Count.name())
+        distinct_count = res.get(DistinctCount.name())
+
+        if not total_count or total_count == 0:
+            return None
+
+        if total_count == distinct_count:
+            logger.debug(
+                f"CardinalityDistribution not applicable for {self.col.name} because all values are distinct."
+            )
+            return None
+
+        try:
+            if dfs is None:
+                return None
+
+            # Calculate threshold
+            threshold = self.threshold_percentage * total_count
+
+            # Use pandas value_counts() for efficient processing - much faster for large datasets
+            combined_value_counts = pd.Series(dtype="object")
+
+            for df in dfs:
+                # Use value_counts() directly on the column - much more memory efficient
+                df_value_counts = df[self.col.name].value_counts()
+                combined_value_counts = combined_value_counts.add(
+                    df_value_counts, fill_value=0
+                )
+
+            # Get top categories that meet the threshold OR are in the top N
+            top_categories = {}
+            others_count = 0
+
+            # Get the top N categories by count
+            top_n_categories = set(combined_value_counts.head(self.min_buckets).index)
+
+            # Process in descending order of frequency
+            for category, count in combined_value_counts.items():
+                # First check if it's in top N, then check threshold
+                if category in top_n_categories or count >= threshold:
+                    top_categories[category] = count
+                else:
+                    others_count += count
+
+            # Build result
+            categories = []
+            counts = []
+            percentages = []
+
+            for category, count in top_categories.items():
+                categories.append(str(category))
+                counts.append(int(count))
+                percentages.append(round((count / total_count) * 100, 2))
+
+            if others_count > 0:
+                categories.append("Others")
+                counts.append(int(others_count))
+                percentages.append(round((others_count / total_count) * 100, 2))
+
+            return {
+                "categories": categories,
+                "counts": counts,
+                "percentages": percentages,
+            }
+
+        except Exception as err:
+            logger.debug(
+                f"Error computing CardinalityDistribution for {self.col.name}: {err}"
+            )
+            return None

metadata/profiler/metrics/hybrid/histogram.py

@@ -151,7 +151,7 @@ class Histogram(HybridMetric):
                 "We are missing the session attribute to compute the Histogram."
             )
 
-        if not (is_quantifiable(self.col.type) or is_concatenable(self.col.type)) or (
+        if not is_quantifiable(self.col.type) or (
             is_value_non_numeric(res.get(Min.name()))
             or is_value_non_numeric(res.get(Max.name()))
         ):

metadata/profiler/metrics/registry.py

@@ -26,6 +26,9 @@ from metadata.profiler.metrics.composed.like_ratio import LikeRatio
 from metadata.profiler.metrics.composed.non_parametric_skew import NonParametricSkew
 from metadata.profiler.metrics.composed.null_ratio import NullRatio
 from metadata.profiler.metrics.composed.unique_ratio import UniqueRatio
+from metadata.profiler.metrics.hybrid.cardinality_distribution import (
+    CardinalityDistribution,
+)
 from metadata.profiler.metrics.hybrid.histogram import Histogram
 from metadata.profiler.metrics.static.column_count import ColumnCount
 from metadata.profiler.metrics.static.column_names import ColumnNames
@@ -104,6 +107,7 @@ class Metrics(MetricRegistry):
 
     # Hybrid Metrics
     HISTOGRAM = Histogram
+    CARDINALITY_DISTRIBUTION = CardinalityDistribution
 
     # Missing Count
     NULL_MISSING_COUNT = NullMissingCount

metadata/profiler/orm/converter/converter_registry.py

@@ -24,6 +24,7 @@ from metadata.profiler.orm.converter.mariadb.converter import MariaDBMapTypes
 from metadata.profiler.orm.converter.mssql.converter import MssqlMapTypes
 from metadata.profiler.orm.converter.redshift.converter import RedshiftMapTypes
 from metadata.profiler.orm.converter.snowflake.converter import SnowflakeMapTypes
+from metadata.profiler.orm.converter.trino import TrinoMapTypes
 
 converter_registry = defaultdict(lambda: CommonMapTypes)
 converter_registry[DatabaseServiceType.BigQuery] = BigqueryMapTypes
@@ -32,3 +33,4 @@ converter_registry[DatabaseServiceType.Redshift] = RedshiftMapTypes
 converter_registry[DatabaseServiceType.Mssql] = MssqlMapTypes
 converter_registry[DatabaseServiceType.AzureSQL] = AzureSqlMapTypes
 converter_registry[DatabaseServiceType.MariaDB] = MariaDBMapTypes
+converter_registry[DatabaseServiceType.Trino] = TrinoMapTypes

metadata/profiler/orm/converter/trino/__init__.py

@@ -0,0 +1,14 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from .converter import TrinoMapTypes
+
+__all__ = ("TrinoMapTypes",)

metadata/profiler/orm/converter/trino/converter.py

@@ -0,0 +1,39 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import Dict, Set
+
+from sqlalchemy.sql.type_api import TypeEngine
+
+from metadata.generated.schema.entity.data.table import DataType
+from metadata.profiler.orm.converter.common import CommonMapTypes
+from metadata.profiler.orm.types.trino import TrinoArray, TrinoMap, TrinoStruct
+
+
+class TrinoMapTypes(CommonMapTypes):
+    _TYPE_MAP_OVERRIDE = {
+        DataType.ARRAY: TrinoArray,
+        DataType.MAP: TrinoMap,
+        DataType.STRUCT: TrinoStruct,
+    }
+    _TYPE_MAP = {
+        **CommonMapTypes._TYPE_MAP,
+        **_TYPE_MAP_OVERRIDE,
+    }
+
+    @classmethod
+    def map_sqa_to_om_types(cls) -> Dict[TypeEngine, Set[DataType]]:
+        """returns an ORM type"""
+        # pylint: disable=import-outside-toplevel
+
+        return {
+            **CommonMapTypes.map_sqa_to_om_types(),
+            **{v: {k} for k, v in cls._TYPE_MAP_OVERRIDE.items()},
+        }

metadata/profiler/orm/functions/conn_test.py

@@ -46,6 +46,11 @@ def _(*_, **__):
     return "SELECT SESSION_USER()"
 
 
+@compiles(ConnTestFn, Dialects.Databricks)
+def _(*_, **__):
+    return "SELECT '42'"
+
+
 @compiles(ConnTestFn, Dialects.Db2)
 @compiles(ConnTestFn, Dialects.IbmDbSa)
 @compiles(ConnTestFn, Dialects.Ibmi)

metadata/profiler/orm/registry.py

@@ -215,3 +215,9 @@ def is_value_non_numeric(value) -> bool:
         return False
     except Exception:
         return False
+
+
+def is_enum(_type) -> bool:
+    if isinstance(_type, DataType):
+        return _type.value == DataType.ENUM.value
+    return issubclass(_type.__class__, Enum)

metadata/profiler/orm/types/__init__.py

@@ -0,0 +1,10 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

metadata/profiler/orm/types/trino.py

@@ -0,0 +1,70 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Type adapter for Trino to handle NamedRowTuple serialization
+"""
+from typing import Any
+
+from sqlalchemy import ARRAY
+from sqlalchemy.engine import Dialect
+from sqlalchemy.sql.sqltypes import String, TypeDecorator
+
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class TrinoTypesMixin:
+    def process_result_value(self, value: Any, dialect: Dialect) -> Any:
+        # pylint: disable=import-outside-toplevel
+        from trino.types import NamedRowTuple
+
+        def _convert_value(obj: Any) -> Any:
+            if isinstance(obj, NamedRowTuple):
+                return {
+                    k: _convert_value(getattr(obj, k))
+                    for k in obj.__annotations__["names"]
+                }
+            elif isinstance(obj, (list, tuple)):
+                return type(obj)(_convert_value(v) for v in obj)
+            elif isinstance(obj, dict):
+                return {k: _convert_value(v) for k, v in obj.items()}
+            return obj
+
+        return _convert_value(value)
+
+
+class TrinoArray(TrinoTypesMixin, TypeDecorator):
+    impl = ARRAY
+    cache_ok = True
+
+    @property
+    def python_type(self):
+        return list
+
+
+class TrinoMap(TrinoTypesMixin, TypeDecorator):
+    impl = String
+    cache_ok = True
+
+    @property
+    def python_type(self):
+        return dict
+
+
+class TrinoStruct(TrinoTypesMixin, TypeDecorator):
+    impl = String
+    cache_ok = True
+
+    @property
+    def python_type(self):
+        return dict

metadata/profiler/processor/default.py

@@ -59,6 +59,7 @@ def get_default_metrics(
         metrics_registry.UNIQUE_RATIO.value,
         metrics_registry.IQR.value,
         metrics_registry.HISTOGRAM.value,
+        metrics_registry.CARDINALITY_DISTRIBUTION.value,
         metrics_registry.NON_PARAMETRIC_SKEW.value,
     ]
 

metadata/sdk/__init__.py

@@ -0,0 +1,188 @@
+"""High-level entry points for the OpenMetadata Python SDK."""
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, Optional
+
+from metadata.sdk.client import OpenMetadata
+from metadata.sdk.config import OpenMetadataConfig
+from metadata.sdk.entities import (
+    APICollections,
+    APIEndpoints,
+    Charts,
+    Classifications,
+    Containers,
+    DashboardDataModels,
+    Dashboards,
+    DashboardServices,
+    Databases,
+    DatabaseSchemas,
+    DatabaseServices,
+    DataContracts,
+    DataProducts,
+    Domains,
+    Glossaries,
+    GlossaryTerms,
+    Metrics,
+    MLModels,
+    Pipelines,
+    Queries,
+    SearchIndexes,
+    StoredProcedures,
+    Tables,
+    Tags,
+    Teams,
+    TestCases,
+    TestDefinitions,
+    TestSuites,
+    Users,
+)
+from metadata.sdk.entities.base import BaseEntity
+
+_global_client: Optional[OpenMetadata] = None
+
+
+def configure(
+    config: OpenMetadataConfig | Mapping[str, Any] | None = None,
+    /,
+    **kwargs: Any,
+) -> OpenMetadata:
+    """Configure the SDK with a connection to OpenMetadata.
+
+    Mirrors the ergonomics of ``stripe.api_key = ...`` by establishing a global
+    client that the entity facades rely on. You can either pass an
+    :class:`OpenMetadataConfig` instance, a mapping, or keyword arguments.
+    """
+
+    global _global_client  # pylint: disable=global-statement
+
+    if config is not None and kwargs:
+        raise TypeError("Pass either a config object or keyword arguments, not both")
+
+    config_obj: OpenMetadataConfig
+    if config is None:
+        if not kwargs:
+            raise ValueError("Provide an OpenMetadataConfig or keyword arguments")
+        config_obj = OpenMetadataConfig(**kwargs)
+    elif isinstance(config, Mapping):
+        config_obj = OpenMetadataConfig(**dict(config))
+    else:
+        config_obj = config
+
+    _global_client = OpenMetadata.initialize(config_obj)
+    return _global_client
+
+
+def client() -> OpenMetadata:
+    """Return the active OpenMetadata client."""
+    if _global_client is None:
+        raise RuntimeError("SDK not configured. Call metadata.sdk.configure(...) first")
+    return _global_client
+
+
+def reset() -> None:
+    """Reset the SDK state, closing any cached client."""
+    global _global_client  # pylint: disable=global-statement
+    OpenMetadata.reset()
+    _global_client = None
+
+
+# Optional lowercase aliases for convenience (similar to stripe.Charge vs stripe.charge)
+base_entity = BaseEntity  # pylint: disable=invalid-name
+api_collections = APICollections  # pylint: disable=invalid-name
+api_endpoints = APIEndpoints  # pylint: disable=invalid-name
+charts = Charts  # pylint: disable=invalid-name
+classifications = Classifications  # pylint: disable=invalid-name
+containers = Containers  # pylint: disable=invalid-name
+dashboard_data_models = DashboardDataModels  # pylint: disable=invalid-name
+dashboard_services = DashboardServices  # pylint: disable=invalid-name
+dashboards = Dashboards  # pylint: disable=invalid-name
+data_contracts = DataContracts  # pylint: disable=invalid-name
+data_products = DataProducts  # pylint: disable=invalid-name
+databases = Databases  # pylint: disable=invalid-name
+database_schemas = DatabaseSchemas  # pylint: disable=invalid-name
+database_services = DatabaseServices  # pylint: disable=invalid-name
+domains = Domains  # pylint: disable=invalid-name
+glossaries = Glossaries  # pylint: disable=invalid-name
+glossary_terms = GlossaryTerms  # pylint: disable=invalid-name
+metrics = Metrics  # pylint: disable=invalid-name
+mlmodels = MLModels  # pylint: disable=invalid-name
+pipelines = Pipelines  # pylint: disable=invalid-name
+queries = Queries  # pylint: disable=invalid-name
+search_indexes = SearchIndexes  # pylint: disable=invalid-name
+stored_procedures = StoredProcedures  # pylint: disable=invalid-name
+tables = Tables  # pylint: disable=invalid-name
+tags = Tags  # pylint: disable=invalid-name
+teams = Teams  # pylint: disable=invalid-name
+test_cases = TestCases  # pylint: disable=invalid-name
+test_definitions = TestDefinitions  # pylint: disable=invalid-name
+test_suites = TestSuites  # pylint: disable=invalid-name
+users = Users  # pylint: disable=invalid-name
+
+__all__ = [
+    "OpenMetadata",
+    "OpenMetadataConfig",
+    "configure",
+    "reset",
+    "client",
+    "BaseEntity",
+    "base_entity",
+    "APICollections",
+    "api_collections",
+    "APIEndpoints",
+    "api_endpoints",
+    "Charts",
+    "charts",
+    "Classifications",
+    "classifications",
+    "Containers",
+    "containers",
+    "DashboardDataModels",
+    "dashboard_data_models",
+    "DashboardServices",
+    "dashboard_services",
+    "Dashboards",
+    "dashboards",
+    "DataContracts",
+    "data_contracts",
+    "DataProducts",
+    "data_products",
+    "Databases",
+    "databases",
+    "DatabaseSchemas",
+    "database_schemas",
+    "DatabaseServices",
+    "database_services",
+    "Domains",
+    "domains",
+    "Glossaries",
+    "glossaries",
+    "GlossaryTerms",
+    "glossary_terms",
+    "Metrics",
+    "metrics",
+    "MLModels",
+    "mlmodels",
+    "Pipelines",
+    "pipelines",
+    "Queries",
+    "queries",
+    "SearchIndexes",
+    "search_indexes",
+    "StoredProcedures",
+    "stored_procedures",
+    "Tables",
+    "tables",
+    "Tags",
+    "tags",
+    "Teams",
+    "teams",
+    "TestCases",
+    "test_cases",
+    "TestDefinitions",
+    "test_definitions",
+    "TestSuites",
+    "test_suites",
+    "Users",
+    "users",
+]

metadata/sdk/api/__init__.py

@@ -0,0 +1,7 @@
+"""
+OpenMetadata SDK API modules
+"""
+from metadata.sdk.api.lineage import Lineage
+from metadata.sdk.api.search import Search
+
+__all__ = ["Search", "Lineage"]