openmetadata-ingestion 1.7.5.3__py3-none-any.whl → 1.8.1.0__py3-none-any.whl

metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py

@@ -29,7 +29,7 @@ class BigQueryProfilerInterface(SQAProfilerInterface):
         from sqlalchemy_bigquery import STRUCT
 
         columns_list = []
-        for key, value in columns.items():
+        for key, value in columns:
             if not isinstance(value, STRUCT):
                 col = Column(f"{parent}.{key}", value)
                 # pylint: disable=protected-access
@@ -38,7 +38,7 @@ class BigQueryProfilerInterface(SQAProfilerInterface):
                 columns_list.append(col)
             else:
                 col = self._get_struct_columns(
-                    value.__dict__.get("_STRUCT_byname"), f"{parent}.{key}"
+                    value.__dict__.get("_STRUCT_fields"), f"{parent}.{key}"
                 )
                 columns_list.extend(col)
         return columns_list
@@ -53,7 +53,7 @@ class BigQueryProfilerInterface(SQAProfilerInterface):
             if isinstance(column.type, STRUCT):
                 columns.extend(
                     self._get_struct_columns(
-                        column.type.__dict__.get("_STRUCT_byname"), column.name
+                        column.type.__dict__.get("_STRUCT_fields"), column.name
                     )
                 )
             else:

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -461,6 +461,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     )
                     logger.error(error)
                     self.status.failed_profiler(error, traceback.format_exc())
+                    break
 
         # If we've exhausted all retries without success, return a tuple of None values
         return None, None, None

metadata/profiler/metrics/static/not_regexp_match_count.py

@@ -18,6 +18,7 @@ from sqlalchemy import case, column, not_
 
 from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
+from metadata.profiler.orm.functions.regexp import RegexpMatchFn
 from metadata.profiler.orm.functions.sum import SumFn
 from metadata.profiler.orm.registry import is_concatenable
 
@@ -58,9 +59,9 @@ class NotRegexCount(StaticMetric):
                 [
                     (
                         not_(
-                            column(self.col.name, self.col.type).regexp_match(
-                                self.expression
-                            )
+                            RegexpMatchFn(
+                                column(self.col.name, self.col.type), self.expression
+                            ),
                         ),
                         0,
                     )

metadata/profiler/metrics/static/regexp_match_count.py

@@ -18,6 +18,7 @@ from sqlalchemy import case, column
 
 from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
+from metadata.profiler.orm.functions.regexp import RegexpMatchFn
 from metadata.profiler.orm.functions.sum import SumFn
 from metadata.profiler.orm.registry import is_concatenable
 
@@ -57,8 +58,8 @@ class RegexCount(StaticMetric):
             case(
                 [
                     (
-                        column(self.col.name, self.col.type).regexp_match(
-                            self.expression
+                        RegexpMatchFn(
+                            column(self.col.name, self.col.type), self.expression
                         ),
                         1,
                     )

metadata/profiler/orm/functions/length.py

@@ -52,6 +52,7 @@ def _(element, compiler, **kw):
 @compiles(LenFn, Dialects.Hana)
 @compiles(LenFn, Dialects.Druid)
 @compiles(LenFn, Dialects.Doris)
+@compiles(LenFn, Dialects.Teradata)
 def _(element, compiler, **kw):
     return "LENGTH(%s)" % compiler.process(element.clauses, **kw)
 

metadata/profiler/orm/functions/md5.py

@@ -40,3 +40,12 @@ def _(element, compiler, **kw):
 @compiles(MD5, PythonDialects.BigQuery.value)
 def _(element, compiler, **kw):
     return f"TO_HEX(MD5(CAST({compiler.process(element.clauses, **kw)} AS STRING)))"
+
+
+@compiles(MD5, PythonDialects.Teradata.value)
+def _(element, compiler, **kw):
+    # There is no MD5 in Teradata or any other hashes
+    # But we can use UDF function hash_md5 published by Teradata Community
+    return (
+        f"HASH_MD5(CAST({compiler.process(element.clauses, **kw)} AS VARCHAR(32000)))"
+    )

metadata/profiler/orm/functions/modulo.py

@@ -61,6 +61,7 @@ def _(element, compiler, **kw):
 @compiles(ModuloFn, Dialects.Vertica)
 @compiles(ModuloFn, Dialects.Hana)
 @compiles(ModuloFn, Dialects.Cockroach)
+@compiles(ModuloFn, Dialects.Teradata)
 def _(element, compiler, **kw):
     """Modulo function for specific dialect"""
     value, base = validate_and_compile(element, compiler, **kw)

metadata/profiler/orm/functions/random_num.py

@@ -98,10 +98,14 @@ def _(*_, **__):
 
 
 @compiles(RandomNumFn, Dialects.Snowflake)
+@compiles(RandomNumFn, Dialects.Teradata)
 def _(*_, **__):
     """We use FROM <table> SAMPLE BERNOULLI (n) for sampling
     in snowflake. We'll return 0 to make sure we get all the rows
     from the already sampled results when executing row::MOD(0, 100) < profile_sample.
+
+    Teradata RANDOM(0,100) function can't be used inside ORDER BY clause. That's why
+    use the same trick.
     """
     return "0"
 

metadata/profiler/orm/functions/regexp.py

@@ -0,0 +1,40 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Define a regexp match function."""
+
+from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.sql.functions import GenericFunction
+
+from metadata.profiler.metrics.core import CACHE
+from metadata.profiler.orm.registry import Dialects
+
+
+class RegexpMatchFn(GenericFunction):
+    name = "regexp_match"
+    inherit_cache = CACHE
+
+
+@compiles(RegexpMatchFn)
+def _(element, compiler, **kw):
+    """Base function for regexp_match"""
+    column, pattern = element.clauses
+    fn = column.regexp_match(pattern)
+    return compiler.process(fn, **kw)
+
+
+@compiles(RegexpMatchFn, Dialects.Databricks)
+def _(element, compiler, **kw):
+    """Databricks function for regexp_match"""
+    column, pattern = element.clauses
+    compiled_column = compiler.process(column, **kw)
+    compiled_pattern = compiler.process(pattern, **kw)
+    return f"REGEXP_LIKE({compiled_column}, {compiled_pattern})"

metadata/profiler/orm/functions/table_metric_computer.py

@@ -16,7 +16,7 @@ Run profiler metrics on the table
 
 import traceback
 from abc import ABC, abstractmethod
-from typing import Callable, List, Optional, Tuple
+from typing import Callable, List, Optional, Tuple, Type
 
 from sqlalchemy import Column, MetaData, Table, func, inspect, literal, select
 from sqlalchemy.sql.expression import ColumnOperators, and_, cte
@@ -27,13 +27,29 @@ from metadata.generated.schema.entity.data.table import TableType
 from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.orm.registry import Dialects
 from metadata.profiler.processor.runner import QueryRunner
+from metadata.profiler.registry import MetricRegistry
+from metadata.utils.dependency_injector.dependency_injector import (
+    DependencyNotFoundError,
+    Inject,
+    inject,
+)
 from metadata.utils.logger import profiler_interface_registry_logger
 
 logger = profiler_interface_registry_logger()
 
+
+@inject
+def get_row_count_metric(metrics: Inject[Type[MetricRegistry]] = None):
+    if metrics is None:
+        raise DependencyNotFoundError(
+            "MetricRegistry dependency not found. Please ensure the MetricRegistry is properly registered."
+        )
+    return metrics.ROW_COUNT().name()
+
+
 COLUMN_COUNT = "columnCount"
 COLUMN_NAMES = "columnNames"
-ROW_COUNT = Metrics.ROW_COUNT().name()
+ROW_COUNT = get_row_count_metric()
 SIZE_IN_BYTES = "sizeInBytes"
 CREATE_DATETIME = "createDateTime"
 
@@ -336,12 +352,16 @@ class BigQueryTableMetricComputer(BaseTableMetricComputer):
             Column("dataset_id") == self.schema_name,
             Column("table_id") == self.table_name,
         ]
+        schema = (
+            self.schema_name.startswith(
+                f"{self.conn_config.credentials.gcpConfig.projectId.root}."
+            )
+            and self.schema_name
+            or f"{self.conn_config.credentials.gcpConfig.projectId.root}.{self.schema_name}"
+        )
         query = self._build_query(
             columns,
-            self._build_table(
-                "__TABLES__",
-                f"{self.conn_config.credentials.gcpConfig.projectId.root}.{self.schema_name}",
-            ),
+            self._build_table("__TABLES__", schema),
             where_clause,
         )
         res = self.runner._session.execute(query).first()
@@ -358,9 +378,15 @@ class BigQueryTableMetricComputer(BaseTableMetricComputer):
 class MySQLTableMetricComputer(BaseTableMetricComputer):
     """MySQL Table Metric Computer"""
 
-    def compute(self):
+    @inject
+    def compute(self, metrics: Inject[Type[MetricRegistry]] = None):
         """compute table metrics for mysql"""
 
+        if metrics is None:
+            raise DependencyNotFoundError(
+                "MetricRegistry dependency not found. Please ensure the MetricRegistry is properly registered."
+            )
+
         columns = [
             Column("TABLE_ROWS").label(ROW_COUNT),
             (Column("data_length") + Column("index_length")).label(SIZE_IN_BYTES),
@@ -386,7 +412,7 @@ class MySQLTableMetricComputer(BaseTableMetricComputer):
         res = res._asdict()
         # innodb row count is an estimate we need to patch the row count with COUNT(*)
         # https://dev.mysql.com/doc/refman/8.3/en/information-schema-innodb-tablestats-table.html
-        row_count = self.runner.select_first_from_table(Metrics.ROW_COUNT().fn())
+        row_count = self.runner.select_first_from_table(metrics.ROW_COUNT().fn())
         res.update({ROW_COUNT: row_count.rowCount})
         return res
 

metadata/profiler/orm/registry.py

@@ -86,6 +86,7 @@ class PythonDialects(Enum):
     SingleStore = "singlestore"
     SQLite = "sqlite"
     Snowflake = "snowflake"
+    Teradata = "teradatasql"
     Trino = "trino"
     Vertica = "vertica"
 

metadata/profiler/processor/core.py

@@ -16,7 +16,7 @@ from __future__ import annotations
 
 import traceback
 from datetime import datetime, timezone
-from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type
+from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type, cast
 
 from pydantic import ValidationError
 from sqlalchemy import Column
@@ -94,7 +94,7 @@ class Profiler(Generic[TMetric]):
         :param profile_sample: % of rows to use for sampling column metrics
         """
         self.global_profiler_configuration: Optional[ProfilerConfiguration] = (
-            global_profiler_configuration.config_value
+            cast(ProfilerConfiguration, global_profiler_configuration.config_value)
             if global_profiler_configuration
             else None
         )

metadata/profiler/processor/default.py

@@ -12,55 +12,54 @@
 """
 Default simple profiler to use
 """
-from typing import List, Optional
+from typing import List, Optional, Type
 
 from sqlalchemy.orm import DeclarativeMeta
 
-from metadata.generated.schema.configuration.profilerConfiguration import (
-    ProfilerConfiguration,
-)
 from metadata.generated.schema.entity.data.table import ColumnProfilerConfig
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.settings.settings import Settings
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.profiler.interface.profiler_interface import ProfilerInterface
 from metadata.profiler.metrics.core import Metric, add_props
-from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.processor.core import Profiler
+from metadata.profiler.registry import MetricRegistry
 
 
 def get_default_metrics(
+    metrics_registry: Type[MetricRegistry],
     table: DeclarativeMeta,
     ometa_client: Optional[OpenMetadata] = None,
     db_service: Optional[DatabaseService] = None,
 ) -> List[Metric]:
     return [
         # Table Metrics
-        Metrics.ROW_COUNT.value,
-        add_props(table=table)(Metrics.COLUMN_COUNT.value),
-        add_props(table=table)(Metrics.COLUMN_NAMES.value),
+        metrics_registry.ROW_COUNT.value,
+        add_props(table=table)(metrics_registry.COLUMN_COUNT.value),
+        add_props(table=table)(metrics_registry.COLUMN_NAMES.value),
         # We'll use the ometa_client & db_service in case we need to fetch info to ES
         add_props(table=table, ometa_client=ometa_client, db_service=db_service)(
-            Metrics.SYSTEM.value
+            metrics_registry.SYSTEM.value
         ),
         # Column Metrics
-        Metrics.MEDIAN.value,
-        Metrics.FIRST_QUARTILE.value,
-        Metrics.THIRD_QUARTILE.value,
-        Metrics.MEAN.value,
-        Metrics.COUNT.value,
-        Metrics.DISTINCT_COUNT.value,
-        Metrics.DISTINCT_RATIO.value,
-        Metrics.MIN.value,
-        Metrics.MAX.value,
-        Metrics.NULL_COUNT.value,
-        Metrics.NULL_RATIO.value,
-        Metrics.STDDEV.value,
-        Metrics.SUM.value,
-        Metrics.UNIQUE_COUNT.value,
-        Metrics.UNIQUE_RATIO.value,
-        Metrics.IQR.value,
-        Metrics.HISTOGRAM.value,
-        Metrics.NON_PARAMETRIC_SKEW.value,
+        metrics_registry.MEDIAN.value,
+        metrics_registry.FIRST_QUARTILE.value,
+        metrics_registry.THIRD_QUARTILE.value,
+        metrics_registry.MEAN.value,
+        metrics_registry.COUNT.value,
+        metrics_registry.DISTINCT_COUNT.value,
+        metrics_registry.DISTINCT_RATIO.value,
+        metrics_registry.MIN.value,
+        metrics_registry.MAX.value,
+        metrics_registry.NULL_COUNT.value,
+        metrics_registry.NULL_RATIO.value,
+        metrics_registry.STDDEV.value,
+        metrics_registry.SUM.value,
+        metrics_registry.UNIQUE_COUNT.value,
+        metrics_registry.UNIQUE_RATIO.value,
+        metrics_registry.IQR.value,
+        metrics_registry.HISTOGRAM.value,
+        metrics_registry.NON_PARAMETRIC_SKEW.value,
     ]
 
 
@@ -74,12 +73,14 @@ class DefaultProfiler(Profiler):
     def __init__(
         self,
         profiler_interface: ProfilerInterface,
+        metrics_registry: Type[MetricRegistry],
         include_columns: Optional[List[ColumnProfilerConfig]] = None,
         exclude_columns: Optional[List[str]] = None,
-        global_profiler_configuration: Optional[ProfilerConfiguration] = None,
+        global_profiler_configuration: Optional[Settings] = None,
         db_service=None,
     ):
         _metrics = get_default_metrics(
+            metrics_registry=metrics_registry,
             table=profiler_interface.table,
             ometa_client=profiler_interface.ometa_client,
             db_service=db_service,

metadata/profiler/processor/metric_filter.py

@@ -33,22 +33,35 @@ from metadata.profiler.metrics.core import (
     SystemMetric,
     TMetric,
 )
-from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.orm.converter.converter_registry import converter_registry
+from metadata.profiler.registry import MetricRegistry
+from metadata.utils.dependency_injector.dependency_injector import (
+    DependencyNotFoundError,
+    Inject,
+    inject,
+)
 from metadata.utils.sqa_like_column import SQALikeColumn
 
 
 class MetricFilter:
     """Metric filter class for profiler"""
 
+    @inject
     def __init__(
         self,
         metrics: Tuple[Type[TMetric]],
         global_profiler_config: Optional[ProfilerConfiguration] = None,
         table_profiler_config: Optional[TableProfilerConfig] = None,
         column_profiler_config: Optional[List[ColumnProfilerConfig]] = None,
+        metrics_registry: Inject[Type[MetricRegistry]] = None,
     ):
+        if metrics_registry is None:
+            raise DependencyNotFoundError(
+                "MetricRegistry dependency not found. Please ensure the MetricRegistry is properly registered."
+            )
+
         self.metrics = metrics
+        self.metrics_registry = metrics_registry
         self.global_profiler_config = global_profiler_config
         self.table_profiler_config = table_profiler_config
         self.column_profiler_config = column_profiler_config
@@ -196,7 +209,7 @@ class MetricFilter:
 
         metrics = [
             Metric.value
-            for Metric in Metrics
+            for Metric in self.metrics_registry
             if Metric.value.name() in {mtrc.value for mtrc in col_dtype_config.metrics}
             and Metric.value in metrics
         ]
@@ -240,7 +253,7 @@ class MetricFilter:
 
         metrics = [
             Metric.value
-            for Metric in Metrics
+            for Metric in self.metrics_registry
             if Metric.value.name().lower() in {mtrc.lower() for mtrc in metric_names}
             and Metric.value in metrics
         ]

metadata/profiler/processor/models.py

@@ -13,20 +13,30 @@
 Models to map profiler definitions
 JSON workflows to the profiler
 """
-from typing import List, Optional
+from typing import List, Optional, Type
 
 from pydantic import BaseModel, BeforeValidator
 from typing_extensions import Annotated
 
-from metadata.profiler.metrics.registry import Metrics
+from metadata.profiler.registry import MetricRegistry
+from metadata.utils.dependency_injector.dependency_injector import (
+    DependencyNotFoundError,
+    Inject,
+    inject,
+)
 
 
-def valid_metric(value: str):
+@inject
+def valid_metric(value: str, metrics: Inject[Type[MetricRegistry]] = None):
     """
     Validate that the input metrics are correctly named
     and can be found in the Registry
     """
-    if not Metrics.get(value.upper()):
+    if metrics is None:
+        raise DependencyNotFoundError(
+            "MetricRegistry dependency not found. Please ensure the MetricRegistry is properly registered."
+        )
+    if not metrics.get(value.upper()):
         raise ValueError(
             f"Metric name {value} is not a proper metric name from the Registry"
         )

metadata/profiler/source/database/base/profiler_resolver.py

@@ -0,0 +1,35 @@
+from abc import ABC, abstractmethod
+from typing import Tuple, Type
+
+from metadata.generated.schema.entity.services.serviceType import ServiceType
+from metadata.profiler.interface.profiler_interface import ProfilerInterface
+from metadata.sampler.sampler_interface import SamplerInterface
+from metadata.utils.service_spec.service_spec import (
+    import_profiler_class,
+    import_sampler_class,
+)
+
+
+class ProfilerResolver(ABC):
+    """Abstract class for the profiler resolver"""
+
+    @staticmethod
+    @abstractmethod
+    def resolve(
+        processing_engine: str, service_type: ServiceType, source_type: str
+    ) -> Tuple[Type[SamplerInterface], Type[ProfilerInterface]]:
+        """Resolve the sampler and profiler based on the processing engine."""
+        raise NotImplementedError
+
+
+class DefaultProfilerResolver(ProfilerResolver):
+    """Default profiler resolver"""
+
+    @staticmethod
+    def resolve(
+        processing_engine: str, service_type: ServiceType, source_type: str
+    ) -> Tuple[Type[SamplerInterface], Type[ProfilerInterface]]:
+        """Resolve the sampler and profiler based on the processing engine."""
+        sampler_class = import_sampler_class(service_type, source_type=source_type)
+        profiler_class = import_profiler_class(service_type, source_type=source_type)
+        return sampler_class, profiler_class