openmetadata-ingestion 1.2.5.2__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/profiler/metrics/system/system.py

@@ -23,6 +23,8 @@ from sqlalchemy.orm import DeclarativeMeta, Session
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
 )
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.profiler.metrics.core import SystemMetric
 from metadata.profiler.metrics.system.dml_operation import (
     DML_OPERATION_MAP,
@@ -46,7 +48,11 @@ from metadata.profiler.orm.registry import Dialects
 from metadata.utils.dispatch import valuedispatch
 from metadata.utils.helpers import deep_size_of_dict
 from metadata.utils.logger import profiler_logger
-from metadata.utils.profiler_utils import get_value_from_cache, set_cache
+from metadata.utils.profiler_utils import (
+    SnowflakeQueryResult,
+    get_value_from_cache,
+    set_cache,
+)
 
 logger = profiler_logger()
 
@@ -285,11 +291,48 @@ def _(
     return metric_results
 
 
+def _snowflake_build_query_result(
+    session: Session,
+    table: DeclarativeMeta,
+    database: str,
+    schema: str,
+    ometa_client: OpenMetadata,
+    db_service: DatabaseService,
+) -> List[SnowflakeQueryResult]:
+    """List and parse snowflake DML query results"""
+    rows = session.execute(
+        text(
+            INFORMATION_SCHEMA_QUERY.format(
+                tablename=table.__tablename__,  # type: ignore
+                insert=DatabaseDMLOperations.INSERT.value,
+                update=DatabaseDMLOperations.UPDATE.value,
+                delete=DatabaseDMLOperations.DELETE.value,
+                merge=DatabaseDMLOperations.MERGE.value,
+            )
+        )
+    )
+    query_results = []
+    for row in rows:
+        result = get_snowflake_system_queries(
+            row=row,
+            database=database,
+            schema=schema,
+            ometa_client=ometa_client,
+            db_service=db_service,
+        )
+        if result:
+            query_results.append(result)
+
+    return query_results
+
+
 @get_system_metrics_for_dialect.register(Dialects.Snowflake)
 def _(
     dialect: str,
     session: Session,
     table: DeclarativeMeta,
+    ometa_client: OpenMetadata,
+    db_service: DatabaseService,
     *args,
     **kwargs,
 ) -> Optional[List[Dict]]:
@@ -315,22 +358,14 @@ def _(
 
     metric_results: List[Dict] = []
 
-    rows = session.execute(
-        text(
-            INFORMATION_SCHEMA_QUERY.format(
-                tablename=table.__tablename__,  # type: ignore
-                insert=DatabaseDMLOperations.INSERT.value,
-                update=DatabaseDMLOperations.UPDATE.value,
-                delete=DatabaseDMLOperations.DELETE.value,
-                merge=DatabaseDMLOperations.MERGE.value,
-            )
-        )
+    query_results = _snowflake_build_query_result(
+        session=session,
+        table=table,
+        database=database,
+        schema=schema,
+        ometa_client=ometa_client,
+        db_service=db_service,
     )
-    query_results = []
-    for row in rows:
-        result = get_snowflake_system_queries(row, database, schema)
-        if result:
-            query_results.append(result)
 
     for query_result in query_results:
         rows_affected = None
@@ -409,12 +444,17 @@ class System(SystemMetric):
             logger.debug("Clearing system cache")
             SYSTEM_QUERY_RESULT_CACHE.clear()
 
+    def _validate_attrs(self, attr_list: List[str]) -> None:
+        """Validate the necessary attributes given via add_props"""
+        for attr in attr_list:
+            if not hasattr(self, attr):
+                raise AttributeError(
+                    f"System requires a table to be set: add_props({attr}=...)(Metrics.SYSTEM.value)"
+                )
+
     def sql(self, session: Session, **kwargs):
         """Implements the SQL logic to fetch system data"""
-        if not hasattr(self, "table"):
-            raise AttributeError(
-                "System requires a table to be set: add_props(table=...)(Metrics.COLUMN_COUNT)"
-            )
+        self._validate_attrs(["table", "ometa_client", "db_service"])
 
         conn_config = kwargs.get("conn_config")
 
@@ -423,6 +463,8 @@ class System(SystemMetric):
             session=session,
             table=self.table,  # pylint: disable=no-member
             conn_config=conn_config,
+            ometa_client=self.ometa_client,  # pylint: disable=no-member
+            db_service=self.db_service,  # pylint: disable=no-member
         )
         self._manage_cache()
         return system_metrics

metadata/profiler/orm/functions/length.py

@@ -50,6 +50,7 @@ def _(element, compiler, **kw):
 @compiles(LenFn, Dialects.Db2)
 @compiles(LenFn, Dialects.Hana)
 @compiles(LenFn, Dialects.Druid)
+@compiles(LenFn, Dialects.Doris)
 def _(element, compiler, **kw):
     return "LENGTH(%s)" % compiler.process(element.clauses, **kw)
 

metadata/profiler/orm/functions/median.py

@@ -186,3 +186,10 @@ def _(elements, compiler, **kwargs):  # pylint: disable=unused-argument
     """.format(
         col=col, table=table, percentile=percentile
     )
+
+
+@compiles(MedianFn, Dialects.Doris)
+def _(elements, compiler, **kwargs):
+    col = compiler.process(elements.clauses.clauses[0])
+    percentile = elements.clauses.clauses[2].value
+    return "percentile_approx(%s, %.2f)" % (col, percentile)

metadata/profiler/orm/functions/table_metric_computer.py

@@ -0,0 +1,462 @@
+#  pylint: disable=protected-access,attribute-defined-outside-init
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Run profiler metrics on the table
+"""
+
+import traceback
+from abc import ABC, abstractmethod
+from typing import Callable, List, Optional, Tuple
+
+from sqlalchemy import Column, MetaData, Table, func, inspect, literal, select
+from sqlalchemy.sql.expression import ColumnOperators, and_, cte
+from sqlalchemy.types import String
+
+from metadata.profiler.metrics.registry import Metrics
+from metadata.profiler.orm.registry import Dialects
+from metadata.profiler.processor.runner import QueryRunner
+from metadata.utils.logger import profiler_interface_registry_logger
+
+logger = profiler_interface_registry_logger()
+
+COLUMN_COUNT = "columnCount"
+COLUMN_NAMES = "columnNames"
+ROW_COUNT = "rowCount"
+SIZE_IN_BYTES = "sizeInBytes"
+CREATE_DATETIME = "createDateTime"
+
+ERROR_MSG = (
+    "Schema/Table name not found in table args. Falling back to default computation"
+)
+
+
+class AbstractTableMetricComputer(ABC):
+    """Base table computer"""
+
+    def __init__(self, runner: QueryRunner, metrics: List[Metrics], conn_config):
+        """Instantiate base table computer"""
+        self._runner = runner
+        self._metrics = metrics
+        self._conn_config = conn_config
+        self._database = self._runner._session.get_bind().url.database
+        self._table = self._runner.table
+
+    @property
+    def database(self):
+        return self._database
+
+    @property
+    def table(self):
+        return self._table
+
+    @property
+    def runner(self):
+        return self._runner
+
+    @property
+    def metrics(self):
+        return self._metrics
+
+    @property
+    def table_name(self):
+        return self._table_name
+
+    @property
+    def schema_name(self):
+        return self._schema_name
+
+    @property
+    def conn_config(self):
+        return self._conn_config
+
+    def _set_table_and_schema_name(self):
+        """get table and schema name from table args
+
+        Args:
+            table (DeclarativeMeta): _description_
+        """
+        try:
+            self._schema_name = self.table.__table_args__.get("schema")
+            self._table_name = self.table.__tablename__
+        except AttributeError:
+            raise AttributeError(ERROR_MSG)
+
+    def _build_table(self, table, schema) -> Table:
+        """build table object from table name and schema name
+
+        Args:
+            table_name (str): table name
+            schema_name (str): schema name
+
+        Returns:
+            Table
+        """
+        if schema:
+            return Table(table, MetaData(), schema=schema)
+        return Table(table, MetaData())
+
+    def _get_col_names_and_count(self) -> Tuple[str, int]:
+        """get column names and count from table
+
+        Args:
+            table (DeclarativeMeta): table object
+
+        Returns:
+            Tuple[str, int]
+        """
+        col_names = literal(",".join(inspect(self.table).c.keys()), type_=String).label(
+            COLUMN_NAMES
+        )
+        col_count = literal(len(inspect(self.table).c)).label(COLUMN_COUNT)
+        return col_names, col_count
+
+    def _build_query(
+        self,
+        columns: List[Column],
+        table: Table,
+        where_clause: Optional[List[ColumnOperators]] = None,
+    ):
+        query = select(*columns).select_from(table)
+        if where_clause:
+            query = query.where(*where_clause)
+
+        return query
+
+    @abstractmethod
+    def compute(self):
+        """Default compute behavior for table metrics"""
+        raise NotImplementedError
+
+
+class BaseTableMetricComputer(AbstractTableMetricComputer):
+    """Base table computer"""
+
+    def _check_and_return(self, res):
+        """Check if the result is None and return the result or fallback
+
+        Args:
+            res (object): result
+        """
+        if res.rowCount is None:
+            return super().compute()
+        return res
+
+    def compute(self):
+        """Default compute behavior for table metrics"""
+        return self.runner.select_first_from_table(
+            *[metric().fn() for metric in self.metrics]
+        )
+
+
+class SnowflakeTableMetricComputer(BaseTableMetricComputer):
+    """Snowflake Table Metric Computer"""
+
+    def compute(self):
+        """Compute table metrics for snowflake"""
+        columns = [
+            Column("ROW_COUNT").label("rowCount"),
+            Column("BYTES").label("sizeInBytes"),
+            Column("CREATED").label("createDateTime"),
+            *self._get_col_names_and_count(),
+        ]
+        where_clause = [
+            func.lower(Column("TABLE_CATALOG")) == self.database.lower(),
+            func.lower(Column("TABLE_SCHEMA")) == self.schema_name.lower(),
+            func.lower(Column("TABLE_NAME")) == self.table_name.lower(),
+        ]
+        query = self._build_query(
+            columns,
+            self._build_table("TABLES", f"{self.database}.INFORMATION_SCHEMA"),
+            where_clause,
+        )
+
+        rest = self._runner._session.execute(query).first()
+        if rest.rowCount is None:
+            # if we don't have any row count, fallback to the base logic
+            return super().compute()
+        return rest
+
+
+class OracleTableMetricComputer(BaseTableMetricComputer):
+    """Oracle Table Metric Computer"""
+
+    def compute(self):
+        """Compute table metrics for oracle"""
+        create_date = cte(
+            self._build_query(
+                [
+                    Column("owner"),
+                    Column("object_name").label("table_name"),
+                    Column("created"),
+                ],
+                self._build_table("dba_objects", None),
+                [
+                    func.lower(Column("owner")) == self.schema_name.lower(),
+                    func.lower(Column("object_name")) == self.table_name.lower(),
+                ],
+            )
+        )
+
+        row_count = cte(
+            self._build_query(
+                [
+                    Column("owner"),
+                    Column("table_name"),
+                    Column("NUM_ROWS"),
+                ],
+                self._build_table("all_tables", None),
+                [
+                    func.lower(Column("owner")) == self.schema_name.lower(),
+                    func.lower(Column("table_name")) == self.table_name.lower(),
+                ],
+            )
+        )
+
+        columns = [
+            Column("NUM_ROWS").label("rowCount"),
+            Column("created").label("createDateTime"),
+            *self._get_col_names_and_count(),
+        ]
+        query = self._build_query(columns, row_count).join(
+            create_date,
+            and_(
+                row_count.c.table_name == create_date.c.table_name,
+                row_count.c.owner == create_date.c.owner,
+            ),
+        )
+
+        res = self.runner._session.execute(query).first()
+        if res.rowCount is None:
+            # if we don't have any row count, fallback to the base logic
+            return super().compute()
+        return res
+
+
+class ClickHouseTableMetricComputer(BaseTableMetricComputer):
+    """ClickHouse Table Metric Computer"""
+
+    def compute(self):
+        """compute table metrics for clickhouse"""
+        columns = [
+            Column("total_rows").label("rowCount"),
+            Column("total_bytes").label("sizeInBytes"),
+            *self._get_col_names_and_count(),
+        ]
+
+        where_clause = [
+            Column("database") == self.schema_name,
+            Column("name") == self.table_name,
+        ]
+
+        query = self._build_query(
+            columns, self._build_table("tables", "system"), where_clause
+        )
+
+        res = self.runner._session.execute(query).first()
+        if res.rowCount is None:
+            # if we don't have any row count, fallback to the base logic
+            return super().compute()
+        return res
+
+
+class BigQueryTableMetricComputer(BaseTableMetricComputer):
+    """BigQuery Table Metric Computer"""
+
+    def compute(self):
+        """compute table metrics for bigquery"""
+        try:
+            return self.tables()
+        except Exception as exc:
+            # if an error occurs fetching data from `__TABLES__`, fallback to `TABLE_STORAGE`
+            logger.debug(f"Error retrieving table metadata from `__TABLES__`: {exc}")
+            return self.table_storage()
+
+    def table_storage(self):
+        """Fall back method if retrieving table metadata from`__TABLES__` fails"""
+        columns = [
+            Column("total_rows").label("rowCount"),
+            Column("total_logical_bytes").label("sizeInBytes"),
+            Column("creation_time").label("createDateTime"),
+            *self._get_col_names_and_count(),
+        ]
+
+        where_clause = [
+            Column("project_id")
+            == self.conn_config.credentials.gcpConfig.projectId.__root__,
+            Column("table_schema") == self.schema_name,
+            Column("table_name") == self.table_name,
+        ]
+
+        query = self._build_query(
+            columns,
+            self._build_table(
+                "TABLE_STORAGE",
+                f"region-{self.conn_config.usageLocation}.INFORMATION_SCHEMA",
+            ),
+            where_clause,
+        )
+
+        res = self.runner._session.execute(query).first()
+        if res.rowCount is None:
+            # if we don't have any row count, fallback to the base logic
+            return super().compute()
+        return res
+
+    def tables(self):
+        """retrieve table metadata from `__TABLES__`"""
+        columns = [
+            Column("row_count").label("rowCount"),
+            Column("size_bytes").label("sizeInBytes"),
+            Column("creation_time").label("createDateTime"),
+            *self._get_col_names_and_count(),
+        ]
+        where_clause = [
+            Column("project_id")
+            == self.conn_config.credentials.gcpConfig.projectId.__root__,
+            Column("dataset_id") == self.schema_name,
+            Column("table_id") == self.table_name,
+        ]
+
+        query = self._build_query(
+            columns,
+            self._build_table(
+                "__TABLES__",
+                f"{self.conn_config.credentials.gcpConfig.projectId.__root__}.{self.schema_name}",
+            ),
+            where_clause,
+        )
+        res = self.runner._session.execute(query).first()
+        if res.rowCount is None:
+            # if we don't have any row count, fallback to the base logic
+            return super().compute()
+        return res
+
+
+class MySQLTableMetricComputer(BaseTableMetricComputer):
+    """MySQL Table Metric Computer"""
+
+    def compute(self):
+        """compute table metrics for mysql"""
+
+        columns = [
+            Column("TABLE_ROWS").label(ROW_COUNT),
+            (Column("data_length") + Column("index_length")).label(SIZE_IN_BYTES),
+            Column("CREATE_TIME").label(CREATE_DATETIME),
+            *self._get_col_names_and_count(),
+        ]
+        where_clause = [
+            Column("TABLE_SCHEMA") == self.schema_name,
+            Column("TABLE_NAME") == self.table_name,
+        ]
+        query = self._build_query(
+            columns, self._build_table("tables", "information_schema"), where_clause
+        )
+
+        res = self.runner._session.execute(query).first()
+        if res.rowCount is None:
+            # if we don't have any row count, fallback to the base logic
+            return super().compute()
+        return res
+
+
+class RedshiftTableMetricComputer(BaseTableMetricComputer):
+    """Redshift Table Metric Computer"""
+
+    def compute(self):
+        """compute table metrics for redshift"""
+        columns = [
+            Column("estimated_visible_rows").label(ROW_COUNT),
+            Column("size").label(SIZE_IN_BYTES),
+            Column("create_time").label(CREATE_DATETIME),
+            *self._get_col_names_and_count(),
+        ]
+
+        where_clause = [
+            Column("schema") == self.schema_name,
+            Column("table") == self.table_name,
+        ]
+
+        query = self._build_query(
+            columns, self._build_table("svv_table_info", "pg_catalog"), where_clause
+        )
+        res = self.runner._session.execute(query).first()
+        if res.rowCount is None:
+            # if we don't have any row count, fallback to the base logic
+            return super().compute()
+        return res
+
+
+class TableMetricComputer:
+    """Table Metric Construct"""
+
+    def __init__(
+        self, dialect: str, runner: QueryRunner, metrics: List[Metrics], conn_config
+    ):
+        """Instantiate table metric computer with a dialect computer"""
+        self._dialect = dialect
+        self._runner = runner
+        self._metrics = metrics
+        self._conn_config = conn_config
+        self.table_metric_computer: AbstractTableMetricComputer = (
+            table_metric_computer_factory.construct(
+                self._dialect,
+                runner=self._runner,
+                metrics=self._metrics,
+                conn_config=self._conn_config,
+            )
+        )
+
+    def compute(self):
+        """Compute table metrics"""
+        return self.table_metric_computer.compute()
+
+
+class TableMetricComputerFactory:
+    """Factory returning the correct construct for the table metrics based on dialect"""
+
+    def __init__(self):
+        self._constructs = {}
+
+    def register(self, dialect: str, construct: Callable):
+        """Register a construct for a dialect"""
+        self._constructs[dialect] = construct
+
+    def construct(self, dialect, **kwargs):
+        """Construct the query"""
+
+        # check if we have registered a construct for the dialect
+        construct = self._constructs.get(dialect)
+        if not construct:
+            construct = self._constructs["base"]
+            return construct(**kwargs)
+
+        try:
+            construct_instance: AbstractTableMetricComputer = construct(**kwargs)
+            construct_instance._set_table_and_schema_name()
+            return construct_instance
+        except Exception:
+            # if an error occurs, fallback to the base construct
+            logger.debug(traceback.format_exc())
+            return self._constructs["base"](**kwargs)
+
+
+table_metric_computer_factory = TableMetricComputerFactory()
+table_metric_computer_factory.register("base", BaseTableMetricComputer)
+table_metric_computer_factory.register(Dialects.Redshift, RedshiftTableMetricComputer)
+table_metric_computer_factory.register(Dialects.MySQL, MySQLTableMetricComputer)
+table_metric_computer_factory.register(Dialects.BigQuery, BigQueryTableMetricComputer)
+table_metric_computer_factory.register(
+    Dialects.ClickHouse, ClickHouseTableMetricComputer
+)
+table_metric_computer_factory.register(Dialects.Oracle, OracleTableMetricComputer)
+table_metric_computer_factory.register(Dialects.Snowflake, SnowflakeTableMetricComputer)

metadata/profiler/orm/registry.py

@@ -56,6 +56,7 @@ class Dialects(Enum):
     ClickHouse = "clickhouse"
     Databricks = "databricks"
     Db2 = "db2"
+    Doris = "pydoris"
     Druid = "druid"
     DynamoDB = "dynamoDB"
     Glue = "glue"