openmetadata-ingestion 1.3.2.0rc2__py3-none-any.whl → 1.3.3.0__py3-none-any.whl

metadata/ingestion/lineage/parser.py

@@ -217,6 +217,11 @@ class LineageParser:
         """
         aliases = self.table_aliases
         values = identifier.value.split(".")
+
+        if len(values) > 4:
+            logger.debug(f"Invalid comparison element from identifier: {identifier}")
+            return None, None
+
         database_name, schema_name, table_or_alias, column_name = (
             [None] * (4 - len(values))
         ) + values
@@ -307,32 +312,39 @@ class LineageParser:
                 comparisons.append(sub)
 
         for comparison in comparisons:
-            if "." not in comparison.left.value or "." not in comparison.right.value:
-                logger.debug(f"Ignoring comparison {comparison}")
-                continue
-
-            table_left, column_left = self.get_comparison_elements(
-                identifier=comparison.left
-            )
-            table_right, column_right = self.get_comparison_elements(
-                identifier=comparison.right
-            )
-
-            if not table_left or not table_right:
-                logger.warning(
-                    f"Can't extract table names when parsing JOIN information from {comparison}"
+            try:
+                if (
+                    "." not in comparison.left.value
+                    or "." not in comparison.right.value
+                ):
+                    logger.debug(f"Ignoring comparison {comparison}")
+                    continue
+
+                table_left, column_left = self.get_comparison_elements(
+                    identifier=comparison.left
+                )
+                table_right, column_right = self.get_comparison_elements(
+                    identifier=comparison.right
                 )
-                logger.debug(f"Query: {sql_statement}")
-                continue
 
-            left_table_column = TableColumn(table=table_left, column=column_left)
-            right_table_column = TableColumn(table=table_right, column=column_right)
+                if not table_left or not table_right:
+                    logger.warning(
+                        f"Can't extract table names when parsing JOIN information from {comparison}"
+                    )
+                    logger.debug(f"Query: {sql_statement}")
+                    continue
+
+                left_table_column = TableColumn(table=table_left, column=column_left)
+                right_table_column = TableColumn(table=table_right, column=column_right)
 
-            # We just send the info once, from Left -> Right.
-            # The backend will prepare the symmetric information.
-            self.stateful_add_table_joins(
-                join_data, left_table_column, right_table_column
-            )
+                # We just send the info once, from Left -> Right.
+                # The backend will prepare the symmetric information.
+                self.stateful_add_table_joins(
+                    join_data, left_table_column, right_table_column
+                )
+            except Exception as exc:
+                logger.debug(f"Cannot process comparison {comparison}: {exc}")
+                logger.debug(traceback.format_exc())
 
     @cached_property
     def table_joins(self) -> Dict[str, List[TableColumnJoin]]:

metadata/ingestion/ometa/routes.py

@@ -90,6 +90,16 @@ from metadata.generated.schema.api.tests.createTestDefinition import (
 from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest
 from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart
 from metadata.generated.schema.dataInsight.kpi.kpi import Kpi
+from metadata.generated.schema.entity.applications.app import App
+from metadata.generated.schema.entity.applications.createAppRequest import (
+    CreateAppRequest,
+)
+from metadata.generated.schema.entity.applications.marketplace.appMarketPlaceDefinition import (
+    AppMarketPlaceDefinition,
+)
+from metadata.generated.schema.entity.applications.marketplace.createAppMarketPlaceDefinitionReq import (
+    CreateAppMarketPlaceDefinitionRequest,
+)
 from metadata.generated.schema.entity.automations.workflow import Workflow
 from metadata.generated.schema.entity.bot import Bot
 from metadata.generated.schema.entity.classification.classification import (
@@ -232,4 +242,9 @@ ROUTES = {
     # Suggestions
     Suggestion.__name__: "/suggestions",
     CreateSuggestionRequest.__name__: "/suggestions",
+    # Apps
+    App.__name__: "/apps",
+    CreateAppRequest.__name__: "/apps",
+    AppMarketPlaceDefinition.__name__: "/apps/marketplace",
+    CreateAppMarketPlaceDefinitionRequest.__name__: "/apps/marketplace",
 }

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -173,7 +173,7 @@ def _(config: DbtCloudConfig):  # pylint: disable=too-many-locals
             params_data["job_definition_id"] = job_id
 
         response = client.get(f"/accounts/{account_id}/runs", data=params_data)
-        if not response and not response.get("data"):
+        if not response or not response.get("data"):
             raise DBTConfigException(
                 "Unable to get the dbt job runs information.\n"
                 "Please check if the auth token is correct and has the necessary scopes to fetch dbt runs"

metadata/ingestion/source/database/oracle/connection.py

@@ -38,6 +38,7 @@ from metadata.ingestion.connections.builders import (
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.oracle.queries import CHECK_ACCESS_TO_DBA
 from metadata.utils.logger import ingestion_logger
 
 CX_ORACLE_LIB_VERSION = "8.3.0"
@@ -136,9 +137,13 @@ def test_connection(
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
+
+    test_conn_queries = {"CheckAccess": CHECK_ACCESS_TO_DBA}
+
     test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        queries=test_conn_queries,
     )

metadata/ingestion/source/database/oracle/queries.py

@@ -88,7 +88,7 @@ WHERE
     type = 'PROCEDURE' and owner = '{schema}'
 """
 )
-
+CHECK_ACCESS_TO_DBA = "SELECT table_name FROM DBA_TABLES where ROWNUM < 2"
 ORACLE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
     """
 WITH SP_HISTORY AS (SELECT

metadata/ingestion/source/database/unitycatalog/connection.py

@@ -68,16 +68,20 @@ def test_connection(
             break
 
     def get_schemas(connection: WorkspaceClient, table_obj: DatabricksTable):
-        for schema in connection.schemas.list(catalog_name=table_obj.catalog_name):
-            table_obj.schema_name = schema.name
-            break
+        for catalog in connection.catalogs.list():
+            for schema in connection.schemas.list(catalog_name=catalog.name):
+                if schema.name:
+                    table_obj.schema_name = schema.name
+                    table_obj.catalog_name = catalog.name
+                    return
 
     def get_tables(connection: WorkspaceClient, table_obj: DatabricksTable):
-        for table in connection.tables.list(
-            catalog_name=table_obj.catalog_name, schema_name=table_obj.schema_name
-        ):
-            table_obj.name = table.name
-            break
+        if table_obj.catalog_name and table_obj.schema_name:
+            for table in connection.tables.list(
+                catalog_name=table_obj.catalog_name, schema_name=table_obj.schema_name
+            ):
+                table_obj.name = table.name
+                break
 
     test_fn = {
         "CheckAccess": connection.catalogs.list,

metadata/ingestion/source/pipeline/dagster/metadata.py

@@ -214,7 +214,9 @@ class DagsterSource(PipelineServiceSource):
             service_name=self.context.pipeline_service,
             pipeline_name=self.context.pipeline,
         )
-        pipeline_entity = self.metadata.get_by_name(entity=Pipeline, fqn=pipeline_fqn)
+        pipeline_entity = self.metadata.get_by_name(
+            entity=Pipeline, fqn=pipeline_fqn, fields=["tasks"]
+        )
         for task in pipeline_entity.tasks or []:
             try:
                 runs = self.client.get_task_runs(

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -189,6 +189,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                 runner=runner,
                 metrics=metrics,
                 conn_config=self.service_connection_config,
+                entity=self.table_entity,
             )
             row = table_metric_computer.compute()
             if row:

metadata/profiler/orm/functions/table_metric_computer.py

@@ -22,6 +22,8 @@ from sqlalchemy import Column, MetaData, Table, func, inspect, literal, select
 from sqlalchemy.sql.expression import ColumnOperators, and_, cte
 from sqlalchemy.types import String
 
+from metadata.generated.schema.entity.data.table import Table as OMTable
+from metadata.generated.schema.entity.data.table import TableType
 from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.orm.registry import Dialects
 from metadata.profiler.processor.runner import QueryRunner
@@ -31,7 +33,7 @@ logger = profiler_interface_registry_logger()
 
 COLUMN_COUNT = "columnCount"
 COLUMN_NAMES = "columnNames"
-ROW_COUNT = "rowCount"
+ROW_COUNT = Metrics.ROW_COUNT().name()
 SIZE_IN_BYTES = "sizeInBytes"
 CREATE_DATETIME = "createDateTime"
 
@@ -43,13 +45,16 @@ ERROR_MSG = (
 class AbstractTableMetricComputer(ABC):
     """Base table computer"""
 
-    def __init__(self, runner: QueryRunner, metrics: List[Metrics], conn_config):
+    def __init__(
+        self, runner: QueryRunner, metrics: List[Metrics], conn_config, entity: OMTable
+    ):
         """Instantiate base table computer"""
         self._runner = runner
         self._metrics = metrics
         self._conn_config = conn_config
         self._database = self._runner._session.get_bind().url.database
         self._table = self._runner.table
+        self._entity = entity
 
     @property
     def database(self):
@@ -141,16 +146,6 @@ class AbstractTableMetricComputer(ABC):
 class BaseTableMetricComputer(AbstractTableMetricComputer):
     """Base table computer"""
 
-    def _check_and_return(self, res):
-        """Check if the result is None and return the result or fallback
-
-        Args:
-            res (object): result
-        """
-        if res.rowCount is None:
-            return super().compute()
-        return res
-
     def compute(self):
         """Default compute behavior for table metrics"""
         return self.runner.select_first_from_table(
@@ -236,7 +231,9 @@ class OracleTableMetricComputer(BaseTableMetricComputer):
         )
 
         res = self.runner._session.execute(query).first()
-        if res.rowCount is None:
+        if res.rowCount is None or (
+            res.rowCount == 0 and self._entity.tableType == TableType.View
+        ):
             # if we don't have any row count, fallback to the base logic
             return super().compute()
         return res
@@ -263,7 +260,9 @@ class ClickHouseTableMetricComputer(BaseTableMetricComputer):
         )
 
         res = self.runner._session.execute(query).first()
-        if res.rowCount is None:
+        if res.rowCount is None or (
+            res.rowCount == 0 and self._entity.tableType == TableType.View
+        ):
             # if we don't have any row count, fallback to the base logic
             return super().compute()
         return res
@@ -307,7 +306,9 @@ class BigQueryTableMetricComputer(BaseTableMetricComputer):
         )
 
         res = self.runner._session.execute(query).first()
-        if res.rowCount is None:
+        if res.rowCount is None or (
+            res.rowCount == 0 and self._entity.tableType == TableType.View
+        ):
             # if we don't have any row count, fallback to the base logic
             return super().compute()
         return res
@@ -336,7 +337,9 @@ class BigQueryTableMetricComputer(BaseTableMetricComputer):
             where_clause,
         )
         res = self.runner._session.execute(query).first()
-        if res.rowCount is None:
+        if res.rowCount is None or (
+            res.rowCount == 0 and self._entity.tableType == TableType.View
+        ):
             # if we don't have any row count, fallback to the base logic
             return super().compute()
         return res
@@ -363,9 +366,16 @@ class MySQLTableMetricComputer(BaseTableMetricComputer):
         )
 
         res = self.runner._session.execute(query).first()
-        if res.rowCount is None:
+        if res.rowCount is None or (
+            res.rowCount == 0 and self._entity.tableType == TableType.View
+        ):
             # if we don't have any row count, fallback to the base logic
             return super().compute()
+        res = res._asdict()
+        # innodb row count is an estimate we need to patch the row count with COUNT(*)
+        # https://dev.mysql.com/doc/refman/8.3/en/information-schema-innodb-tablestats-table.html
+        row_count = self.runner.select_first_from_table(Metrics.ROW_COUNT().fn())
+        res.update({ROW_COUNT: row_count.rowCount})
         return res
 
 
@@ -390,7 +400,9 @@ class RedshiftTableMetricComputer(BaseTableMetricComputer):
             columns, self._build_table("svv_table_info", "pg_catalog"), where_clause
         )
         res = self.runner._session.execute(query).first()
-        if res.rowCount is None:
+        if res.rowCount is None or (
+            res.rowCount == 0 and self._entity.tableType == TableType.View
+        ):
             # if we don't have any row count, fallback to the base logic
             return super().compute()
         return res
@@ -400,9 +412,15 @@ class TableMetricComputer:
     """Table Metric Construct"""
 
     def __init__(
-        self, dialect: str, runner: QueryRunner, metrics: List[Metrics], conn_config
+        self,
+        dialect: str,
+        runner: QueryRunner,
+        metrics: List[Metrics],
+        conn_config,
+        entity: OMTable,
     ):
         """Instantiate table metric computer with a dialect computer"""
+        self._entity = entity
         self._dialect = dialect
         self._runner = runner
         self._metrics = metrics
@@ -413,6 +431,7 @@ class TableMetricComputer:
                 runner=self._runner,
                 metrics=self._metrics,
                 conn_config=self._conn_config,
+                entity=self._entity,
             )
         )