openmetadata-ingestion 1.5.0.0rc2__py3-none-any.whl → 1.5.2.0__py3-none-any.whl

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHierarchy.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-08-22T13:36:48+00:00
+#   timestamp: 2024-09-02T10:08:12+00:00
 
 from __future__ import annotations
 

metadata/ingestion/connections/builders.py

@@ -33,6 +33,9 @@ from metadata.generated.schema.entity.services.connections.database.common.iamAu
 from metadata.ingestion.connections.headers import inject_query_header_by_conn
 from metadata.ingestion.connections.secrets import connection_with_options_secrets
 from metadata.utils.constants import BUILDER_PASSWORD_ATTR
+from metadata.utils.logger import cli_logger
+
+logger = cli_logger()
 
 
 @connection_with_options_secrets
@@ -152,6 +155,9 @@ def _add_password(url: str, connection) -> str:
                         Region=connection.authType.awsConfig.awsRegion,
                     )
                 )
+    if not password:
+        logger.warning("No password has been provided in connection")
+        password = SecretStr("")
     url += f":{quote_plus(password.get_secret_value())}"
     return url
 

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -313,17 +313,18 @@ class TableauSource(DashboardServiceSource):
     @staticmethod
     def _get_data_model_column_fqn(
         data_model_entity: DashboardDataModel, column: str
-    ) -> Optional[str]:
+    ) -> Optional[List[str]]:
         """
         Get fqn of column if exist in table entity
         """
         if not data_model_entity:
             return None
+        columns = []
         for tbl_column in data_model_entity.columns:
             for child_column in tbl_column.children or []:
                 if column.lower() == child_column.name.root.lower():
-                    return child_column.fullyQualifiedName.root
-        return None
+                    columns.append(child_column.fullyQualifiedName.root)
+        return columns
 
     def _get_column_lineage(  # pylint: disable=arguments-differ
         self,
@@ -342,13 +343,15 @@ class TableauSource(DashboardServiceSource):
                     from_column = get_column_fqn(
                         table_entity=table_entity, column=column.name
                     )
-                    to_column = self._get_data_model_column_fqn(
+                    to_columns = self._get_data_model_column_fqn(
                         data_model_entity=data_model_entity,
                         column=column.id,
                     )
-                    column_lineage.append(
-                        ColumnLineage(fromColumns=[from_column], toColumn=to_column)
-                    )
+                    for to_column in to_columns:
+                        column_lineage.append(
+                            ColumnLineage(fromColumns=[from_column], toColumn=to_column)
+                        )
+            return column_lineage
         except Exception as exc:
             logger.debug(f"Error to get column lineage: {exc}")
             logger.debug(traceback.format_exc())

metadata/ingestion/source/database/athena/models.py

@@ -22,19 +22,19 @@ class QueryExecutionIdsResponse(BaseModel):
     QueryExecutionIds: Optional[List[str]] = None
 
 
-class Status(BaseModel):
+class AthenaStatus(BaseModel):
     State: Optional[str] = "FAILED"  # Default value
     SubmissionDateTime: Optional[datetime] = None
 
 
-class Statistics(BaseModel):
+class AthenaStatistics(BaseModel):
     TotalExecutionTimeInMillis: Optional[int] = None
 
 
 class AthenaQueryExecution(BaseModel):
     Query: Optional[str] = None
-    Statistics: Optional[Statistics] = None
-    Status: Optional[Status] = None
+    Statistics: Optional[AthenaStatistics] = None
+    Status: Optional[AthenaStatus] = None
 
 
 class AthenaQueryExecutionList(BaseModel):

metadata/ingestion/source/database/column_type_parser.py

@@ -148,7 +148,7 @@ class ColumnTypeParser:
         "MEDIUMBLOB": "MEDIUMBLOB",
         "MEDIUMINT": "INT",
         "MEDIUMTEXT": "MEDIUMTEXT",
-        "MONEY": "NUMBER",
+        "MONEY": "MONEY",
         "NCHAR": "CHAR",
         "NTEXT": "NTEXT",
         "NULL": "NULL",

metadata/ingestion/source/database/dbt/metadata.py

@@ -162,7 +162,7 @@ class DbtSource(DbtServiceSource):
                     return owner
 
                 # If owner is not found, try to find the owner in OMD using email
-                owner = self.metadata.get_reference_by_email(name=dbt_owner)
+                owner = self.metadata.get_reference_by_email(email=dbt_owner)
 
                 if not owner:
                     logger.warning(

metadata/ingestion/source/database/postgres/converter_orm.py

@@ -0,0 +1,52 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Converter logic to transform an OpenMetadata Table Entity for Redshift
+to an SQLAlchemy ORM class.
+"""
+
+from typing import Dict, Set
+
+from sqlalchemy.sql.sqltypes import TypeEngine
+
+from metadata.generated.schema.entity.data.table import Column, DataType
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseServiceType,
+)
+from metadata.ingestion.source.database.postgres.types.money import PostgresMoney
+from metadata.profiler.orm.converter.common import CommonMapTypes
+from metadata.profiler.orm.converter.converter_registry import converter_registry
+
+_CUSTOM_TYPE_MAP = {DataType.MONEY: PostgresMoney}
+_CUSTOM_TYPE_MAP_reversed = {v: {k} for k, v in _CUSTOM_TYPE_MAP.items()}
+
+
+class PostgrestMapTypes(CommonMapTypes):
+    """Postgres type mapper, inherits from CommonMapTypes"""
+
+    def return_custom_type(self, col: Column, table_service_type):
+        return _CUSTOM_TYPE_MAP.get(
+            col.dataType, super().return_custom_type(col, table_service_type)
+        )
+
+    @staticmethod
+    def map_sqa_to_om_types() -> Dict[TypeEngine, Set[DataType]]:
+        """returns an ORM type"""
+        # pylint: disable=import-outside-toplevel
+
+        return {
+            **CommonMapTypes.map_sqa_to_om_types(),
+            **_CUSTOM_TYPE_MAP_reversed,
+        }
+
+
+converter_registry[DatabaseServiceType.Postgres] = PostgrestMapTypes

metadata/ingestion/source/database/postgres/metadata.py

@@ -65,6 +65,7 @@ from metadata.ingestion.source.database.postgres.utils import (
 )
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_database
+from metadata.utils.importer import import_side_effects
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import (
     get_all_table_comments,
@@ -76,6 +77,11 @@ from metadata.utils.sqlalchemy_utils import (
 )
 from metadata.utils.tag_utils import get_ometa_tag_and_classification
 
+import_side_effects(
+    "metadata.ingestion.source.database.postgres.converter_orm",
+    "metadata.ingestion.source.database.postgres.metrics",
+)
+
 TableKey = namedtuple("TableKey", ["schema", "table_name"])
 
 logger = ingestion_logger()

metadata/ingestion/source/database/postgres/metrics.py

@@ -0,0 +1,42 @@
+from sqlalchemy.ext.compiler import compiles
+
+from metadata.ingestion.source.database.postgres.types.money import PostgresMoney
+from metadata.profiler.metrics.static.mean import AvgFn
+from metadata.profiler.metrics.static.stddev import StdDevFn
+from metadata.profiler.orm.functions.median import MedianFn
+from metadata.profiler.orm.registry import Dialects
+
+
+@compiles(AvgFn, Dialects.Postgres)
+def avg(element, compiler, **kw):
+    """
+    Cast to decimal to get around potential integer overflow error
+    """
+    proc = compiler.process(element.clauses, **kw)
+    if isinstance(list(element.clauses)[0].type, PostgresMoney):
+        return f"{element.name}({PostgresMoney.compile_as_float(proc)})"
+    return f"{element.name}({proc})"
+
+
+@compiles(StdDevFn, Dialects.Postgres)
+def stddev(element, compiler, **kw):
+    """Returns stdv for clickhouse database and handle empty tables.
+    If table is empty, clickhouse returns NaN.
+    """
+    proc = compiler.process(element.clauses, **kw)
+    if isinstance(list(element.clauses)[0].type, PostgresMoney):
+        return f"STDDEV_POP({PostgresMoney.compile_as_float(proc)})"
+    return f"STDDEV_POP({proc})"
+
+
+@compiles(MedianFn, Dialects.Postgres)
+def median(elements, compiler, **kwargs):  # pylint: disable=unused-argument
+    col, _, percentile = [
+        compiler.process(element, **kwargs) for element in elements.clauses
+    ]
+    if isinstance(list(elements.clauses)[0], PostgresMoney):
+        return "percentile_cont(%.2f) WITHIN GROUP (ORDER BY %s ASC)" % (
+            percentile,
+            PostgresMoney.compile_as_float(col),
+        )
+    return MedianFn.default_fn(elements, compiler, **kwargs)

metadata/ingestion/source/database/postgres/types/money.py

@@ -0,0 +1,37 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Custom sqlalchemy type for Postgres MONEY type
+"""
+
+
+from sqlalchemy.dialects.postgresql import MONEY
+from sqlalchemy.sql.sqltypes import String
+
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class PostgresMoney(MONEY):
+    """Wrapper for Postgres MONEY type"""
+
+    impl = String
+    cache_ok = True
+
+    @property
+    def quantifiable(self):
+        return True
+
+    @staticmethod
+    def compile_as_float(expr: str) -> str:
+        return f"(({expr})::numeric::float8)"

metadata/ingestion/source/database/redshift/connection.py

@@ -12,6 +12,7 @@
 """
 Source connection handler
 """
+from functools import partial
 from typing import Optional
 
 from sqlalchemy.engine import Engine
@@ -27,8 +28,15 @@ from metadata.ingestion.connections.builders import (
     get_connection_args_common,
     get_connection_url_common,
 )
-from metadata.ingestion.connections.test_connections import test_connection_db_common
+from metadata.ingestion.connections.test_connections import (
+    SourceConnectionException,
+    execute_inspector_func,
+    test_connection_engine_step,
+    test_connection_steps,
+    test_query,
+)
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.connections import kill_active_connections
 from metadata.ingestion.source.database.redshift.queries import (
     REDSHIFT_GET_DATABASE_NAMES,
     REDSHIFT_TEST_GET_QUERIES,
@@ -57,15 +65,35 @@ def test_connection(
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    queries = {
-        "GetQueries": REDSHIFT_TEST_GET_QUERIES,
-        "GetDatabases": REDSHIFT_GET_DATABASE_NAMES,
-        "GetPartitionTableDetails": REDSHIFT_TEST_PARTITION_DETAILS,
+
+    def test_get_queries_permissions(engine_: Engine):
+        """Check if we have the right permissions to list queries"""
+        with engine_.connect() as conn:
+            res = conn.execute(REDSHIFT_TEST_GET_QUERIES).fetchone()
+            if not all(res):
+                raise SourceConnectionException(
+                    f"We don't have the right permissions to list queries - {res}"
+                )
+
+    test_fn = {
+        "CheckAccess": partial(test_connection_engine_step, engine),
+        "GetSchemas": partial(execute_inspector_func, engine, "get_schema_names"),
+        "GetTables": partial(execute_inspector_func, engine, "get_table_names"),
+        "GetViews": partial(execute_inspector_func, engine, "get_view_names"),
+        "GetQueries": partial(test_get_queries_permissions, engine),
+        "GetDatabases": partial(
+            test_query, statement=REDSHIFT_GET_DATABASE_NAMES, engine=engine
+        ),
+        "GetPartitionTableDetails": partial(
+            test_query, statement=REDSHIFT_TEST_PARTITION_DETAILS, engine=engine
+        ),
     }
-    test_connection_db_common(
+
+    test_connection_steps(
         metadata=metadata,
-        engine=engine,
-        service_connection=service_connection,
+        test_fn=test_fn,
+        service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
-        queries=queries,
     )
+
+    kill_active_connections(engine)

metadata/ingestion/source/database/redshift/queries.py

@@ -222,11 +222,10 @@ SELECT datname FROM pg_database
 """
 
 REDSHIFT_TEST_GET_QUERIES = """
-(select 1 from pg_catalog.svv_table_info limit 1)
-UNION
-(select 1 from pg_catalog.stl_querytext limit 1)
-UNION
-(select 1 from pg_catalog.stl_query limit 1)
+SELECT 
+    has_table_privilege('svv_table_info', 'SELECT') as can_access_svv_table_info,
+    has_table_privilege('stl_querytext', 'SELECT') as can_access_stl_querytext,
+    has_table_privilege('stl_query', 'SELECT') as can_access_stl_query;
 """