openmetadata-ingestion 1.5.4.1__py3-none-any.whl → 1.5.6.0__py3-none-any.whl

metadata/generated/schema/type/entityHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHierarchy.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-09-19T13:48:25+00:00
+#   timestamp: 2024-10-03T13:02:21+00:00
 
 from __future__ import annotations
 

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -22,13 +22,14 @@ from pydantic import BaseModel
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
-from metadata.generated.schema.type.entityLineage import EntitiesEdge
+from metadata.generated.schema.type.basic import FullyQualifiedEntityName, Uuid
+from metadata.generated.schema.type.entityLineage import ColumnLineage, EntitiesEdge
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.lineage.parser import LINEAGE_PARSING_TIMEOUT
 from metadata.ingestion.models.patch_request import build_patch
 from metadata.ingestion.ometa.client import REST, APIError
-from metadata.ingestion.ometa.utils import get_entity_type, quote
+from metadata.ingestion.ometa.utils import get_entity_type, model_str, quote
 from metadata.utils.logger import ometa_logger
 from metadata.utils.lru_cache import LRU_CACHE_SIZE, LRUCache
 
@@ -138,6 +139,18 @@ class OMetaLineageMixin(Generic[T]):
                         )
                     )
 
+                    serialized_col_details = []
+                    for col_lin in data.edge.lineageDetails.columnsLineage or []:
+                        serialized_col_details.append(ColumnLineage(**col_lin))
+                    data.edge.lineageDetails.columnsLineage = serialized_col_details
+
+                    serialized_col_details_og = []
+                    for col_lin in original.edge.lineageDetails.columnsLineage or []:
+                        serialized_col_details_og.append(ColumnLineage(**col_lin))
+                    original.edge.lineageDetails.columnsLineage = (
+                        serialized_col_details_og
+                    )
+
                     # Keep the pipeline information from the original
                     # lineage if available
                     if (
@@ -248,7 +261,7 @@ class OMetaLineageMixin(Generic[T]):
     def get_lineage_by_id(
         self,
         entity: Union[Type[T], str],
-        entity_id: str,
+        entity_id: Union[str, Uuid],
         up_depth: int = 1,
         down_depth: int = 1,
     ) -> Optional[Dict[str, Any]]:
@@ -260,13 +273,16 @@ class OMetaLineageMixin(Generic[T]):
         :param down_depth: Downstream depth of lineage (default=1, min=0, max=3)
         """
         return self._get_lineage(
-            entity=entity, path=entity_id, up_depth=up_depth, down_depth=down_depth
+            entity=entity,
+            path=model_str(entity_id),
+            up_depth=up_depth,
+            down_depth=down_depth,
         )
 
     def get_lineage_by_name(
         self,
         entity: Union[Type[T], str],
-        fqn: str,
+        fqn: Union[str, FullyQualifiedEntityName],
         up_depth: int = 1,
         down_depth: int = 1,
     ) -> Optional[Dict[str, Any]]:
@@ -279,7 +295,7 @@ class OMetaLineageMixin(Generic[T]):
         """
         return self._get_lineage(
             entity=entity,
-            path=f"name/{quote(fqn)}",
+            path=f"name/{quote(model_str(fqn))}",
             up_depth=up_depth,
             down_depth=down_depth,
         )

metadata/ingestion/sink/metadata_rest.py

@@ -565,9 +565,9 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
             f"Successfully ingested profile metrics for {record.table.fullyQualifiedName.root}"
         )
 
-        if record.sample_data:
+        if record.sample_data and record.sample_data.store:
             table_data = self.metadata.ingest_table_sample_data(
-                table=record.table, sample_data=record.sample_data
+                table=record.table, sample_data=record.sample_data.data
             )
             if not table_data:
                 self.status.failed(

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -544,12 +544,12 @@ class PowerbiSource(DashboardServiceSource):
                         to_entity=report_entity, from_entity=datamodel_entity
                     )
 
-                    # create the lineage between table and datamodel
-                    yield from self.create_table_datamodel_lineage(
-                        db_service_name=db_service_name,
-                        tables=dataset.tables,
-                        datamodel_entity=datamodel_entity,
-                    )
+                    for table in dataset.tables or []:
+                        yield self._get_table_and_datamodel_lineage(
+                            db_service_name=db_service_name,
+                            table=table,
+                            datamodel_entity=datamodel_entity,
+                        )
 
                     # create the lineage between table and datamodel using the pbit files
                     if self.client.file_client:
@@ -678,20 +678,6 @@ class PowerbiSource(DashboardServiceSource):
                 )
             )
 
-    def create_table_datamodel_lineage(
-        self,
-        db_service_name: str,
-        tables: Optional[List[PowerBiTable]],
-        datamodel_entity: Optional[DashboardDataModel],
-    ) -> Iterable[Either[CreateDashboardRequest]]:
-        """Method to create lineage between table and datamodels"""
-        for table in tables or []:
-            yield self._get_table_and_datamodel_lineage(
-                db_service_name=db_service_name,
-                table=table,
-                datamodel_entity=datamodel_entity,
-            )
-
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: Union[PowerBIDashboard, PowerBIReport],

metadata/ingestion/source/database/bigquery/connection.py

@@ -17,7 +17,6 @@ from datetime import datetime
 from functools import partial
 from typing import Optional
 
-from google.api_core.exceptions import NotFound
 from google.cloud.datacatalog_v1 import PolicyTagManagerClient
 from sqlalchemy.engine import Engine
 
@@ -150,8 +149,8 @@ def test_connection(
         test_fn = {
             "CheckAccess": partial(test_connection_engine_step, engine),
             "GetSchemas": partial(execute_inspector_func, engine, "get_schema_names"),
-            "GetTables": partial(get_table_view_names, engine),
-            "GetViews": partial(get_table_view_names, engine),
+            "GetTables": partial(execute_inspector_func, engine, "get_table_names"),
+            "GetViews": partial(execute_inspector_func, engine, "get_view_names"),
             "GetTags": test_tags,
             "GetQueries": partial(
                 test_query,
@@ -171,28 +170,3 @@ def test_connection(
         )
 
     test_connection_inner(engine)
-
-
-def get_table_view_names(connection, schema=None):
-    with connection.connect() as conn:
-        current_schema = schema
-        client = conn.connection._client
-        item_types = ["TABLE", "EXTERNAL", "VIEW", "MATERIALIZED_VIEW"]
-        datasets = client.list_datasets()
-        result = []
-        for dataset in datasets:
-            if current_schema is not None and current_schema != dataset.dataset_id:
-                continue
-
-            try:
-                tables = client.list_tables(dataset.reference, page_size=1)
-                for table in tables:
-                    if table.table_type in item_types:
-                        break
-            except NotFound:
-                # It's possible that the dataset was deleted between when we
-                # fetched the list of datasets and when we try to list the
-                # tables from it. See:
-                # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/105
-                pass
-        return result

metadata/ingestion/source/database/dbt/dbt_service.py

@@ -160,7 +160,7 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
         # This step is necessary as the manifest file may not always adhere to the schema definition
         # and the presence of other nodes can hinder the ingestion process from progressing any further.
         # Therefore, we are only retaining the essential data for further processing.
-        required_manifest_keys = ["nodes", "sources", "metadata"]
+        required_manifest_keys = {"nodes", "sources", "metadata"}
         manifest_dict.update(
             {
                 key: {}
@@ -169,6 +169,52 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
             }
         )
 
+        required_nodes_keys = {
+            "schema_",
+            "schema",
+            "name",
+            "resource_type",
+            "path",
+            "unique_id",
+            "fqn",
+            "alias",
+            "checksum",
+            "config",
+            "column_name",
+            "test_metadata",
+            "original_file_path",
+            "root_path",
+            "database",
+            "tags",
+            "description",
+            "columns",
+            "meta",
+            "owner",
+            "created_at",
+            "group",
+            "sources",
+            "compiled",
+            "docs",
+            "version",
+            "latest_version",
+            "package_name",
+            "depends_on",
+            "compiled_code",
+            "compiled_sql",
+            "raw_code",
+            "raw_sql",
+            "language",
+        }
+
+        for node, value in manifest_dict.get(  # pylint: disable=unused-variable
+            "nodes"
+        ).items():
+            keys_to_delete = [
+                key for key in value if key.lower() not in required_nodes_keys
+            ]
+            for key in keys_to_delete:
+                del value[key]
+
     def get_dbt_files(self) -> Iterable[DbtFiles]:
         dbt_files = get_dbt_details(self.source_config.dbtConfigSource)
         for dbt_file in dbt_files:

metadata/ingestion/source/database/iceberg/catalog/rest.py

@@ -49,7 +49,7 @@ class IcebergRestCatalog(IcebergCatalogBase):
 
         parameters = {
             "warehouse": catalog.warehouseLocation,
-            "uri": catalog.connection.uri,
+            "uri": str(catalog.connection.uri),
             "credential": credential,
             "token": catalog.connection.token.get_secret_value()
             if catalog.connection.token

metadata/ingestion/source/database/redshift/connection.py

@@ -16,7 +16,6 @@ from functools import partial
 from typing import Optional
 
 from sqlalchemy.engine import Engine
-from sqlalchemy.sql import text
 
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
@@ -39,7 +38,6 @@ from metadata.ingestion.connections.test_connections import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import kill_active_connections
 from metadata.ingestion.source.database.redshift.queries import (
-    REDSHIFT_GET_ALL_RELATIONS,
     REDSHIFT_GET_DATABASE_NAMES,
     REDSHIFT_TEST_GET_QUERIES,
     REDSHIFT_TEST_PARTITION_DETAILS,
@@ -67,11 +65,6 @@ def test_connection(
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    table_and_view_query = text(
-        REDSHIFT_GET_ALL_RELATIONS.format(
-            schema_clause="", table_clause="", limit_clause="LIMIT 1"
-        )
-    )
 
     def test_get_queries_permissions(engine_: Engine):
         """Check if we have the right permissions to list queries"""
@@ -85,8 +78,8 @@ def test_connection(
     test_fn = {
         "CheckAccess": partial(test_connection_engine_step, engine),
         "GetSchemas": partial(execute_inspector_func, engine, "get_schema_names"),
-        "GetTables": partial(test_query, statement=table_and_view_query, engine=engine),
-        "GetViews": partial(test_query, statement=table_and_view_query, engine=engine),
+        "GetTables": partial(execute_inspector_func, engine, "get_table_names"),
+        "GetViews": partial(execute_inspector_func, engine, "get_view_names"),
         "GetQueries": partial(test_get_queries_permissions, engine),
         "GetDatabases": partial(
             test_query, statement=REDSHIFT_GET_DATABASE_NAMES, engine=engine

metadata/ingestion/source/database/redshift/queries.py

@@ -236,7 +236,7 @@ REDSHIFT_TEST_PARTITION_DETAILS = "select * from SVV_TABLE_INFO limit 1"
 # hence we are appending "create view <schema>.<table> as " to select query
 # to generate the column level lineage
 REDSHIFT_GET_ALL_RELATIONS = """
-    (SELECT
+    SELECT
         c.relkind,
         n.oid as "schema_oid",
         n.nspname as "schema",
@@ -255,9 +255,8 @@ REDSHIFT_GET_ALL_RELATIONS = """
             JOIN pg_catalog.pg_user u ON u.usesysid = c.relowner
     WHERE c.relkind IN ('r', 'v', 'm', 'S', 'f')
         AND n.nspname !~ '^pg_' {schema_clause} {table_clause}
-        {limit_clause})
     UNION
-    (SELECT
+    SELECT
         'r' AS "relkind",
         s.esoid AS "schema_oid",
         s.schemaname AS "schema",
@@ -273,8 +272,7 @@ REDSHIFT_GET_ALL_RELATIONS = """
         JOIN svv_external_schemas s ON s.schemaname = t.schemaname
         JOIN pg_catalog.pg_user u ON u.usesysid = s.esowner
     where 1 {schema_clause} {table_clause}
-    ORDER BY "relkind", "schema_oid", "schema"
-    {limit_clause});
+    ORDER BY "relkind", "schema_oid", "schema";
     """
 
 

metadata/ingestion/source/database/redshift/utils.py

@@ -386,7 +386,7 @@ def _get_all_relation_info(self, connection, **kw):  # pylint: disable=unused-ar
     result = connection.execute(
         sa.text(
             REDSHIFT_GET_ALL_RELATIONS.format(
-                schema_clause=schema_clause, table_clause=table_clause, limit_clause=""
+                schema_clause=schema_clause, table_clause=table_clause
             )
         )
     )

metadata/ingestion/source/database/salesforce/connection.py

@@ -15,7 +15,6 @@ Source connection handler
 from typing import Optional
 
 from simple_salesforce.api import Salesforce
-from sqlalchemy.engine import Engine
 
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
@@ -27,14 +26,17 @@ from metadata.ingestion.connections.test_connections import test_connection_step
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 
 
-def get_connection(connection: SalesforceConnection) -> Engine:
+def get_connection(connection: SalesforceConnection) -> Salesforce:
     """
     Create connection
     """
     return Salesforce(
         username=connection.username,
         password=connection.password.get_secret_value(),
-        security_token=connection.securityToken.get_secret_value(),
+        security_token=connection.securityToken.get_secret_value()
+        if connection.securityToken
+        else "",
+        organizationId=connection.organizationId if connection.organizationId else "",
         domain=connection.salesforceDomain,
         version=connection.salesforceApiVersion,
         **connection.connectionArguments.root if connection.connectionArguments else {},

metadata/ingestion/source/database/snowflake/connection.py

@@ -45,7 +45,6 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_TEST_FETCH_TAG,
     SNOWFLAKE_TEST_GET_QUERIES,
     SNOWFLAKE_TEST_GET_TABLES,
-    SNOWFLAKE_TEST_GET_VIEWS,
 )
 from metadata.utils.logger import ingestion_logger
 
@@ -177,11 +176,7 @@ def test_connection(
             statement=SNOWFLAKE_TEST_GET_TABLES,
             engine_wrapper=engine_wrapper,
         ),
-        "GetViews": partial(
-            test_table_query,
-            statement=SNOWFLAKE_TEST_GET_VIEWS,
-            engine_wrapper=engine_wrapper,
-        ),
+        "GetViews": partial(execute_inspector_func, engine_wrapper, "get_view_names"),
         "GetQueries": partial(
             test_query, statement=SNOWFLAKE_TEST_GET_QUERIES, engine=engine
         ),

metadata/ingestion/source/database/snowflake/queries.py

@@ -245,10 +245,6 @@ SNOWFLAKE_TEST_GET_TABLES = """
 SELECT TABLE_NAME FROM "{database_name}".information_schema.tables LIMIT 1
 """
 
-SNOWFLAKE_TEST_GET_VIEWS = """
-SELECT TABLE_NAME FROM "{database_name}".information_schema.views LIMIT 1
-"""
-
 SNOWFLAKE_GET_DATABASES = "SHOW DATABASES"