openmetadata-ingestion 1.6.6.6__py3-none-any.whl → 1.6.7.1__py3-none-any.whl

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-03-29T04:07:20+00:00
+#   timestamp: 2025-04-02T12:35:09+00:00
 
 from __future__ import annotations
 

metadata/ingestion/models/custom_basemodel_validation.py

@@ -20,22 +20,27 @@ logger = logging.getLogger("metadata")
 RESTRICTED_KEYWORDS = ["::", ">"]
 RESERVED_COLON_KEYWORD = "__reserved__colon__"
 RESERVED_ARROW_KEYWORD = "__reserved__arrow__"
+RESERVED_QUOTE_KEYWORD = "__reserved__quote__"
 
 CREATE_ADJACENT_MODELS = {"ProfilerResponse", "SampleData"}
 NAME_FIELDS = {"EntityName", "str", "ColumnName", "TableData"}
-FETCH_MODELS = {"Table", "CustomColumnName"}
+FETCH_MODELS = {"Table", "CustomColumnName", "DashboardDataModel"}
 FIELD_NAMES = {"name", "columns", "root"}
 
 
 def revert_separators(value):
-    return value.replace(RESERVED_COLON_KEYWORD, "::").replace(
-        RESERVED_ARROW_KEYWORD, ">"
+    return (
+        value.replace(RESERVED_COLON_KEYWORD, "::")
+        .replace(RESERVED_ARROW_KEYWORD, ">")
+        .replace(RESERVED_QUOTE_KEYWORD, '"')
     )
 
 
 def replace_separators(value):
-    return value.replace("::", RESERVED_COLON_KEYWORD).replace(
-        ">", RESERVED_ARROW_KEYWORD
+    return (
+        value.replace("::", RESERVED_COLON_KEYWORD)
+        .replace(">", RESERVED_ARROW_KEYWORD)
+        .replace('"', RESERVED_QUOTE_KEYWORD)
     )
 
 

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -171,9 +171,13 @@ class OMetaLineageMixin(Generic[T]):
 
         except APIError as err:
             logger.debug(traceback.format_exc())
-            error = f"Error {err.status_code} trying to PUT lineage for {data.model_dump_json()}: {str(err)}"
-            logger.error(error)
-            return {"error": error}
+            logger.error(
+                "Error %s trying to PUT lineage for %s: %s",
+                err.status_code,
+                data.model_dump_json(),
+                str(err),
+            )
+            raise err
 
         from_entity_lineage = self.get_lineage_by_id(
             data.edge.fromEntity.type, str(data.edge.fromEntity.id.root)
@@ -393,10 +397,6 @@ class OMetaLineageMixin(Generic[T]):
                     resp = self.add_lineage(
                         lineage_request.right, check_patch=check_patch
                     )
-                    if resp.get("error"):
-                        logger.error(resp["error"])
-                        continue
-
                     entity_name = resp.get("entity", {}).get("name")
                     for node in resp.get("nodes", []):
                         logger.info(

metadata/ingestion/ometa/mixins/table_mixin.py

@@ -13,6 +13,7 @@ Mixin class containing Table specific methods
 
 To be used by OpenMetadata class
 """
+import base64
 import traceback
 from typing import List, Optional, Type, TypeVar
 
@@ -56,6 +57,7 @@ class OMetaTableMixin:
 
     client: REST
 
+    # pylint: disable=too-many-nested-blocks
     def ingest_table_sample_data(
         self, table: Table, sample_data: TableData
     ) -> Optional[TableData]:
@@ -67,9 +69,37 @@ class OMetaTableMixin:
         """
         resp = None
         try:
+            # Pre-process sample data to handle binary/non-UTF-8 data before serialization
+            if sample_data and sample_data.rows:
+
+                for row in sample_data.rows:
+                    if not row:
+                        continue
+                    for col_idx, value in enumerate(row):
+                        # Handle binary data explicitly
+                        if isinstance(value, bytes):
+                            # Convert binary data to Base64-encoded string
+                            try:
+                                row[
+                                    col_idx
+                                ] = f"[base64]{base64.b64encode(value).decode('ascii', errors='ignore')}"
+                            except Exception as _:
+                                row[col_idx] = f"[binary]{value}"
+
+            try:
+                data = sample_data.model_dump_json()
+            except Exception as _:
+                logger.debug(traceback.format_exc())
+                logger.warning(
+                    f"Error serializing sample data for {table.fullyQualifiedName.root}"
+                    " please check if the data is valid"
+                )
+                return None
+
+            # Now safely serialize to JSON
             resp = self.client.put(
                 f"{self.get_suffix(Table)}/{table.id.root}/sampleData",
-                data=sample_data.model_dump_json(),
+                data=data,
             )
         except Exception as exc:
             logger.debug(traceback.format_exc())

metadata/ingestion/sink/metadata_rest.py

@@ -250,13 +250,6 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
     @_run_dispatch.register
     def write_lineage(self, add_lineage: AddLineageRequest) -> Either[Dict[str, Any]]:
         created_lineage = self.metadata.add_lineage(add_lineage, check_patch=True)
-        if created_lineage.get("error"):
-            return Either(
-                left=StackTraceError(
-                    name="AddLineageRequestError", error=created_lineage["error"]
-                )
-            )
-
         return Either(right=created_lineage["entity"]["fullyQualifiedName"])
 
     @_run_dispatch.register

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -14,6 +14,7 @@ REST Auth & Client for PowerBi
 import json
 import math
 import traceback
+from copy import deepcopy
 from time import sleep
 from typing import List, Optional, Tuple
 
@@ -349,7 +350,13 @@ class PowerBiApiClient:
             api_url = f"/myorg/{admin}groups"
             entities_per_page = self.pagination_entity_per_page
             failed_indexes = []
-            params_data = GETGROUPS_DEFAULT_PARAMS
+            parsed_filter_query = None
+            if filter_pattern:
+                parsed_filter_query = self.create_filter_query(filter_pattern)
+                logger.debug(f"Filter query applied = {parsed_filter_query}")
+            params_data = deepcopy(GETGROUPS_DEFAULT_PARAMS)
+            if parsed_filter_query:
+                params_data["$filter"] = parsed_filter_query
             response = self.client.get(api_url, data=params_data)
             if (
                 not response
@@ -372,14 +379,18 @@ class PowerBiApiClient:
                     logger.warning(f"Error processing GetGroups response: {exc}")
                     count = 0
             indexes = math.ceil(count / entities_per_page)
+            logger.debug(
+                f"Total {count} workspaces found, Will run {indexes} iterations fetching"
+                f" maximum {entities_per_page} workspaces in a single iteration"
+            )
             workspaces = []
             for index in range(indexes):
                 params_data = {
                     "$top": str(entities_per_page),
                     "$skip": str(index * entities_per_page),
                 }
-                if filter_pattern:
-                    params_data["$filter"] = self.create_filter_query(filter_pattern)
+                if parsed_filter_query:
+                    params_data["$filter"] = parsed_filter_query
 
                 response = self.client.get(api_url, data=params_data)
                 if (

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -14,6 +14,9 @@ import re
 import traceback
 from typing import Any, Iterable, List, Optional, Union
 
+from pydantic import EmailStr
+from pydantic_core import PydanticCustomError
+
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
 from metadata.generated.schema.api.data.createDashboardDataModel import (
@@ -48,6 +51,7 @@ from metadata.generated.schema.type.basic import (
     Markdown,
     SourceUrl,
 )
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
@@ -487,6 +491,7 @@ class PowerbiSource(DashboardServiceSource):
                     "name": table.name,
                     "displayName": table.name,
                     "description": table.description,
+                    "children": [],
                 }
                 child_columns = self._get_child_columns(table=table)
                 child_measures = self._get_child_measures(table=table)
@@ -533,6 +538,7 @@ class PowerbiSource(DashboardServiceSource):
                         serviceType=DashboardServiceType.PowerBI.value,
                         columns=self._get_column_info(dataset),
                         project=self.get_project_name(dashboard_details=dataset),
+                        owners=self.get_owner_ref(dashboard_details=dataset),
                     )
                     yield Either(right=data_model_request)
                     self.register_record_datamodel(datamodel_request=data_model_request)
@@ -940,3 +946,44 @@ class PowerbiSource(DashboardServiceSource):
                 f"Error fetching project name for {dashboard_details.id}: {exc}"
             )
         return None
+
+    def get_owner_ref(  # pylint: disable=unused-argument, useless-return
+        self, dashboard_details: Any
+    ) -> Optional[EntityReferenceList]:
+        """
+        Method to process the dashboard owners
+        """
+        try:
+            owner_ref_list = []  # to assign multiple owners to entity if they exist
+            for owner in dashboard_details.users or []:
+                if owner.email:
+                    owner_ref = None
+                    try:
+                        owner_email = EmailStr._validate(owner.email)
+                        owner_ref = self.metadata.get_reference_by_email(
+                            owner_email.lower()
+                        )
+                    except PydanticCustomError:
+                        logger.warning(
+                            f"Could not fetch owner data for email: {owner.email}"
+                        )
+                        if owner.displayName:
+                            owner_ref = self.metadata.get_reference_by_name(
+                                name=owner.displayName
+                            )
+                    except Exception as err:
+                        logger.warning(
+                            f"Error processing current owner data in {dashboard_details.id}: {err}"
+                        )
+                    if owner_ref:
+                        owner_ref_list.append(owner_ref.root[0])
+            if len(owner_ref_list) > 0:
+                logger.debug(
+                    f"Successfully fetched owners data for {dashboard_details.id}"
+                )
+                return EntityReferenceList(root=owner_ref_list)
+            return None
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not fetch owner data due to {err}")
+        return None

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -32,6 +32,15 @@ class Tile(BaseModel):
     reportId: Optional[str] = None
 
 
+class PowerBIUser(BaseModel):
+    """
+    PowerBI User Model
+    """
+
+    displayName: Optional[str] = None
+    email: Optional[str] = Field(alias="emailAddress", default=None)
+
+
 class PowerBIDashboard(BaseModel):
     """
     PowerBI PowerBIDashboard Model
@@ -43,6 +52,7 @@ class PowerBIDashboard(BaseModel):
     webUrl: Optional[str] = None
     embedUrl: Optional[str] = None
     tiles: Optional[List[Tile]] = []
+    users: Optional[List[PowerBIUser]] = []
 
 
 class PowerBIReport(BaseModel):
@@ -54,6 +64,7 @@ class PowerBIReport(BaseModel):
     id: str
     name: str
     datasetId: Optional[str] = None
+    users: Optional[List[PowerBIUser]] = []
 
 
 class DashboardsResponse(BaseModel):
@@ -162,6 +173,7 @@ class Dataset(BaseModel):
     name: str
     tables: Optional[List[PowerBiTable]] = []
     description: Optional[str] = None
+    users: Optional[List[PowerBIUser]] = []
 
 
 class DatasetResponse(BaseModel):

metadata/ingestion/source/database/db2/service_spec.py

@@ -1,4 +1,3 @@
-from metadata.ingestion.source.database.db2.lineage import Db2LineageSource
 from metadata.ingestion.source.database.db2.metadata import Db2Source
 from metadata.profiler.interface.sqlalchemy.db2.profiler_interface import (
     DB2ProfilerInterface,
@@ -6,7 +5,5 @@ from metadata.profiler.interface.sqlalchemy.db2.profiler_interface import (
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
 ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=Db2Source,
-    profiler_class=DB2ProfilerInterface,
-    lineage_source_class=Db2LineageSource,
+    metadata_source_class=Db2Source, profiler_class=DB2ProfilerInterface
 )

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -353,40 +353,103 @@ def _(config: DbtS3Config):
 def _(config: DbtGcsConfig):
     try:
         bucket_name, prefix = get_dbt_prefix_config(config)
-        from google.cloud import storage  # pylint: disable=import-outside-toplevel
+        # pylint: disable=import-outside-toplevel
+        from google.auth.exceptions import DefaultCredentialsError, GoogleAuthError
+        from google.cloud import storage
 
-        set_google_credentials(gcp_credentials=config.dbtSecurityConfig)
+        try:
+            set_google_credentials(
+                gcp_credentials=config.dbtSecurityConfig, single_project=True
+            )
+        except (ValueError, GoogleAuthError) as cred_exc:
+            logger.error(
+                f"Failed to set Google Cloud credentials: {str(cred_exc)}. "
+                "Please ensure your credentials are properly formatted and valid."
+            )
+            raise DBTConfigException(
+                "Invalid Google Cloud credentials. Please check the format and validity of your credentials."
+            ) from cred_exc
+
+        try:
+            client = storage.Client()
+        except DefaultCredentialsError as client_exc:
+            logger.error(
+                f"Failed to create Google Cloud Storage client: {str(client_exc)}. "
+                "Please ensure you have valid credentials configured."
+            )
+            raise DBTConfigException(
+                "Failed to initialize Google Cloud Storage client. Please verify your credentials."
+            ) from client_exc
 
-        client = storage.Client()
         if not bucket_name:
-            buckets = client.list_buckets()
+            try:
+                buckets = client.list_buckets()
+            except Exception as bucket_exc:
+                logger.error(f"Failed to list GCS buckets: {str(bucket_exc)}")
+                raise DBTConfigException(
+                    "Unable to list GCS buckets. Please check your permissions and credentials."
+                ) from bucket_exc
         else:
-            buckets = [client.get_bucket(bucket_name)]
+            try:
+                buckets = [client.get_bucket(bucket_name)]
+            except Exception as bucket_exc:
+                logger.error(
+                    f"Failed to access GCS bucket {bucket_name}: {str(bucket_exc)}"
+                )
+                raise DBTConfigException(
+                    f"Unable to access GCS bucket {bucket_name}."
+                    "Please verify the bucket exists and you have proper permissions."
+                ) from bucket_exc
+
         for bucket in buckets:
-            if prefix:
-                obj_list = client.list_blobs(bucket.name, prefix=prefix)
-            else:
-                obj_list = client.list_blobs(bucket.name)
+            try:
+                obj_list = client.list_blobs(
+                    bucket.name, prefix=prefix if prefix else None
+                )
 
-            yield from download_dbt_files(
-                blob_grouped_by_directory=get_blobs_grouped_by_dir(
-                    blobs=[blob.name for blob in obj_list]
-                ),
-                config=config,
-                client=client,
-                bucket_name=bucket.name,
-            )
+                yield from download_dbt_files(
+                    blob_grouped_by_directory=get_blobs_grouped_by_dir(
+                        blobs=[blob.name for blob in obj_list]
+                    ),
+                    config=config,
+                    client=client,
+                    bucket_name=bucket.name,
+                )
+            except Exception as blob_exc:
+                logger.error(
+                    f"Failed to process blobs in bucket {bucket.name}: {str(blob_exc)}"
+                )
+                logger.debug(traceback.format_exc())
+
+    except DBTConfigException as dbt_exc:
+        raise dbt_exc
     except Exception as exc:
         logger.debug(traceback.format_exc())
-        raise DBTConfigException(f"Error fetching dbt files from gcs: {exc}")
+        raise DBTConfigException(f"Error fetching dbt files from GCS: {exc}")
 
 
 @get_dbt_details.register
 def _(config: DbtAzureConfig):
     try:
         bucket_name, prefix = get_dbt_prefix_config(config)
+        # pylint: disable=import-outside-toplevel
+        from azure.core.exceptions import AzureError, ClientAuthenticationError
 
-        client = AzureClient(config.dbtSecurityConfig).create_blob_client()
+        try:
+            client = AzureClient(config.dbtSecurityConfig).create_blob_client()
+        except ClientAuthenticationError as auth_exc:
+            logger.error(
+                f"Failed to authenticate with Azure: {str(auth_exc)}. "
+                "Please check your Azure credentials and permissions."
+            )
+            raise DBTConfigException(
+                "Authentication failed with Azure. Please verify your credentials and permissions."
+            ) from auth_exc
+        except AzureError as azure_exc:
+            logger.error(f"Failed to create Azure client: {str(azure_exc)}")
+            raise DBTConfigException(
+                "Failed to initialize Azure client. Please check your Azure configuration."
+            ) from azure_exc
 
         if not bucket_name:
             container_dicts = client.list_containers()

metadata/ingestion/source/database/druid/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.druid.lineage import DruidLineageSource
 from metadata.ingestion.source.database.druid.metadata import DruidSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=DruidSource,
-    lineage_source_class=DruidLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=DruidSource)

metadata/ingestion/source/database/greenplum/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.greenplum.lineage import GreenplumLineageSource
 from metadata.ingestion.source.database.greenplum.metadata import GreenplumSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=GreenplumSource,
-    lineage_source_class=GreenplumLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=GreenplumSource)

metadata/ingestion/source/database/hive/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.hive.lineage import HiveLineageSource
 from metadata.ingestion.source.database.hive.metadata import HiveSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=HiveSource,
-    lineage_source_class=HiveLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=HiveSource)

metadata/ingestion/source/database/impala/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.impala.lineage import ImpalaLineageSource
 from metadata.ingestion.source.database.impala.metadata import ImpalaSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=ImpalaSource,
-    lineage_source_class=ImpalaLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=ImpalaSource)

metadata/ingestion/source/database/mariadb/service_spec.py

@@ -1,12 +1,4 @@
-from metadata.ingestion.source.database.mariadb.lineage import MariadbLineageSource
 from metadata.ingestion.source.database.mariadb.metadata import MariadbSource
-from metadata.profiler.interface.sqlalchemy.mariadb.profiler_interface import (
-    MariaDBProfilerInterface,
-)
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=MariadbSource,
-    lineage_source_class=MariadbLineageSource,
-    profiler_class=MariaDBProfilerInterface,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=MariadbSource)

metadata/ingestion/source/database/pinotdb/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.pinotdb.lineage import PinotdbLineageSource
 from metadata.ingestion.source.database.pinotdb.metadata import PinotdbSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=PinotdbSource,
-    lineage_source_class=PinotdbLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=PinotdbSource)

metadata/ingestion/source/database/postgres/lineage.py

@@ -31,6 +31,7 @@ from metadata.ingestion.source.database.postgres.queries import POSTGRES_SQL_STA
 from metadata.ingestion.source.database.postgres.query_parser import (
     PostgresQueryParserSource,
 )
+from metadata.utils.db_utils import PUBLIC_SCHEMA
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -89,7 +90,7 @@ class PostgresLineageSource(PostgresQueryParserSource, LineageSource):
                             aborted=self.get_aborted_status(row),
                             databaseName=self.get_database_name(row),
                             serviceName=self.config.serviceName,
-                            databaseSchema=self.get_schema_name(row),
+                            databaseSchema=self.get_schema_name(row) or PUBLIC_SCHEMA,
                             duration=row.get("duration"),
                         )
                     except Exception as err: