openmetadata-ingestion 1.4.8.0__py3-none-any.whl → 1.5.0.0rc2__py3-none-any.whl

metadata/ingestion/source/database/unitycatalog/client.py

@@ -14,6 +14,8 @@ Client to interact with databricks apis
 import json
 import traceback
 
+from requests import HTTPError
+
 from metadata.ingestion.source.database.databricks.client import (
     API_TIMEOUT,
     DatabricksClient,
@@ -27,6 +29,7 @@ from metadata.utils.logger import ingestion_logger
 logger = ingestion_logger()
 TABLE_LINEAGE_PATH = "/lineage-tracking/table-lineage/get"
 COLUMN_LINEAGE_PATH = "/lineage-tracking/column-lineage/get"
+TABLES_PATH = "/unity-catalog/tables"
 
 
 class UnityCatalogClient(DatabricksClient):
@@ -85,3 +88,21 @@ class UnityCatalogClient(DatabricksClient):
             logger.error(exc)
 
         return LineageColumnStreams()
+
+    def get_owner_info(self, full_table_name: str) -> str:
+        """
+        get owner info from tables API
+        """
+        try:
+            response = self.client.get(
+                f"{self.base_url}{TABLES_PATH}/{full_table_name}",
+                headers=self.headers,
+                timeout=API_TIMEOUT,
+            )
+            if response.status_code != 200:
+                raise HTTPError(response.text)
+            return response.json().get("owner")
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.error(exc)
+        return

metadata/ingestion/source/database/unitycatalog/lineage.py

@@ -55,7 +55,7 @@ class UnitycatalogLineageSource(Source):
         super().__init__()
         self.config = config
         self.metadata = metadata
-        self.service_connection = self.config.serviceConnection.__root__.config
+        self.service_connection = self.config.serviceConnection.root.config
         self.source_config = self.config.sourceConfig.config
         self.client = UnityCatalogClient(self.service_connection)
         self.connection_obj = get_connection(self.service_connection)
@@ -76,8 +76,8 @@ class UnitycatalogLineageSource(Source):
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
         """Create class instance"""
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: UnityCatalogConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: UnityCatalogConnection = config.serviceConnection.root.config
         if not isinstance(connection, UnityCatalogConnection):
             raise InvalidSourceException(
                 f"Expected UnityCatalogConnection, but got {connection}"
@@ -90,7 +90,7 @@ class UnitycatalogLineageSource(Source):
         col_lineage = []
         for column in to_table.columns:
             column_streams = self.client.get_column_lineage(
-                databricks_table_fqn, column_name=column.name.__root__
+                databricks_table_fqn, column_name=column.name.root
             )
             from_columns = []
             for col in column_streams.upstream_cols:
@@ -102,7 +102,7 @@ class UnitycatalogLineageSource(Source):
                 col_lineage.append(
                     ColumnLineage(
                         fromColumns=from_columns,
-                        toColumn=column.fullyQualifiedName.__root__,
+                        toColumn=column.fullyQualifiedName.root,
                     )
                 )
         if col_lineage:
@@ -119,9 +119,9 @@ class UnitycatalogLineageSource(Source):
             entity=Database, params={"service": self.config.serviceName}
         ):
             for table in self.metadata.list_all_entities(
-                entity=Table, params={"database": database.fullyQualifiedName.__root__}
+                entity=Table, params={"database": database.fullyQualifiedName.root}
             ):
-                databricks_table_fqn = f"{table.database.name}.{table.databaseSchema.name}.{table.name.__root__}"
+                databricks_table_fqn = f"{table.database.name}.{table.databaseSchema.name}.{table.name.root}"
                 table_streams: LineageTableStreams = self.client.get_table_lineage(
                     databricks_table_fqn
                 )

metadata/ingestion/source/database/unitycatalog/metadata.py

@@ -49,6 +49,12 @@ from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+)
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -60,6 +66,7 @@ from metadata.ingestion.source.database.external_table_lineage_mixin import (
 )
 from metadata.ingestion.source.database.multi_db_source import MultiDBSource
 from metadata.ingestion.source.database.stored_procedures_mixin import QueryByProcedure
+from metadata.ingestion.source.database.unitycatalog.client import UnityCatalogClient
 from metadata.ingestion.source.database.unitycatalog.connection import get_connection
 from metadata.ingestion.source.database.unitycatalog.models import (
     ColumnJson,
@@ -94,10 +101,11 @@ class UnitycatalogSource(
         self.context.get_global().table_views = []
         self.metadata = metadata
         self.service_connection: UnityCatalogConnection = (
-            self.config.serviceConnection.__root__.config
+            self.config.serviceConnection.root.config
         )
         self.external_location_map = {}
         self.client = get_connection(self.service_connection)
+        self.api_client = UnityCatalogClient(self.service_connection)
         self.connection_obj = self.client
         self.table_constraints = []
         self.context.storage_location = None
@@ -114,8 +122,8 @@ class UnitycatalogSource(
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: UnityCatalogConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: UnityCatalogConnection = config.serviceConnection.root.config
         if not isinstance(connection, UnityCatalogConnection):
             raise InvalidSourceException(
                 f"Expected UnityCatalogConnection, but got {connection}"
@@ -146,9 +154,11 @@ class UnitycatalogSource(
                     )
                     if filter_by_database(
                         self.config.sourceConfig.config.databaseFilterPattern,
-                        database_fqn
-                        if self.config.sourceConfig.config.useFqnForFiltering
-                        else catalog_name,
+                        (
+                            database_fqn
+                            if self.config.sourceConfig.config.useFqnForFiltering
+                            else catalog_name
+                        ),
                     ):
                         self.status.filter(
                             database_fqn,
@@ -195,9 +205,11 @@ class UnitycatalogSource(
                 )
                 if filter_by_schema(
                     self.config.sourceConfig.config.schemaFilterPattern,
-                    schema_fqn
-                    if self.config.sourceConfig.config.useFqnForFiltering
-                    else schema.name,
+                    (
+                        schema_fqn
+                        if self.config.sourceConfig.config.useFqnForFiltering
+                        else schema.name
+                    ),
                 ):
                     self.status.filter(schema_fqn, "Schema Filtered Out")
                     continue
@@ -220,12 +232,14 @@ class UnitycatalogSource(
         """
         yield Either(
             right=CreateDatabaseSchemaRequest(
-                name=schema_name,
-                database=fqn.build(
-                    metadata=self.metadata,
-                    entity_type=Database,
-                    service_name=self.context.get().database_service,
-                    database_name=self.context.get().database,
+                name=EntityName(schema_name),
+                database=FullyQualifiedEntityName(
+                    fqn.build(
+                        metadata=self.metadata,
+                        entity_type=Database,
+                        service_name=self.context.get().database_service,
+                        database_name=self.context.get().database,
+                    )
                 ),
             )
         )
@@ -257,9 +271,11 @@ class UnitycatalogSource(
                 )
                 if filter_by_table(
                     self.config.sourceConfig.config.tableFilterPattern,
-                    table_fqn
-                    if self.config.sourceConfig.config.useFqnForFiltering
-                    else table_name,
+                    (
+                        table_fqn
+                        if self.config.sourceConfig.config.useFqnForFiltering
+                        else table_name
+                    ),
                 ):
                     self.status.filter(
                         table_fqn,
@@ -287,7 +303,7 @@ class UnitycatalogSource(
                 )
 
     def yield_table(
-        self, table_name_and_type: Tuple[str, str]
+        self, table_name_and_type: Tuple[str, TableType]
     ) -> Iterable[Either[CreateTableRequest]]:
         """
         From topology.
@@ -301,9 +317,8 @@ class UnitycatalogSource(
             self.external_location_map[
                 (db_name, schema_name, table_name)
             ] = table.storage_location
-        table_constraints = None
         try:
-            columns = self.get_columns(table.columns)
+            columns = list(self.get_columns(table.columns))
             (
                 primary_constraints,
                 foreign_constraints,
@@ -314,18 +329,21 @@ class UnitycatalogSource(
             )
 
             table_request = CreateTableRequest(
-                name=table_name,
+                name=EntityName(table_name),
                 tableType=table_type,
                 description=table.comment,
                 columns=columns,
                 tableConstraints=table_constraints,
-                databaseSchema=fqn.build(
-                    metadata=self.metadata,
-                    entity_type=DatabaseSchema,
-                    service_name=self.context.get().database_service,
-                    database_name=self.context.get().database,
-                    schema_name=schema_name,
+                databaseSchema=FullyQualifiedEntityName(
+                    fqn.build(
+                        metadata=self.metadata,
+                        entity_type=DatabaseSchema,
+                        service_name=self.context.get().database_service,
+                        database_name=self.context.get().database,
+                        schema_name=schema_name,
+                    )
                 ),
+                owners=self.get_owner_ref(table_name),
             )
             yield Either(right=table_request)
 
@@ -403,7 +421,7 @@ class UnitycatalogSource(
                 for parent_column in column.parent_columns:
                     col_fqn = fqn._build(referred_table_fqn, parent_column, quote=False)
                     if col_fqn:
-                        referred_column_fqns.append(col_fqn)
+                        referred_column_fqns.append(FullyQualifiedEntityName(col_fqn))
             else:
                 continue
 
@@ -443,12 +461,12 @@ class UnitycatalogSource(
         """
         try:
             if column.children is None:
-                if column_json.metadata:
-                    column.description = column_json.metadata.comment
+                if column_json.metadata and column_json.metadata.comment:
+                    column.description = Markdown(column_json.metadata.comment)
             else:
                 for i, child in enumerate(column.children):
-                    if column_json.metadata:
-                        column.description = column_json.metadata.comment
+                    if column_json.metadata and column_json.metadata.comment:
+                        column.description = Markdown(column_json.metadata.comment)
                     if (
                         column_json.type
                         and isinstance(column_json.type, Type)
@@ -479,21 +497,27 @@ class UnitycatalogSource(
         """
 
         for column in column_data:
-            if column.type_text.lower().startswith("union"):
-                column.type_text = column.Type.replace(" ", "")
-            if column.type_text.lower() == "struct":
-                column.type_text = "struct<>"
+            parsed_string = {}
+            if column.type_text:
+                if column.type_text.lower().startswith("union"):
+                    column.type_text = column.type_text.replace(" ", "")
+                if (
+                    column.type_text.lower() == "struct"
+                    or column.type_text.lower() == "array"
+                ):
+                    column.type_text = column.type_text.lower() + "<>"
 
-            parsed_string = ColumnTypeParser._parse_datatype_string(  # pylint: disable=protected-access
-                column.type_text.lower()
-            )
+                parsed_string = ColumnTypeParser._parse_datatype_string(  # pylint: disable=protected-access
+                    column.type_text.lower()
+                )
             parsed_string["name"] = column.name[:256]
             parsed_string["dataLength"] = parsed_string.get("dataLength", 1)
-            parsed_string["description"] = column.comment
+            if column.comment:
+                parsed_string["description"] = Markdown(column.comment)
             parsed_column = Column(**parsed_string)
             self.add_complex_datatype_descriptions(
                 column=parsed_column,
-                column_json=ColumnJson.parse_obj(json.loads(column.type_json)),
+                column_json=ColumnJson.model_validate(json.loads(column.type_json)),
             )
             yield parsed_column
 
@@ -533,3 +557,19 @@ class UnitycatalogSource(
 
     def close(self):
         """Nothing to close"""
+
+    def get_owner_ref(self, table_name: str) -> Optional[EntityReferenceList]:
+        """
+        Method to process the table owners
+        """
+        try:
+            full_table_name = f"{self.context.get().database}.{self.context.get().database_schema}.{table_name}"
+            owner = self.api_client.get_owner_info(full_table_name)
+            if not owner:
+                return
+            owner_ref = self.metadata.get_reference_by_email(email=owner)
+            return owner_ref
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error processing owner for table {table_name}: {exc}")
+        return

metadata/ingestion/source/database/unitycatalog/models.py

@@ -19,16 +19,16 @@ from pydantic import BaseModel
 
 
 class DatabricksTable(BaseModel):
-    name: Optional[str]
-    catalog_name: Optional[str]
-    schema_name: Optional[str]
+    name: Optional[str] = None
+    catalog_name: Optional[str] = None
+    schema_name: Optional[str] = None
 
 
 class DatabricksColumn(BaseModel):
-    name: Optional[str]
-    catalog_name: Optional[str]
-    schema_name: Optional[str]
-    table_name: Optional[str]
+    name: Optional[str] = None
+    catalog_name: Optional[str] = None
+    schema_name: Optional[str] = None
+    table_name: Optional[str] = None
 
 
 class LineageTableStreams(BaseModel):
@@ -48,24 +48,24 @@ class ForeignConstrains(BaseModel):
 
 
 class Metadata(BaseModel):
-    comment: Optional[str]
+    comment: Optional[str] = None
 
 
 class ColumnJson(BaseModel):
-    name: Optional[str]
-    type: Optional[Union["Type", str]]
-    metadata: Optional[Metadata]
+    name: Optional[str] = None
+    type: Optional[Union["Type", str]] = None
+    metadata: Optional[Metadata] = None
 
 
 class ElementType(BaseModel):
-    type: Optional[str]
-    fields: Optional[List[ColumnJson]]
+    type: Optional[str] = None
+    fields: Optional[List[ColumnJson]] = None
 
 
 class Type(BaseModel):
-    type: Optional[str]
-    elementType: Optional[Union[ElementType, str]]
-    fields: Optional[List[ColumnJson]]
+    type: Optional[str] = None
+    elementType: Optional[Union[ElementType, str]] = None
+    fields: Optional[List[ColumnJson]] = None
 
 
 ColumnJson.update_forward_refs()

metadata/ingestion/source/database/unitycatalog/query_parser.py

@@ -54,8 +54,8 @@ class UnityCatalogQueryParserSource(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
         """Create class instance"""
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: UnityCatalogConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: UnityCatalogConnection = config.serviceConnection.root.config
         if not isinstance(connection, UnityCatalogConnection):
             raise InvalidSourceException(
                 f"Expected UnityCatalogConnection, but got {connection}"

metadata/ingestion/source/database/usage_source.py

@@ -12,11 +12,13 @@
 Usage Source Module
 """
 import csv
+import os
 import traceback
 from abc import ABC
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from typing import Iterable
 
+from metadata.generated.schema.type.basic import DateTime
 from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.source.database.query_parser_source import QueryParserSource
@@ -37,34 +39,45 @@ class UsageSource(QueryParserSource, ABC):
         Method to handle the usage from query logs
         """
         try:
-            query_list = []
-            with open(
-                self.config.sourceConfig.config.queryLogFilePath, "r", encoding="utf-8"
-            ) as fin:
-                for record in csv.DictReader(fin):
-                    query_dict = dict(record)
-                    analysis_date = (
-                        datetime.utcnow()
-                        if not query_dict.get("start_time")
-                        else datetime.strptime(
-                            query_dict.get("start_time"), "%Y-%m-%d %H:%M:%S.%f"
+            query_log_path = self.config.sourceConfig.config.queryLogFilePath
+            if os.path.isfile(query_log_path):
+                file_paths = [query_log_path]
+            elif os.path.isdir(query_log_path):
+                file_paths = [
+                    os.path.join(query_log_path, f)
+                    for f in os.listdir(query_log_path)
+                    if f.endswith(".csv")
+                ]
+            else:
+                raise ValueError(f"{query_log_path} is neither a file nor a directory.")
+            for file_path in file_paths:
+                query_list = []
+                with open(file_path, "r", encoding="utf-8") as fin:
+                    for record in csv.DictReader(fin):
+                        query_dict = dict(record)
+
+                        analysis_date = (
+                            datetime.now(timezone.utc)
+                            if not query_dict.get("start_time")
+                            else datetime.strptime(
+                                query_dict.get("start_time"), "%Y-%m-%d %H:%M:%S.%f"
+                            )
                         )
-                    )
-                    query_list.append(
-                        TableQuery(
-                            query=query_dict["query_text"],
-                            userName=query_dict.get("user_name", ""),
-                            startTime=query_dict.get("start_time", ""),
-                            endTime=query_dict.get("end_time", ""),
-                            duration=query_dict.get("duration"),
-                            analysisDate=analysis_date,
-                            aborted=self.get_aborted_status(query_dict),
-                            databaseName=self.get_database_name(query_dict),
-                            serviceName=self.config.serviceName,
-                            databaseSchema=self.get_schema_name(query_dict),
+                        query_list.append(
+                            TableQuery(
+                                query=query_dict["query_text"],
+                                userName=query_dict.get("user_name", ""),
+                                startTime=query_dict.get("start_time", ""),
+                                endTime=query_dict.get("end_time", ""),
+                                duration=query_dict.get("duration"),
+                                analysisDate=DateTime(analysis_date),
+                                aborted=self.get_aborted_status(query_dict),
+                                databaseName=self.get_database_name(query_dict),
+                                serviceName=self.config.serviceName,
+                                databaseSchema=self.get_schema_name(query_dict),
+                            )
                         )
-                    )
-            yield TableQueries(queries=query_list)
+                yield TableQueries(queries=query_list)
         except Exception as err:
             logger.debug(traceback.format_exc())
             logger.warning(f"Failed to read queries form log file due to: {err}")
@@ -93,15 +106,15 @@ class UsageSource(QueryParserSource, ABC):
                 f"Scanning query logs for {(self.start + timedelta(days=days)).date()} - "
                 f"{(self.start + timedelta(days=days + 1)).date()}"
             )
+            query = None
             try:
+                query = self.get_sql_statement(
+                    start_time=self.start + timedelta(days=days),
+                    end_time=self.start + timedelta(days=days + 1),
+                )
                 for engine in self.get_engine():
                     with engine.connect() as conn:
-                        rows = conn.execute(
-                            self.get_sql_statement(
-                                start_time=self.start + timedelta(days=days),
-                                end_time=self.start + timedelta(days=days + 1),
-                            )
-                        )
+                        rows = conn.execute(query)
                         queries = []
                         for row in rows:
                             row = dict(row)
@@ -118,7 +131,7 @@ class UsageSource(QueryParserSource, ABC):
                                         userName=row["user_name"],
                                         startTime=str(row["start_time"]),
                                         endTime=str(row["end_time"]),
-                                        analysisDate=row["start_time"],
+                                        analysisDate=DateTime(row["start_time"]),
                                         aborted=self.get_aborted_status(row),
                                         databaseName=self.get_database_name(row),
                                         duration=row.get("duration"),
@@ -133,6 +146,10 @@ class UsageSource(QueryParserSource, ABC):
                                 )
                     yield TableQueries(queries=queries)
             except Exception as exc:
+                if query:
+                    logger.debug(
+                        f"###### USAGE QUERY #######\n{query}\n##########################"
+                    )
                 logger.debug(traceback.format_exc())
                 logger.error(f"Source usage processing error: {exc}")
 

metadata/ingestion/source/database/vertica/metadata.py

@@ -277,8 +277,8 @@ class VerticaSource(CommonDbSourceService, MultiDBSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: VerticaConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: VerticaConnection = config.serviceConnection.root.config
         if not isinstance(connection, VerticaConnection):
             raise InvalidSourceException(
                 f"Expected VerticaConnection, but got {connection}"
@@ -303,7 +303,7 @@ class VerticaSource(CommonDbSourceService, MultiDBSource):
         yield from self._execute_database_query(VERTICA_LIST_DATABASES)
 
     def get_database_names(self) -> Iterable[str]:
-        configured_db = self.config.serviceConnection.__root__.config.database
+        configured_db = self.config.serviceConnection.root.config.database
         if configured_db:
             self.set_inspector(database_name=configured_db)
             self.set_schema_description_map()

metadata/ingestion/source/database/vertica/query_parser.py

@@ -47,8 +47,8 @@ class VerticaQueryParserSource(QueryParserSource, ABC):
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
         """Create class instance"""
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: VerticaConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: VerticaConnection = config.serviceConnection.root.config
         if not isinstance(connection, VerticaConnection):
             raise InvalidSourceException(
                 f"Expected VerticaConnection, but got {connection}"
@@ -56,7 +56,7 @@ class VerticaQueryParserSource(QueryParserSource, ABC):
         return cls(config, metadata)
 
     def get_table_query(self) -> Iterable[TableQuery]:
-        database = self.config.serviceConnection.__root__.config.database
+        database = self.config.serviceConnection.root.config.database
         if database:
             yield from super().get_table_query()
         else:
@@ -64,6 +64,6 @@ class VerticaQueryParserSource(QueryParserSource, ABC):
             for res in results:
                 row = list(res)
                 logger.info(f"Ingesting from database: {row[0]}")
-                self.config.serviceConnection.__root__.config.database = row[0]
+                self.config.serviceConnection.root.config.database = row[0]
                 self.engine = get_connection(self.service_connection)
                 yield from super().get_table_query()

metadata/ingestion/source/messaging/common_broker_source.py

@@ -33,6 +33,7 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
 from metadata.generated.schema.type.schema import SchemaType, Topic
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.models.ometa_topic_data import OMetaTopicSampleData
@@ -74,6 +75,7 @@ class CommonBrokerSource(MessagingServiceSource, ABC):
     ):
         super().__init__(config, metadata)
         self.generate_sample_data = self.config.sourceConfig.config.generateSampleData
+        self.service_connection = self.config.serviceConnection.root.config
         self.admin_client = self.connection.admin_client
         self.schema_registry_client = self.connection.schema_registry_client
         self.context.processed_schemas = {}
@@ -105,8 +107,8 @@ class CommonBrokerSource(MessagingServiceSource, ABC):
             topic_schema = self._parse_topic_metadata(topic_details.topic_name)
             logger.info(f"Fetching topic config {topic_details.topic_name}")
             topic = CreateTopicRequest(
-                name=topic_details.topic_name,
-                service=self.context.get().messaging_service,
+                name=EntityName(topic_details.topic_name),
+                service=FullyQualifiedEntityName(self.context.get().messaging_service),
                 partitions=len(topic_details.topic_metadata.partitions),
                 replicationFactor=len(
                     topic_details.topic_metadata.partitions.get(0).replicas
@@ -235,15 +237,27 @@ class CommonBrokerSource(MessagingServiceSource, ABC):
         return None
 
     def _parse_topic_metadata(self, topic_name: str) -> Optional[Schema]:
+
+        # To find topic in artifact registry, dafault is "<topic_name>-value"
+        # But suffix can be overridden using schemaRegistryTopicSuffixName
+        topic_schema_registry_name = (
+            topic_name + self.service_connection.schemaRegistryTopicSuffixName
+        )
+
         try:
             if self.schema_registry_client:
                 registered_schema = self.schema_registry_client.get_latest_version(
-                    topic_name + "-value"
+                    topic_schema_registry_name
                 )
                 return registered_schema.schema
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.warning(f"Failed to get schema for topic [{topic_name}]: {exc}")
+            logger.warning(
+                (
+                    f"Failed to get schema for topic [{topic_name}] "
+                    f"(looking for {topic_schema_registry_name}) in registry: {exc}"
+                )
+            )
             self.status.warning(
                 topic_name, f"failed to get schema: {exc} for topic {topic_name}"
             )

metadata/ingestion/source/messaging/kafka/connection.py

@@ -95,7 +95,7 @@ def get_connection(
     schema_registry_client = None
     consumer_client = None
     if connection.schemaRegistryURL:
-        schema_registry_config["url"] = connection.schemaRegistryURL
+        schema_registry_config["url"] = str(connection.schemaRegistryURL)
         schema_registry_client = SchemaRegistryClient(schema_registry_config)
         consumer_config["bootstrap.servers"] = connection.bootstrapServers
         if "group.id" not in consumer_config: