openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/source/database/hive/custom_hive_connection.py

@@ -0,0 +1,195 @@
+import contextlib
+import getpass
+import ssl
+
+import thrift.protocol.TBinaryProtocol
+import thrift.transport.TSocket
+import thrift.transport.TTransport
+from pyhive.hive import Connection as BaseConnection
+from pyhive.hive import _check_status, get_installed_sasl
+from TCLIService import TCLIService, ttypes
+
+
+class CustomHiveConnection(BaseConnection):
+    """Custom Hive connection that integrates puretransport and SSL certificate support"""
+
+    def __init__(
+        self,
+        host=None,
+        port=None,
+        scheme=None,
+        username=None,
+        database="default",
+        auth=None,
+        configuration=None,
+        kerberos_service_name=None,
+        password=None,
+        check_hostname=None,
+        ssl_cert=None,
+        thrift_transport=None,
+        use_ssl=False,
+        ssl_certfile=None,
+        ssl_keyfile=None,
+        ssl_ca_certs=None,
+        ssl_cert_reqs=None,
+        ssl_check_hostname=None,
+    ):
+        """Connect to HiveServer2 with integrated puretransport and SSL support"""
+
+        # Handle HTTPS scheme with SSL context
+        if scheme in ("https", "http") and thrift_transport is None:
+            port = port or 1000
+            ssl_context = None
+            if scheme == "https":
+                from ssl import create_default_context
+
+                ssl_context = create_default_context()
+                ssl_context.check_hostname = check_hostname == "true"
+                ssl_cert = ssl_cert or "none"
+                ssl_cert_parameter_map = {
+                    "none": 0,  # CERT_NONE
+                    "optional": 1,  # CERT_OPTIONAL
+                    "required": 2,  # CERT_REQUIRED
+                }
+                ssl_context.verify_mode = ssl_cert_parameter_map.get(ssl_cert, 0)
+            thrift_transport = thrift.transport.THttpClient.THttpClient(
+                uri_or_host="{scheme}://{host}:{port}/cliservice/".format(
+                    scheme=scheme, host=host, port=port
+                ),
+                ssl_context=ssl_context,
+            )
+
+            if auth in ("BASIC", "NOSASL", "NONE", None):
+                # Always needs the Authorization header
+                self._set_authorization_header(thrift_transport, username, password)
+            elif auth == "KERBEROS" and kerberos_service_name:
+                self._set_kerberos_header(thrift_transport, kerberos_service_name, host)
+            else:
+                raise ValueError(
+                    "Authentication is not valid use one of:"
+                    "BASIC, NOSASL, KERBEROS, NONE"
+                )
+            host, port, auth, kerberos_service_name, password = (
+                None,
+                None,
+                None,
+                None,
+                None,
+            )
+
+        username = username or getpass.getuser()
+        configuration = configuration or {}
+
+        if (password is not None) != (auth in ("LDAP", "CUSTOM")):
+            raise ValueError(
+                "Password should be set if and only if in LDAP or CUSTOM mode; "
+                "Remove password or use one of those modes"
+            )
+        if (kerberos_service_name is not None) != (auth == "KERBEROS"):
+            raise ValueError(
+                "kerberos_service_name should be set if and only if in KERBEROS mode"
+            )
+
+        # Use puretransport if SSL is enabled or if thrift_transport is provided
+        if use_ssl or thrift_transport is not None:
+            if thrift_transport is not None:
+                # Use provided thrift_transport
+                self._transport = thrift_transport
+            else:
+                # Create puretransport with SSL
+                import puretransport
+
+                # Prepare socket_kwargs for SSL
+                socket_kwargs = {}
+                if ssl_certfile:
+                    socket_kwargs["certfile"] = ssl_certfile
+                if ssl_keyfile:
+                    socket_kwargs["keyfile"] = ssl_keyfile
+                if ssl_ca_certs:
+                    socket_kwargs["ca_certs"] = ssl_ca_certs
+                if ssl_cert_reqs is not None:
+                    socket_kwargs["cert_reqs"] = ssl_cert_reqs
+                elif use_ssl:
+                    socket_kwargs["cert_reqs"] = ssl.CERT_NONE
+
+                # Create puretransport
+                self._transport = puretransport.transport_factory(
+                    host=host or "localhost",
+                    port=port or 10000,
+                    username=username,
+                    password=password or username,
+                    use_ssl=use_ssl,
+                    socket_kwargs=socket_kwargs if socket_kwargs else None,
+                )
+        else:
+            # Use standard connection logic
+            if port is None:
+                port = 10000
+            if auth is None:
+                auth = "NONE"
+            socket = thrift.transport.TSocket.TSocket(host, port)
+            if auth == "NOSASL":
+                # NOSASL corresponds to hive.server2.authentication=NOSASL in hive-site.xml
+                self._transport = thrift.transport.TTransport.TBufferedTransport(socket)
+            elif auth in ("LDAP", "KERBEROS", "NONE", "CUSTOM"):
+                # Defer import so package dependency is optional
+                import thrift_sasl
+
+                if auth == "KERBEROS":
+                    # KERBEROS mode in hive.server2.authentication is GSSAPI in sasl library
+                    sasl_auth = "GSSAPI"
+                else:
+                    sasl_auth = "PLAIN"
+                    if password is None:
+                        # Password doesn't matter in NONE mode, just needs to be nonempty.
+                        password = "x"
+
+                self._transport = thrift_sasl.TSaslClientTransport(
+                    lambda: get_installed_sasl(
+                        host=host,
+                        sasl_auth=sasl_auth,
+                        service=kerberos_service_name,
+                        username=username,
+                        password=password,
+                    ),
+                    sasl_auth,
+                    socket,
+                )
+            else:
+                # All HS2 config options:
+                # https://cwiki.apache.org/confluence/display/Hive/Setting+Up+HiveServer2#SettingUpHiveServer2-Configuration
+                # PAM currently left to end user via thrift_transport option.
+                raise NotImplementedError(
+                    "Only NONE, NOSASL, LDAP, KERBEROS, CUSTOM "
+                    "authentication are supported, got {}".format(auth)
+                )
+
+        protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(self._transport)
+        self._client = TCLIService.Client(protocol)
+        # oldest version that still contains features we care about
+        # "V6 uses binary type for binary payload (was string) and uses columnar result set"
+        protocol_version = ttypes.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6
+
+        try:
+            self._transport.open()
+            open_session_req = ttypes.TOpenSessionReq(
+                client_protocol=protocol_version,
+                configuration=configuration,
+                username=username,
+            )
+            response = self._client.OpenSession(open_session_req)
+            _check_status(response)
+            assert (
+                response.sessionHandle is not None
+            ), "Expected a session from OpenSession"
+            self._sessionHandle = response.sessionHandle
+            assert (
+                response.serverProtocolVersion == protocol_version
+            ), "Unable to handle protocol version {}".format(
+                response.serverProtocolVersion
+            )
+            with contextlib.closing(self.cursor()) as cursor:
+                cursor.execute("USE `{}`".format(database))
+        except:
+            self._transport.close()
+            raise

metadata/ingestion/source/database/iceberg/metadata.py

@@ -11,6 +11,7 @@
 """
 Iceberg source methods.
 """
+import time
 import traceback
 from typing import Any, Iterable, Optional, Tuple
 
@@ -169,6 +170,43 @@ class IcebergSource(DatabaseServiceSource):
         yield Either(right=schema_request)
         self.register_record_schema_request(schema_request=schema_request)
 
+    def _load_iceberg_table(self, table_identifier: str):
+        """
+        load iceberg table properly with handling
+        network connection error
+        """
+        try:
+            from botocore.exceptions import EndpointConnectionError
+
+            # Add retry logic for transient network issues
+            max_retries = 3
+            retry_delay = 1
+
+            for attempt in range(max_retries):
+                try:
+                    table = self.iceberg.load_table(table_identifier)
+                    # Success, exit retry loop
+                    return table
+                except (OSError, EndpointConnectionError) as e:
+                    if "Couldn't resolve host name" in str(
+                        e
+                    ) or "NETWORK_CONNECTION" in str(e):
+                        if attempt < max_retries - 1:
+                            logger.warning(
+                                f"Network error loading table {table_identifier}, "
+                                f"retrying in {retry_delay} seconds (attempt {attempt + 1}/{max_retries}): {e}"
+                            )
+                            time.sleep(retry_delay)
+                            retry_delay *= 2  # Exponential backoff
+                        else:
+                            logger.warning(f"Maximum retries reached::: {exc}")
+                    else:
+                        logger.warning(f"Other error than host connection::: {exc}")
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not load iceberg table properly {exc}")
+        return None
+
     def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]:
         """
         Prepares the table name to be sent to stage.
@@ -178,9 +216,14 @@ class IcebergSource(DatabaseServiceSource):
 
         for table_identifier in self.iceberg.list_tables(namespace):
             try:
-                table = self.iceberg.load_table(table_identifier)
+                table = self._load_iceberg_table(table_identifier)
                 # extract table name from table identifier, which does not include catalog name
                 table_name = get_table_name_as_str(table_identifier)
+                if not table:
+                    logger.debug(
+                        f"iceberg Table could not be fetched for table name = {table_name}"
+                    )
+                    continue
                 table_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,

metadata/ingestion/source/database/iceberg/models.py

@@ -51,24 +51,33 @@ class IcebergTable(BaseModel):
         """Responsible for parsing the needed information from a PyIceberg Table."""
         iceberg_columns = table.schema().fields
 
+        partition_columns = []
+        for partition in table.spec().fields:
+            partition_column_name = get_column_from_partition(
+                iceberg_columns, partition
+            )
+            column_partition_type = get_column_partition_type(
+                iceberg_columns, partition
+            )
+
+            if not (partition_column_name and column_partition_type):
+                continue
+
+            partition_columns.append(
+                PartitionColumnDetails(
+                    columnName=partition_column_name,
+                    intervalType=column_partition_type,
+                    interval=None,
+                )
+            )
+
         return IcebergTable(
             name=name,
             tableType=table_type,
             description=table.properties.get("comment"),
             owners=owners,
             columns=[IcebergColumnParser.parse(column) for column in iceberg_columns],
-            tablePartition=TablePartition(
-                columns=[
-                    PartitionColumnDetails(
-                        columnName=get_column_from_partition(
-                            iceberg_columns, partition
-                        ),
-                        intervalType=get_column_partition_type(
-                            iceberg_columns, partition
-                        ),
-                        interval=None,
-                    )
-                    for partition in table.spec().fields
-                ]
+            tablePartition=(
+                TablePartition(columns=partition_columns) if partition_columns else None
             ),
         )

metadata/ingestion/source/database/lineage_processors.py

@@ -0,0 +1,386 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Mixin class with common Stored Procedures logic aimed at lineage.
+"""
+import re
+import time
+import traceback
+from datetime import datetime
+from multiprocessing import Queue
+from typing import Dict, Iterable, List, Optional, Union
+
+import networkx as nx
+from pydantic import BaseModel, ConfigDict, Field
+
+from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
+from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
+from metadata.generated.schema.entity.data.storedProcedure import StoredProcedure
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.type.basic import SqlQuery, Timestamp
+from metadata.generated.schema.type.entityLineage import Source as LineageSource
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.tableQuery import TableQuery
+from metadata.ingestion.api.models import Either
+from metadata.ingestion.lineage.models import Dialect
+from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query
+from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.models import TableView
+from metadata.utils import fqn
+from metadata.utils.db_utils import get_view_lineage
+from metadata.utils.logger import ingestion_logger
+from metadata.utils.time_utils import datetime_to_timestamp
+
+logger = ingestion_logger()
+
+# pylint: disable=invalid-name
+
+
+class QueryByProcedure(BaseModel):
+    """
+    Query(ies) executed by each stored procedure
+    """
+
+    procedure_name: str = Field(None, alias="PROCEDURE_NAME")
+    query_type: str = Field(..., alias="QUERY_TYPE")
+    query_database_name: Optional[str] = Field(None, alias="QUERY_DATABASE_NAME")
+    query_schema_name: Optional[str] = Field(None, alias="QUERY_SCHEMA_NAME")
+    procedure_text: str = Field(..., alias="PROCEDURE_TEXT")
+    procedure_start_time: datetime = Field(..., alias="PROCEDURE_START_TIME")
+    procedure_end_time: datetime = Field(..., alias="PROCEDURE_END_TIME")
+    query_start_time: Optional[datetime] = Field(None, alias="QUERY_START_TIME")
+    query_duration: Optional[float] = Field(None, alias="QUERY_DURATION")
+    query_text: str = Field(..., alias="QUERY_TEXT")
+    query_user_name: Optional[str] = Field(None, alias="QUERY_USER_NAME")
+
+    model_config = ConfigDict(populate_by_name=True)
+
+
+class ProcedureAndQuery(BaseModel):
+    """
+    Model to hold the procedure and its queries
+    """
+
+    procedure: StoredProcedure
+    query_by_procedure: QueryByProcedure
+
+    model_config = ConfigDict(populate_by_name=True)
+
+
+class ProcedureAndProcedureGraph(BaseModel):
+    """
+    Model to hold the procedure and its graph
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    procedure: StoredProcedure
+    graph: nx.DiGraph
+
+
+def is_lineage_query(query_type: str, query_text: str) -> bool:
+    """Check if it's worth it to parse the query for lineage"""
+
+    logger.debug(
+        f"Validating query lineage for type [{query_type}] and text [{query_text}]"
+    )
+
+    if query_type in ("MERGE", "UPDATE", "CREATE_TABLE_AS_SELECT"):
+        return True
+
+    if query_type == "INSERT" and re.search(
+        "^.*insert.*into.*select.*$", query_text.replace("\n", " "), re.IGNORECASE
+    ):
+        return True
+
+    return False
+
+
+def _yield_procedure_lineage(
+    metadata: OpenMetadata,
+    service_name: str,
+    dialect: Dialect,
+    processCrossDatabaseLineage: bool,
+    crossDatabaseServiceNames: List[str],
+    parsingTimeoutLimit: int,
+    query_by_procedure: QueryByProcedure,
+    procedure: StoredProcedure,
+    procedure_graph_map: Dict[str, ProcedureAndProcedureGraph],
+    enableTempTableLineage: bool,
+) -> Iterable[Either[AddLineageRequest]]:
+    """Add procedure lineage from its query"""
+    graph = None
+    if enableTempTableLineage:
+        if not procedure_graph_map.get(procedure.fullyQualifiedName.root):
+            # Map to store the directed graph for each procedure with its FQN as key
+            procedure_graph_map[
+                procedure.fullyQualifiedName.root
+            ] = ProcedureAndProcedureGraph(procedure=procedure, graph=nx.DiGraph())
+
+        graph = procedure_graph_map.get(procedure.fullyQualifiedName.root).graph
+
+    # Prepare service names for lineage processing
+    service_names = [service_name]
+    if processCrossDatabaseLineage and crossDatabaseServiceNames:
+        service_names.extend(crossDatabaseServiceNames)
+
+    if is_lineage_query(
+        query_type=query_by_procedure.query_type,
+        query_text=query_by_procedure.query_text,
+    ):
+        for either_lineage in get_lineage_by_query(
+            metadata,
+            query=query_by_procedure.query_text,
+            service_names=service_names,
+            database_name=query_by_procedure.query_database_name,
+            schema_name=query_by_procedure.query_schema_name,
+            dialect=dialect,
+            timeout_seconds=parsingTimeoutLimit,
+            lineage_source=LineageSource.QueryLineage,
+            graph=graph,
+        ):
+            if either_lineage.left is None and either_lineage.right.edge.lineageDetails:
+                either_lineage.right.edge.lineageDetails.pipeline = EntityReference(
+                    id=procedure.id,
+                    type="storedProcedure",
+                )
+
+            yield either_lineage
+
+
+def procedure_lineage_processor(
+    procedure_and_queries: List[ProcedureAndQuery],
+    queue: Queue,
+    metadata: OpenMetadata,
+    service_name: str,
+    dialect: Dialect,
+    processCrossDatabaseLineage: bool,
+    crossDatabaseServiceNames: List[str],
+    parsingTimeoutLimit: int,
+    procedure_graph_map: Dict[str, ProcedureAndProcedureGraph],
+    enableTempTableLineage: bool,
+) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
+    """
+    Process the procedure and its queries to add lineage
+    """
+    for procedure_and_query in procedure_and_queries:
+        try:
+            for lineage in _yield_procedure_lineage(
+                query_by_procedure=procedure_and_query.query_by_procedure,
+                procedure=procedure_and_query.procedure,
+                metadata=metadata,
+                service_name=service_name,
+                dialect=dialect,
+                processCrossDatabaseLineage=processCrossDatabaseLineage,
+                crossDatabaseServiceNames=crossDatabaseServiceNames,
+                parsingTimeoutLimit=parsingTimeoutLimit,
+                procedure_graph_map=procedure_graph_map,
+                enableTempTableLineage=enableTempTableLineage,
+            ):
+                if lineage and lineage.right is not None:
+                    queue.put(
+                        Either(
+                            right=OMetaLineageRequest(
+                                override_lineage=False,
+                                lineage_request=lineage.right,
+                                entity=StoredProcedure,
+                                entity_fqn=procedure_and_query.procedure.fullyQualifiedName.root,
+                            )
+                        )
+                    )
+                else:
+                    queue.put(lineage)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Could not get lineage for store procedure "
+                f"'{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]."
+            )
+        try:
+            for lineage in yield_procedure_query(
+                query_by_procedure=procedure_and_query.query_by_procedure,
+                procedure=procedure_and_query.procedure,
+                service_name=service_name,
+            ):
+                queue.put(lineage)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Could not get query for store procedure "
+                f"'{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]."
+            )
+
+
+def yield_procedure_query(
+    query_by_procedure: QueryByProcedure, procedure: StoredProcedure, service_name: str
+) -> Iterable[Either[CreateQueryRequest]]:
+    """Check the queries triggered by the procedure and add their lineage, if any"""
+    stored_procedure_query_lineage = is_lineage_query(
+        query_type=query_by_procedure.query_type,
+        query_text=query_by_procedure.query_text,
+    )
+
+    yield Either(
+        right=CreateQueryRequest(
+            query=SqlQuery(query_by_procedure.query_text),
+            query_type=query_by_procedure.query_type,
+            duration=query_by_procedure.query_duration,
+            queryDate=Timestamp(
+                root=datetime_to_timestamp(query_by_procedure.query_start_time, True)
+            ),
+            triggeredBy=EntityReference(
+                id=procedure.id,
+                type="storedProcedure",
+            ),
+            processedLineage=bool(stored_procedure_query_lineage),
+            service=service_name,
+        )
+    )
+
+
+# Function that will run in separate processes - defined at module level for pickling
+def process_chunk_in_subprocess(chunk, processor_fn, queue, *args):
+    """
+    Process a chunk of data in a subprocess.
+
+    Args:
+        chunk_and_processor_fn: Tuple containing (chunk, processor_fn, queue, *args)
+
+    Returns:
+        True if processing succeeded, False otherwise
+    """
+    try:
+        # Process each item in the chunk
+        processor_fn(chunk, queue, *args)
+        time.sleep(0.1)
+        return True
+    except Exception as e:
+        logger.error(f"Error processing chunk in subprocess: {e}")
+        logger.error(traceback.format_exc())
+        return False
+
+
+def _query_already_processed(metadata: OpenMetadata, table_query: TableQuery) -> bool:
+    """
+    Check if a query has already been processed by validating if exists
+    in ES with lineageProcessed as True
+    """
+    checksums = metadata.es_get_queries_with_lineage(
+        service_name=table_query.serviceName,
+    )
+    return fqn.get_query_checksum(table_query.query) in checksums or {}
+
+
+def query_lineage_processor(
+    table_queries: List[TableQuery],
+    queue: Queue,
+    metadata: OpenMetadata,
+    dialect: Dialect,
+    graph: nx.DiGraph,
+    processCrossDatabaseLineage: bool,
+    crossDatabaseServiceNames: List[str],
+    parsingTimeoutLimit: int,
+    serviceName: str,
+) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
+    """
+    Generate lineage for a list of table queries
+    """
+
+    for table_query in table_queries or []:
+        if not _query_already_processed(metadata, table_query):
+            # Prepare service names for lineage processing
+            service_names = [table_query.serviceName]
+            if processCrossDatabaseLineage and crossDatabaseServiceNames:
+                service_names.extend(crossDatabaseServiceNames)
+
+            lineages: Iterable[Either[AddLineageRequest]] = get_lineage_by_query(
+                metadata,
+                query=table_query.query,
+                service_names=service_names,
+                database_name=table_query.databaseName,
+                schema_name=table_query.databaseSchema,
+                dialect=dialect,
+                timeout_seconds=parsingTimeoutLimit,
+                graph=graph,
+            )
+
+            for lineage_request in lineages or []:
+                queue.put(lineage_request)
+
+                # If we identified lineage properly, ingest the original query
+                if lineage_request.right:
+                    queue.put(
+                        Either(
+                            right=CreateQueryRequest(
+                                query=SqlQuery(table_query.query),
+                                query_type=table_query.query_type,
+                                duration=table_query.duration,
+                                processedLineage=True,
+                                service=serviceName,
+                            )
+                        )
+                    )
+
+
+def view_lineage_processor(
+    views: List[TableView],
+    queue: Queue,
+    metadata: OpenMetadata,
+    service_name: str,
+    connectionType: str,
+    processCrossDatabaseLineage: bool,
+    crossDatabaseServiceNames: List[str],
+    parsingTimeoutLimit: int,
+    overrideViewLineage: bool,
+) -> Iterable[Either[AddLineageRequest]]:
+    """
+    Generate lineage for a list of views
+    """
+    try:
+        for view in views:
+            # Prepare service names for lineage processing
+            service_names = [service_name]
+            if processCrossDatabaseLineage and crossDatabaseServiceNames:
+                service_names.extend(crossDatabaseServiceNames)
+
+            for lineage in get_view_lineage(
+                view=view,
+                metadata=metadata,
+                service_names=service_names,
+                connection_type=connectionType,
+                timeout_seconds=parsingTimeoutLimit,
+            ):
+                if lineage.right is not None:
+                    view_fqn = fqn.build(
+                        metadata=metadata,
+                        entity_type=Table,
+                        service_name=service_name,
+                        database_name=view.db_name,
+                        schema_name=view.schema_name,
+                        table_name=view.table_name,
+                        skip_es_search=True,
+                    )
+                    queue.put(
+                        Either(
+                            right=OMetaLineageRequest(
+                                lineage_request=lineage.right,
+                                override_lineage=overrideViewLineage,
+                                entity_fqn=view_fqn,
+                                entity=Table,
+                            )
+                        )
+                    )
+                else:
+                    queue.put(lineage)
+    except Exception as exc:
+        logger.debug(traceback.format_exc())
+        logger.warning(f"Error processing view {view}: {exc}")