openmetadata-ingestion 1.6.6.1__py3-none-any.whl → 1.6.6.2__py3-none-any.whl

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-03-14T09:22:07+00:00
+#   timestamp: 2025-03-19T07:32:25+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-03-14T09:22:07+00:00
+#   timestamp: 2025-03-19T07:32:25+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-03-14T09:22:07+00:00
+#   timestamp: 2025-03-19T07:32:25+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-03-14T09:22:07+00:00
+#   timestamp: 2025-03-19T07:32:25+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-03-14T09:22:07+00:00
+#   timestamp: 2025-03-19T07:32:25+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-03-14T09:22:07+00:00
+#   timestamp: 2025-03-19T07:32:25+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-03-14T09:22:07+00:00
+#   timestamp: 2025-03-19T07:32:25+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-03-14T09:22:07+00:00
+#   timestamp: 2025-03-19T07:32:25+00:00
 
 from __future__ import annotations
 

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -171,13 +171,9 @@ class OMetaLineageMixin(Generic[T]):
 
         except APIError as err:
             logger.debug(traceback.format_exc())
-            logger.error(
-                "Error %s trying to PUT lineage for %s: %s",
-                err.status_code,
-                data.model_dump_json(),
-                str(err),
-            )
-            raise err
+            error = f"Error {err.status_code} trying to PUT lineage for {data.model_dump_json()}: {str(err)}"
+            logger.error(error)
+            return {"error": error}
 
         from_entity_lineage = self.get_lineage_by_id(
             data.edge.fromEntity.type, str(data.edge.fromEntity.id.root)
@@ -397,6 +393,10 @@ class OMetaLineageMixin(Generic[T]):
                     resp = self.add_lineage(
                         lineage_request.right, check_patch=check_patch
                     )
+                    if resp.get("error"):
+                        logger.error(resp["error"])
+                        continue
+
                     entity_name = resp.get("entity", {}).get("name")
                     for node in resp.get("nodes", []):
                         logger.info(

metadata/ingestion/sink/metadata_rest.py

@@ -250,6 +250,13 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
     @_run_dispatch.register
     def write_lineage(self, add_lineage: AddLineageRequest) -> Either[Dict[str, Any]]:
         created_lineage = self.metadata.add_lineage(add_lineage, check_patch=True)
+        if created_lineage.get("error"):
+            return Either(
+                left=StackTraceError(
+                    name="AddLineageRequestError", error=created_lineage["error"]
+                )
+            )
+
         return Either(right=created_lineage["entity"]["fullyQualifiedName"])
 
     @_run_dispatch.register

metadata/ingestion/source/dashboard/tableau/client.py

@@ -26,6 +26,7 @@ from metadata.ingestion.source.dashboard.tableau import (
     TABLEAU_GET_WORKBOOKS_PARAM_DICT,
 )
 from metadata.ingestion.source.dashboard.tableau.models import (
+    CustomSQLTablesResponse,
     DataSource,
     TableauChart,
     TableauDashboard,
@@ -35,6 +36,7 @@ from metadata.ingestion.source.dashboard.tableau.models import (
 )
 from metadata.ingestion.source.dashboard.tableau.queries import (
     TABLEAU_DATASOURCES_QUERY,
+    TALEAU_GET_CUSTOM_SQL_QUERY,
 )
 from metadata.utils.logger import ometa_logger
 
@@ -80,6 +82,7 @@ class TableauClient:
         self.config = config
         self._client.sign_in().json()
         self.pagination_limit = pagination_limit
+        self.custom_sql_table_queries: Dict[str, List[str]] = {}
 
     @cached_property
     def server_info(self) -> Callable:
@@ -272,5 +275,52 @@ class TableauClient:
             logger.warning("Unable to fetch Data Sources")
         return None
 
+    def get_custom_sql_table_queries(self, dashboard_id: str) -> Optional[List[str]]:
+        """
+        Get custom SQL table queries for a specific dashboard/workbook ID
+        """
+        logger.debug(f"Getting custom SQL table queries for dashboard {dashboard_id}")
+
+        if dashboard_id in self.custom_sql_table_queries:
+            logger.debug(f"Found cached queries for dashboard {dashboard_id}")
+            return self.custom_sql_table_queries[dashboard_id]
+
+        return None
+
+    def cache_custom_sql_tables(self) -> None:
+        """
+        Fetch all custom SQL tables and cache their queries by workbook ID
+        """
+        try:
+            result = self._client.metadata_graphql_query(
+                query=TALEAU_GET_CUSTOM_SQL_QUERY
+            )
+            if not result or not (response_json := result.json()):
+                logger.debug("No result returned from GraphQL query")
+                return
+
+            response = CustomSQLTablesResponse(**response_json)
+            if not response.data:
+                logger.debug("No data found in GraphQL response")
+                return
+
+            for tables in response.data.values():
+                for table in tables:
+                    if not (table.query and table.downstreamWorkbooks):
+                        logger.debug(
+                            f"Skipping table {table} - missing query or workbooks"
+                        )
+                        continue
+
+                    query = table.query
+                    for workbook in table.downstreamWorkbooks:
+                        self.custom_sql_table_queries.setdefault(
+                            workbook.luid, []
+                        ).append(query)
+
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Unable to fetch Custom SQL Tables")
+
     def sign_out(self) -> None:
         self._client.sign_out()

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -114,6 +114,13 @@ class TableauSource(DashboardServiceSource):
             )
         return cls(config, metadata)
 
+    def prepare(self):
+        """
+        Prepare the source before ingestion
+        we will fetch the custom sql tables from the tableau server
+        """
+        self.client.cache_custom_sql_tables()
+
     def get_dashboards_list(self) -> Optional[List[TableauDashboard]]:
         return self.client.get_workbooks()
 
@@ -131,6 +138,11 @@ class TableauSource(DashboardServiceSource):
         # Get the tableau data sources
         dashboard.dataModels = self.client.get_datasources(dashboard_id=dashboard.id)
 
+        # Get custom SQL queries
+        dashboard.custom_sql_queries = self.client.get_custom_sql_table_queries(
+            dashboard_id=dashboard.id
+        )
+
         return dashboard
 
     def get_owner_ref(
@@ -634,6 +646,62 @@ class TableauSource(DashboardServiceSource):
                                 db_service_entity=db_service_entity,
                                 upstream_data_model_entity=data_model_entity,
                             )
+
+                    # Process custom SQL queries if available
+                    if dashboard_details.custom_sql_queries:
+                        for query in dashboard_details.custom_sql_queries:
+                            try:
+                                db_service_entity = None
+                                if db_service_name:
+                                    db_service_entity = self.metadata.get_by_name(
+                                        entity=DatabaseService, fqn=db_service_name
+                                    )
+                                lineage_parser = LineageParser(
+                                    query,
+                                    ConnectionTypeDialectMapper.dialect_of(
+                                        db_service_entity.serviceType.value
+                                    )
+                                    if db_service_entity
+                                    else None,
+                                )
+                                for source_table in lineage_parser.source_tables or []:
+                                    database_schema_table = fqn.split_table_name(
+                                        str(source_table)
+                                    )
+                                    database_name = database_schema_table.get(
+                                        "database"
+                                    )
+                                    if isinstance(
+                                        db_service_entity.connection.config,
+                                        BigQueryConnection,
+                                    ):
+                                        database_name = None
+                                    database_name = get_database_name_for_lineage(
+                                        db_service_entity, database_name
+                                    )
+                                    schema_name = self.check_database_schema_name(
+                                        database_schema_table.get("database_schema")
+                                    )
+                                    table_name = database_schema_table.get("table")
+                                    from_entities = search_table_entities(
+                                        metadata=self.metadata,
+                                        database=database_name,
+                                        service_name=db_service_entity.fullyQualifiedName.root,
+                                        database_schema=schema_name,
+                                        table=table_name,
+                                    )
+                                    for table_entity in from_entities:
+                                        yield self._get_add_lineage_request(
+                                            to_entity=data_model_entity,
+                                            from_entity=table_entity,
+                                            sql=query,
+                                        )
+                            except Exception as err:
+                                logger.debug(traceback.format_exc())
+                                logger.error(
+                                    f"Error processing custom SQL query lineage: {err}"
+                                )
+
                 except Exception as err:
                     yield Either(
                         left=StackTraceError(

metadata/ingestion/source/dashboard/tableau/models.py

@@ -84,15 +84,33 @@ def transform_tags(raw: Union[Dict[str, Any], List[TableauTag]]) -> List[Tableau
     return tags
 
 
+class TableauWorkbook(BaseModel):
+    """
+    Model for downstream workbook information
+    """
+
+    luid: Optional[str] = None
+    name: Optional[str] = None
+
+
 class CustomSQLTable(TableauBaseModel):
     """
     GraphQL API CustomSQLTable schema
     https://help.tableau.com/current/api/metadata_api/en-us/reference/customsqltable.doc.html
     """
 
+    downstreamWorkbooks: Optional[List[TableauWorkbook]] = None
     query: Optional[str] = None
 
 
+class CustomSQLTablesResponse(BaseModel):
+    """
+    Model for the custom SQL tables response
+    """
+
+    data: Dict[str, List[CustomSQLTable]]
+
+
 class UpstreamColumn(BaseModel):
     id: str
     name: Optional[str] = None
@@ -173,6 +191,7 @@ class TableauDashboard(TableauBaseModel):
     webpageUrl: Optional[str] = None
     charts: Optional[List[TableauChart]] = None
     dataModels: List[DataSource] = []
+    custom_sql_queries: Optional[List[str]] = None
 
 
 class TableAndQuery(BaseModel):

metadata/ingestion/source/dashboard/tableau/queries.py

@@ -77,3 +77,17 @@ workbooks(filter:{{luid: "{workbook_id}"}}){{
   }}
 }}
 """
+
+TALEAU_GET_CUSTOM_SQL_QUERY = """
+{
+  customSQLTables {
+    name
+    id
+    downstreamWorkbooks {
+      luid
+      name
+    }
+    query
+  }
+}
+"""

metadata/profiler/orm/converter/base.py

@@ -31,7 +31,7 @@ Base = declarative_base()
 SQA_RESERVED_ATTRIBUTES = ["metadata"]
 
 
-def check_snowflake_case_sensitive(table_service_type, table_or_col) -> Optional[bool]:
+def check_case_sensitive(table_service_type, table_or_col) -> Optional[bool]:
     """Check whether column or table name are not uppercase for snowflake table.
     If so, then force quoting, If not return None to let engine backend handle the logic.
 
@@ -40,7 +40,10 @@ def check_snowflake_case_sensitive(table_service_type, table_or_col) -> Optional
     Return:
         None or True
     """
-    if table_service_type == databaseService.DatabaseServiceType.Snowflake:
+    if table_service_type in {
+        databaseService.DatabaseServiceType.Snowflake,
+        databaseService.DatabaseServiceType.Oracle,
+    }:
         return True if not str(table_or_col).isupper() else None
 
     return None
@@ -81,9 +84,10 @@ def build_orm_col(
     if _quote is not None:
         quote = _quote
     else:
+
         quote = check_if_should_quote_column_name(
             table_service_type
-        ) or check_snowflake_case_sensitive(table_service_type, col.name.root)
+        ) or check_case_sensitive(table_service_type, col.name.root)
 
     return sqlalchemy.Column(
         name=str(col.name.root),
@@ -150,9 +154,7 @@ def ometa_to_sqa_orm(
             "__table_args__": {
                 "schema": orm_schema_name,
                 "extend_existing": True,  # Recreates the table ORM object if it already exists. Useful for testing
-                "quote": check_snowflake_case_sensitive(
-                    table.serviceType, table.name.root
-                )
+                "quote": check_case_sensitive(table.serviceType, table.name.root)
                 or None,
             },
             **cols,

metadata/sampler/partition.py

@@ -26,6 +26,9 @@ from metadata.generated.schema.entity.services.databaseService import (
     DatabaseServiceType,
 )
 from metadata.sampler.models import TableConfig
+from metadata.utils.logger import sampler_logger
+
+logger = sampler_logger()
 
 
 def validate_athena_injected_partitioning(
@@ -164,8 +167,12 @@ def _handle_bigquery_partition(
                 partitionIntegerRangeEnd=10000,
             )
         # TODO: Allow External Hive Partitioning for profiler
-        raise TypeError(
-            f"Unsupported partition type {partition.intervalType}. Skipping table"
+        logger.info(
+            f"Unknown partition type {partition.intervalType}. Cannot infer partition. Partitionning "
+            "will be set to None. If partitionning is required for your table, you can set it "
+            "directly in the table config. More information can be found at "
+            "https://docs.open-metadata.org/latest/how-to-guides/data-quality-observability/profiler/"
+            "workflow#4.-updating-profiler-setting-at-the-table-level"
         )
 
     return None