acryl-datahub 1.2.0.7rc3__py3-none-any.whl → 1.2.0.8rc1__py3-none-any.whl

datahub/sdk/dashboard.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from datetime import datetime
 from typing import Dict, List, Optional, Type, Union
 
+from deprecated.sphinx import deprecated
 from typing_extensions import Self
 
 import datahub.metadata.schema_classes as models
@@ -24,12 +25,14 @@ from datahub.sdk._shared import (
     HasTerms,
     LinksInputType,
     OwnersInputType,
+    ParentContainerInputType,
     TagsInputType,
     TermsInputType,
 )
 from datahub.sdk.chart import Chart
 from datahub.sdk.dataset import Dataset
 from datahub.sdk.entity import Entity, ExtraAspectsType
+from datahub.utilities.sentinels import Unset, unset
 
 
 class Dashboard(
@@ -64,7 +67,7 @@ class Dashboard(
         display_name: Optional[str] = None,
         platform_instance: Optional[DataPlatformInstanceUrnOrStr] = None,
         # Dashboard properties.
-        description: str = "",
+        description: Optional[str] = None,
         external_url: Optional[str] = None,
         dashboard_url: Optional[str] = None,
         custom_properties: Optional[Dict[str, str]] = None,
@@ -74,6 +77,7 @@ class Dashboard(
         charts: Optional[List[Union[ChartUrnOrStr, Chart]]] = None,
         dashboards: Optional[List[Union[DashboardUrnOrStr, Dashboard]]] = None,
         # Standard aspects.
+        parent_container: ParentContainerInputType | Unset = unset,
         subtype: Optional[str] = None,
         owners: Optional[OwnersInputType] = None,
         links: Optional[LinksInputType] = None,
@@ -94,18 +98,7 @@ class Dashboard(
         self._set_platform_instance(platform, platform_instance)
 
         # Initialize DashboardInfoClass with default values
-        dashboard_info = models.DashboardInfoClass(
-            title=display_name or name,
-            description=description or "",
-            lastModified=models.ChangeAuditStampsClass(
-                lastModified=None,
-            ),
-            customProperties={},
-            chartEdges=[],
-            datasetEdges=[],
-            dashboards=[],
-        )
-
+        dashboard_info = self._ensure_dashboard_props(display_name=display_name)
         if last_modified:
             dashboard_info.lastModified = models.ChangeAuditStampsClass(
                 lastModified=models.AuditStampClass(
@@ -114,7 +107,6 @@ class Dashboard(
                 ),
             )
 
-        # Set additional properties
         if description is not None:
             self.set_description(description)
         if display_name is not None:
@@ -129,6 +121,15 @@ class Dashboard(
             self.set_last_modified(last_modified)
         if last_refreshed is not None:
             self.set_last_refreshed(last_refreshed)
+        if input_datasets is not None:
+            self.set_input_datasets(input_datasets)
+        if charts is not None:
+            self.set_charts(charts)
+        if dashboards is not None:
+            self.set_dashboards(dashboards)
+
+        if parent_container is not unset:
+            self._set_container(parent_container)
         if subtype is not None:
             self.set_subtype(subtype)
         if owners is not None:
@@ -141,12 +142,6 @@ class Dashboard(
             self.set_terms(terms)
         if domain is not None:
             self.set_domain(domain)
-        if input_datasets is not None:
-            self.set_input_datasets(input_datasets)
-        if charts is not None:
-            self.set_charts(charts)
-        if dashboards is not None:
-            self.set_dashboards(dashboards)
 
     @classmethod
     def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
@@ -162,11 +157,13 @@ class Dashboard(
         assert isinstance(self._urn, DashboardUrn)
         return self._urn
 
-    def _ensure_dashboard_props(self) -> models.DashboardInfoClass:
+    def _ensure_dashboard_props(
+        self, display_name: Optional[str] = None
+    ) -> models.DashboardInfoClass:
         """Get the dashboard properties safely."""
         return self._setdefault_aspect(
             models.DashboardInfoClass(
-                title=self.urn.dashboard_id,
+                title=display_name or self.urn.dashboard_id,
                 description="",
                 lastModified=models.ChangeAuditStampsClass(
                     lastModified=models.AuditStampClass(
@@ -186,60 +183,52 @@ class Dashboard(
         return self.urn.dashboard_id
 
     @property
+    @deprecated("Use display_name instead", version="1.2.0.7")
     def title(self) -> str:
-        """Get the title of the dashboard."""
-        return self._ensure_dashboard_props().title
+        """Get the display name of the dashboard."""
+        return self.display_name
 
+    @deprecated("Use set_display_name instead", version="1.2.0.7")
     def set_title(self, title: str) -> None:
-        """Set the title of the dashboard."""
-        props = self._ensure_dashboard_props()
-        props.title = title
-        self._set_aspect(props)
+        """Set the display name of the dashboard."""
+        self.set_display_name(title)
 
     @property
     def description(self) -> Optional[str]:
         """Get the description of the dashboard."""
-        props = self._ensure_dashboard_props()
-        return props.description
+        # Because description is a required field, we treat "" as None.
+        return self._ensure_dashboard_props().description or None
 
     def set_description(self, description: str) -> None:
         """Set the description of the dashboard."""
-        props = self._ensure_dashboard_props()
-        props.description = description
-        self._set_aspect(props)
+        self._ensure_dashboard_props().description = description
 
     @property
-    def display_name(self) -> Optional[str]:
+    def display_name(self) -> str:
         """Get the display name of the dashboard."""
-        return self.title
+        return self._ensure_dashboard_props().title
 
     def set_display_name(self, display_name: str) -> None:
         """Set the display name of the dashboard."""
-        self.set_title(display_name)
+        self._ensure_dashboard_props().title = display_name
 
     @property
     def external_url(self) -> Optional[str]:
         """Get the external URL of the dashboard."""
-        props = self._ensure_dashboard_props()
-        return props.externalUrl
+        return self._ensure_dashboard_props().externalUrl
 
     def set_external_url(self, external_url: str) -> None:
         """Set the external URL of the dashboard."""
-        props = self._ensure_dashboard_props()
-        props.externalUrl = external_url
-        self._set_aspect(props)
+        self._ensure_dashboard_props().externalUrl = external_url
 
     @property
     def dashboard_url(self) -> Optional[str]:
         """Get the dashboard URL."""
-        props = self._ensure_dashboard_props()
-        return props.dashboardUrl
+        return self._ensure_dashboard_props().dashboardUrl
 
     def set_dashboard_url(self, dashboard_url: str) -> None:
         """Set the dashboard URL."""
-        props = self._ensure_dashboard_props()
-        props.dashboardUrl = dashboard_url
-        self._set_aspect(props)
+        self._ensure_dashboard_props().dashboardUrl = dashboard_url
 
     @property
     def custom_properties(self) -> Dict[str, str]:
@@ -249,9 +238,7 @@ class Dashboard(
 
     def set_custom_properties(self, custom_properties: Dict[str, str]) -> None:
         """Set the custom properties of the dashboard."""
-        props = self._ensure_dashboard_props()
-        props.customProperties = custom_properties
-        self._set_aspect(props)
+        self._ensure_dashboard_props().customProperties = custom_properties
 
     @property
     def last_modified(self) -> Optional[datetime]:
@@ -263,14 +250,12 @@ class Dashboard(
 
     def set_last_modified(self, last_modified: datetime) -> None:
         """Set the last modification timestamp of the dashboard."""
-        props = self._ensure_dashboard_props()
-        props.lastModified = models.ChangeAuditStampsClass(
+        self._ensure_dashboard_props().lastModified = models.ChangeAuditStampsClass(
             lastModified=models.AuditStampClass(
                 time=int(last_modified.timestamp()),
                 actor="urn:li:corpuser:datahub",
             ),
         )
-        self._set_aspect(props)
 
     @property
     def last_refreshed(self) -> Optional[datetime]:
@@ -284,9 +269,7 @@ class Dashboard(
 
     def set_last_refreshed(self, last_refreshed: datetime) -> None:
         """Set the last refresh timestamp of the dashboard."""
-        props = self._ensure_dashboard_props()
-        props.lastRefreshed = int(last_refreshed.timestamp())
-        self._set_aspect(props)
+        self._ensure_dashboard_props().lastRefreshed = int(last_refreshed.timestamp())
 
     @property
     def input_datasets(self) -> List[DatasetUrn]:
@@ -310,7 +293,6 @@ class Dashboard(
                 dataset_urn = DatasetUrn.from_string(dataset)
             dataset_edges.append(models.EdgeClass(destinationUrn=str(dataset_urn)))
         props.datasetEdges = dataset_edges
-        self._set_aspect(props)
 
     def add_input_dataset(self, input_dataset: Union[DatasetUrnOrStr, Dataset]) -> None:
         """Add an input dataset to the dashboard."""
@@ -326,7 +308,6 @@ class Dashboard(
                 models.EdgeClass(destinationUrn=str(input_dataset_urn))
             )
         props.datasetEdges = dataset_edges
-        self._set_aspect(props)
 
     def remove_input_dataset(
         self, input_dataset: Union[DatasetUrnOrStr, Dataset]
@@ -342,7 +323,6 @@ class Dashboard(
             for edge in (props.datasetEdges or [])
             if edge.destinationUrn != str(input_dataset_urn)
         ]
-        self._set_aspect(props)
 
     @property
     def charts(self) -> List[ChartUrn]:
@@ -363,7 +343,6 @@ class Dashboard(
                 chart_urn = ChartUrn.from_string(chart)
             chart_edges.append(models.EdgeClass(destinationUrn=str(chart_urn)))
         props.chartEdges = chart_edges
-        self._set_aspect(props)
 
     def add_chart(self, chart: Union[ChartUrnOrStr, Chart]) -> None:
         """Add a chart to the dashboard."""
@@ -381,7 +360,6 @@ class Dashboard(
         if str(chart_urn) not in existing_urns:
             chart_edges.append(models.EdgeClass(destinationUrn=str(chart_urn)))
         props.chartEdges = chart_edges
-        self._set_aspect(props)
 
     def remove_chart(self, chart: Union[ChartUrnOrStr, Chart]) -> None:
         """Remove a chart from the dashboard."""
@@ -395,7 +373,6 @@ class Dashboard(
             for edge in (props.chartEdges or [])
             if edge.destinationUrn != str(chart_urn)
         ]
-        self._set_aspect(props)
 
     @property
     def dashboards(self) -> List[DashboardUrn]:
@@ -417,7 +394,6 @@ class Dashboard(
             else:
                 dashboard_urn = DashboardUrn.from_string(dashboard)
             props.dashboards.append(models.EdgeClass(destinationUrn=str(dashboard_urn)))
-        self._set_aspect(props)
 
     def add_dashboard(self, dashboard: Union[DashboardUrnOrStr, Dashboard]) -> None:
         """Add a dashboard to the dashboard."""

datahub/sql_parsing/sqlglot_lineage.py

@@ -543,16 +543,19 @@ def _select_statement_cll(
     root_scope: sqlglot.optimizer.Scope,
     column_resolver: _ColumnResolver,
     output_table: Optional[_TableName],
+    table_name_schema_mapping: Dict[_TableName, SchemaInfo],
+    default_db: Optional[str] = None,
+    default_schema: Optional[str] = None,
 ) -> List[_ColumnLineageInfo]:
     column_lineage: List[_ColumnLineageInfo] = []
 
     try:
-        # List output columns.
         output_columns = [
             (select_col.alias_or_name, select_col) for select_col in statement.selects
         ]
         logger.debug("output columns: %s", [col[0] for col in output_columns])
-        for output_col, original_col_expression in output_columns:
+
+        for output_col, _original_col_expression in output_columns:
             if not output_col or output_col == "*":
                 # If schema information is available, the * will be expanded to the actual columns.
                 # Otherwise, we can't process it.
@@ -576,13 +579,14 @@ def _select_statement_cll(
                 trim_selects=False,
                 # We don't need to pass the schema in here, since we've already qualified the columns.
             )
-            # import pathlib
-            # pathlib.Path("sqlglot.html").write_text(
-            #     str(lineage_node.to_html(dialect=dialect))
-            # )
 
             # Generate SELECT lineage.
-            direct_raw_col_upstreams = _get_direct_raw_col_upstreams(lineage_node)
+            direct_raw_col_upstreams = _get_direct_raw_col_upstreams(
+                lineage_node,
+                dialect,
+                default_db,
+                default_schema,
+            )
 
             # Fuzzy resolve the output column.
             original_col_expression = lineage_node.expression
@@ -601,7 +605,7 @@ def _select_statement_cll(
             if original_col_expression.type:
                 output_col_type = original_col_expression.type
 
-            # Fuzzy resolve upstream columns.
+            # Resolve upstream columns - table names should already be qualified from placeholder processing
             direct_resolved_col_upstreams = {
                 _ColumnRef(
                     table=edge.table,
@@ -706,6 +710,9 @@ def _column_level_lineage(
         root_scope=root_scope,
         column_resolver=column_resolver,
         output_table=downstream_table,
+        table_name_schema_mapping=table_name_schema_mapping,
+        default_db=default_db,
+        default_schema=default_schema,
     )
 
     joins: Optional[List[_JoinInfo]] = None
@@ -726,6 +733,9 @@ def _column_level_lineage(
 
 def _get_direct_raw_col_upstreams(
     lineage_node: sqlglot.lineage.Node,
+    dialect: Optional[sqlglot.Dialect] = None,
+    default_db: Optional[str] = None,
+    default_schema: Optional[str] = None,
 ) -> OrderedSet[_ColumnRef]:
     # Using an OrderedSet here to deduplicate upstreams while preserving "discovery" order.
     direct_raw_col_upstreams: OrderedSet[_ColumnRef] = OrderedSet()
@@ -755,6 +765,53 @@ def _get_direct_raw_col_upstreams(
             direct_raw_col_upstreams.add(
                 _ColumnRef(table=table_ref, column=normalized_col)
             )
+        elif isinstance(node.expression, sqlglot.exp.Placeholder) and node.name != "*":
+            # Handle placeholder expressions from lateral joins.
+            #
+            # In newer SQLGlot versions, columns from lateral subqueries appear as sqlglot.exp.Placeholder
+            # expressions instead of regular table references. This is critical for lateral join column lineage.
+            #
+            # Example: In "SELECT t2.value FROM t1, LATERAL (SELECT value FROM t2 WHERE t1.id = t2.id) t2"
+            # The "t2.value" column reference creates a placeholder with name like '"my_table2"."value"'
+            # which we need to parse to establish the lineage: output.value <- my_table2.value
+            #
+            # Without this handling, lateral join column lineage would be incomplete/missing.
+            try:
+                parsed = sqlglot.parse_one(node.name, dialect=dialect)
+                if isinstance(parsed, sqlglot.exp.Column) and parsed.table:
+                    table_ref = _TableName.from_sqlglot_table(
+                        sqlglot.parse_one(
+                            parsed.table, into=sqlglot.exp.Table, dialect=dialect
+                        )
+                    )
+
+                    # SQLGlot's qualification process doesn't fully qualify placeholder names from lateral joins.
+                    # Even after statement-level qualification, these placeholders remain unqualified (e.g., "t2.value").
+                    # We need this runtime qualification to ensure proper lineage resolution.
+                    # Only qualify if this appears to be a real table reference (not a temporary construct)
+                    if (
+                        not (table_ref.database or table_ref.db_schema)
+                        and dialect is not None
+                    ):
+                        table_ref = table_ref.qualified(
+                            dialect=dialect,
+                            default_db=default_db,
+                            default_schema=default_schema,
+                        )
+
+                    # Extract column name using proper isinstance check
+                    if isinstance(parsed.this, sqlglot.exp.Identifier):
+                        column_name = parsed.this.name
+                    else:
+                        column_name = str(parsed.this)
+                    direct_raw_col_upstreams.add(
+                        _ColumnRef(table=table_ref, column=column_name)
+                    )
+            except Exception as e:
+                logger.debug(
+                    f"Failed to parse placeholder column expression: {node.name} with dialect {dialect}. The exception was: {e}",
+                    exc_info=True,
+                )
         else:
             # This branch doesn't matter. For example, a count(*) column would go here, and
             # we don't get any column-level lineage for that.
@@ -857,7 +914,7 @@ def _get_raw_col_upstreams_for_expression(
             trim_selects=False,
         )
 
-        return _get_direct_raw_col_upstreams(node)
+        return _get_direct_raw_col_upstreams(node, dialect, None, None)
     finally:
         scope.expression = original_expression
 
@@ -872,8 +929,9 @@ def _list_joins(
 
     scope: sqlglot.optimizer.Scope
     for scope in root_scope.traverse():
+        # PART 1: Handle regular explicit JOINs (updated API)
         join: sqlglot.exp.Join
-        for join in scope.find_all(sqlglot.exp.Join):
+        for join in scope.expression.find_all(sqlglot.exp.Join):
             left_side_tables: OrderedSet[_TableName] = OrderedSet()
             from_clause: sqlglot.exp.From
             for from_clause in scope.find_all(sqlglot.exp.From):
@@ -949,6 +1007,36 @@ def _list_joins(
                 )
             )
 
+        # Handle LATERAL constructs
+        for lateral in scope.expression.find_all(sqlglot.exp.Lateral):
+            # Get tables from non-lateral FROM clauses
+            qualified_left: OrderedSet[_TableName] = OrderedSet()
+            for from_clause in scope.find_all(sqlglot.exp.From):
+                if not isinstance(from_clause.this, sqlglot.exp.Lateral):
+                    qualified_left.update(
+                        _get_join_side_tables(from_clause.this, dialect, scope)
+                    )
+
+            # Get tables from lateral subquery
+            qualified_right: OrderedSet[_TableName] = OrderedSet()
+            if lateral.this and isinstance(lateral.this, sqlglot.exp.Subquery):
+                qualified_right.update(
+                    _TableName.from_sqlglot_table(t)
+                    for t in lateral.this.find_all(sqlglot.exp.Table)
+                )
+            qualified_right.update(qualified_left)
+
+            if qualified_left and qualified_right:
+                joins.append(
+                    _JoinInfo(
+                        join_type="LATERAL JOIN",
+                        left_tables=list(qualified_left),
+                        right_tables=list(qualified_right),
+                        on_clause=None,
+                        columns_involved=[],
+                    )
+                )
+
     return joins
 
 
@@ -1186,25 +1274,30 @@ def _translate_internal_joins(
 ) -> List[JoinInfo]:
     joins = []
     for raw_join in raw_joins:
-        joins.append(
-            JoinInfo(
-                join_type=raw_join.join_type,
-                left_tables=[
-                    table_name_urn_mapping[table] for table in raw_join.left_tables
-                ],
-                right_tables=[
-                    table_name_urn_mapping[table] for table in raw_join.right_tables
-                ],
-                on_clause=raw_join.on_clause,
-                columns_involved=[
-                    ColumnRef(
-                        table=table_name_urn_mapping[col.table],
-                        column=col.column,
-                    )
-                    for col in raw_join.columns_involved
-                ],
+        try:
+            joins.append(
+                JoinInfo(
+                    join_type=raw_join.join_type,
+                    left_tables=[
+                        table_name_urn_mapping[table] for table in raw_join.left_tables
+                    ],
+                    right_tables=[
+                        table_name_urn_mapping[table] for table in raw_join.right_tables
+                    ],
+                    on_clause=raw_join.on_clause,
+                    columns_involved=[
+                        ColumnRef(
+                            table=table_name_urn_mapping[col.table],
+                            column=col.column,
+                        )
+                        for col in raw_join.columns_involved
+                    ],
+                )
             )
-        )
+        except KeyError as e:
+            # Skip joins that reference tables we can't resolve (e.g., from CTE subqueries)
+            logger.debug(f"Skipping join with unresolvable table: {e}")
+            continue
     return joins
 
 

datahub/sql_parsing/sqlglot_utils.py

@@ -115,6 +115,8 @@ def _expression_to_string(
     return expression.sql(dialect=get_dialect(platform))
 
 
+PLACEHOLDER_BACKWARD_FINGERPRINT_NORMALIZATION = re.compile(r"(%s|\$\d|\?)")
+
 _BASIC_NORMALIZATION_RULES = {
     # Remove /* */ comments.
     re.compile(r"/\*.*?\*/", re.DOTALL): "",
@@ -130,7 +132,9 @@ _BASIC_NORMALIZATION_RULES = {
     re.compile(r"'[^']*'"): "?",
     # Replace sequences of IN/VALUES with a single placeholder.
     # The r" ?" makes it more robust to uneven spacing.
-    re.compile(r"\b(IN|VALUES)\s*\( ?\?(?:, ?\?)* ?\)", re.IGNORECASE): r"\1 (?)",
+    re.compile(
+        r"\b(IN|VALUES)\s*\( ?(?:%s|\$\d|\?)(?:, ?(?:%s|\$\d|\?))* ?\)", re.IGNORECASE
+    ): r"\1 (?)",
     # Normalize parenthesis spacing.
     re.compile(r"\( "): "(",
     re.compile(r" \)"): ")",
@@ -139,6 +143,9 @@ _BASIC_NORMALIZATION_RULES = {
     # e.g. "col1,col2" -> "col1, col2"
     re.compile(r"\b ,"): ",",
     re.compile(r"\b,\b"): ", ",
+    # MAKE SURE THAT THIS IS AFTER THE ABOVE REPLACEMENT
+    # Replace all versions of placeholders with generic ? placeholder.
+    PLACEHOLDER_BACKWARD_FINGERPRINT_NORMALIZATION: "?",
 }
 _TABLE_NAME_NORMALIZATION_RULES = {
     # Replace UUID-like strings with a placeholder (both - and _ variants).
@@ -262,6 +269,10 @@ def get_query_fingerprint_debug(
         if not fast:
             dialect = get_dialect(platform)
             expression_sql = generalize_query(expression, dialect=dialect)
+            # Normalize placeholders for consistent fingerprinting -> this only needs to be backward compatible with earlier sqglot generated generalized queries where the placeholders were always ?
+            expression_sql = PLACEHOLDER_BACKWARD_FINGERPRINT_NORMALIZATION.sub(
+                "?", expression_sql
+            )
         else:
             expression_sql = generalize_query_fast(expression, dialect=platform)
     except (ValueError, sqlglot.errors.SqlglotError) as e: