acryl-datahub 0.15.0rc15__py3-none-any.whl → 0.15.0rc17__py3-none-any.whl

datahub/ingestion/source/snowflake/snowflake_utils.py

@@ -1,6 +1,6 @@
 import abc
 from functools import cached_property
-from typing import ClassVar, Literal, Optional, Tuple
+from typing import ClassVar, List, Literal, Optional, Tuple
 
 from datahub.configuration.pattern_utils import is_schema_allowed
 from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance
@@ -184,6 +184,46 @@ def _is_sys_table(table_name: str) -> bool:
     return table_name.lower().startswith("sys$")
 
 
+def _split_qualified_name(qualified_name: str) -> List[str]:
+    """
+    Split a qualified name into its constituent parts.
+
+    >>> _split_qualified_name("db.my_schema.my_table")
+    ['db', 'my_schema', 'my_table']
+    >>> _split_qualified_name('"db"."my_schema"."my_table"')
+    ['db', 'my_schema', 'my_table']
+    >>> _split_qualified_name('TEST_DB.TEST_SCHEMA."TABLE.WITH.DOTS"')
+    ['TEST_DB', 'TEST_SCHEMA', 'TABLE.WITH.DOTS']
+    >>> _split_qualified_name('TEST_DB."SCHEMA.WITH.DOTS".MY_TABLE')
+    ['TEST_DB', 'SCHEMA.WITH.DOTS', 'MY_TABLE']
+    """
+
+    # Fast path - no quotes.
+    if '"' not in qualified_name:
+        return qualified_name.split(".")
+
+    # First pass - split on dots that are not inside quotes.
+    in_quote = False
+    parts: List[List[str]] = [[]]
+    for char in qualified_name:
+        if char == '"':
+            in_quote = not in_quote
+        elif char == "." and not in_quote:
+            parts.append([])
+        else:
+            parts[-1].append(char)
+
+    # Second pass - remove outer pairs of quotes.
+    result = []
+    for part in parts:
+        if len(part) > 2 and part[0] == '"' and part[-1] == '"':
+            part = part[1:-1]
+
+        result.append("".join(part))
+
+    return result
+
+
 # Qualified Object names from snowflake audit logs have quotes for for snowflake quoted identifiers,
 # For example "test-database"."test-schema".test_table
 # whereas we generate urns without quotes even for quoted identifiers for backward compatibility
@@ -192,7 +232,7 @@ def _is_sys_table(table_name: str) -> bool:
 def _cleanup_qualified_name(
     qualified_name: str, structured_reporter: SourceReport
 ) -> str:
-    name_parts = qualified_name.split(".")
+    name_parts = _split_qualified_name(qualified_name)
     if len(name_parts) != 3:
         if not _is_sys_table(qualified_name):
             structured_reporter.info(
@@ -203,9 +243,9 @@ def _cleanup_qualified_name(
             )
         return qualified_name.replace('"', "")
     return _combine_identifier_parts(
-        db_name=name_parts[0].strip('"'),
-        schema_name=name_parts[1].strip('"'),
-        table_name=name_parts[2].strip('"'),
+        db_name=name_parts[0],
+        schema_name=name_parts[1],
+        table_name=name_parts[2],
     )
 
 

datahub/ingestion/source/snowflake/snowflake_v2.py

@@ -17,6 +17,9 @@ from datahub.ingestion.api.decorators import (
     support_status,
 )
 from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage
+from datahub.ingestion.api.incremental_properties_helper import (
+    auto_incremental_properties,
+)
 from datahub.ingestion.api.source import (
     CapabilityReport,
     MetadataWorkUnitProcessor,
@@ -446,6 +449,9 @@ class SnowflakeV2Source(
             functools.partial(
                 auto_incremental_lineage, self.config.incremental_lineage
             ),
+            functools.partial(
+                auto_incremental_properties, self.config.incremental_properties
+            ),
             StaleEntityRemovalHandler.create(
                 self, self.config, self.ctx
             ).workunit_processor,

datahub/ingestion/source/state/redundant_run_skip_handler.py

@@ -69,7 +69,7 @@ class RedundantRunSkipHandler(
         platform: Optional[str] = None
         source_class = type(self.source)
         if hasattr(source_class, "get_platform_name"):
-            platform = source_class.get_platform_name()  # type: ignore
+            platform = source_class.get_platform_name()
 
         # Default name for everything else
         job_name_suffix = self.get_job_name_suffix()

datahub/ingestion/source/tableau/tableau.py

@@ -353,7 +353,7 @@ class TableauConfig(
 
     project_path_separator: str = Field(
         default="/",
-        description="The separator used for the project_pattern field between project names. By default, we use a slash. "
+        description="The separator used for the project_path_pattern field between project names. By default, we use a slash. "
         "You can change this if your Tableau projects contain slashes in their names, and you'd like to filter by project.",
     )
 
@@ -959,19 +959,36 @@ class TableauSiteSource:
         return is_allowed
 
     def _is_denied_project(self, project: TableauProject) -> bool:
-        # Either project name or project path should exist in deny
-        for deny_pattern in self.config.project_pattern.deny:
-            # Either name or project path is denied
-            if re.match(
-                deny_pattern, project.name, self.config.project_pattern.regex_flags
-            ) or re.match(
-                deny_pattern,
-                self._get_project_path(project),
-                self.config.project_pattern.regex_flags,
-            ):
-                return True
-        logger.info(f"project({project.name}) is not denied as per project_pattern")
-        return False
+        """
+        Why use an explicit denial check instead of the `AllowDenyPattern.allowed` method?
+
+        Consider a scenario where a Tableau site contains four projects: A, B, C, and D, with the following hierarchical relationship:
+
+        - **A**
+          - **B** (Child of A)
+          - **C** (Child of A)
+        - **D**
+
+        In this setup:
+
+        - `project_pattern` is configured with `allow: ["A"]` and `deny: ["B"]`.
+        - `extract_project_hierarchy` is set to `True`.
+
+        The goal is to extract assets from project A and its children while explicitly denying the child project B.
+
+        If we rely solely on the `project_pattern.allowed()` method, project C's assets will not be ingested.
+        This happens because project C is not explicitly included in the `allow` list, nor is it part of the `deny` list.
+        However, since `extract_project_hierarchy` is enabled, project C should ideally be included in the ingestion process unless explicitly denied.
+
+        To address this, the function explicitly checks the deny regex to ensure that project C’s assets are ingested if it is not specifically denied in the deny list. This approach ensures that the hierarchy is respected while adhering to the configured allow/deny rules.
+        """
+
+        # Either project_pattern or project_path_pattern is set in a recipe
+        # TableauConfig.projects_backward_compatibility ensures that at least one of these properties is configured.
+
+        return self.config.project_pattern.denied(
+            project.name
+        ) or self.config.project_path_pattern.denied(self._get_project_path(project))
 
     def _init_tableau_project_registry(self, all_project_map: dict) -> None:
         list_of_skip_projects: List[TableauProject] = []
@@ -999,9 +1016,11 @@ class TableauSiteSource:
             for project in list_of_skip_projects:
                 if (
                     project.parent_id in projects_to_ingest
-                    and self._is_denied_project(project) is False
+                    and not self._is_denied_project(project)
                 ):
-                    logger.debug(f"Project {project.name} is added in project registry")
+                    logger.debug(
+                        f"Project {project.name} is added in project registry as it's a child project and not explicitly denied in `deny` list"
+                    )
                     projects_to_ingest[project.id] = project
 
         # We rely on automatic browse paths (v2) when creating containers. That's why we need to sort the projects here.

datahub/ingestion/source/tableau/tableau_common.py

@@ -979,7 +979,6 @@ def get_filter_pages(query_filter: dict, page_size: int) -> List[dict]:
         len(query_filter.keys()) == 1
         and query_filter.get(c.ID_WITH_IN)
         and isinstance(query_filter[c.ID_WITH_IN], list)
-        and len(query_filter[c.ID_WITH_IN]) > 100 * page_size
     ):
         ids = query_filter[c.ID_WITH_IN]
         filter_pages = [

datahub/ingestion/source/unity/source.py

@@ -556,6 +556,8 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
                 )
 
         if table_props:
+            # TODO: use auto_incremental_properties workunit processor instead
+            # Consider enabling incremental_properties by default
             patch_builder = create_dataset_props_patch_builder(dataset_urn, table_props)
             for patch_mcp in patch_builder.build():
                 yield MetadataWorkUnit(

datahub/ingestion/source/unity/usage.py

@@ -7,7 +7,6 @@ from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, Set,
 
 import pyspark
 from databricks.sdk.service.sql import QueryStatementType
-from sqllineage.runner import LineageRunner
 
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.source_helpers import auto_empty_dataset_usage_statistics
@@ -22,7 +21,9 @@ from datahub.ingestion.source.unity.proxy_types import (
 from datahub.ingestion.source.unity.report import UnityCatalogReport
 from datahub.ingestion.source.usage.usage_common import UsageAggregator
 from datahub.metadata.schema_classes import OperationClass
+from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
 from datahub.sql_parsing.sqlglot_utils import get_query_fingerprint
+from datahub.utilities.urns.dataset_urn import DatasetUrn
 
 logger = logging.getLogger(__name__)
 
@@ -48,6 +49,7 @@ class UnityCatalogUsageExtractor:
     proxy: UnityCatalogApiProxy
     table_urn_builder: Callable[[TableReference], str]
     user_urn_builder: Callable[[str], str]
+    platform: str = "databricks"
 
     def __post_init__(self):
         self.usage_aggregator = UsageAggregator[TableReference](self.config)
@@ -173,7 +175,7 @@ class UnityCatalogUsageExtractor:
         self, query: Query, table_map: TableMap
     ) -> Optional[QueryTableInfo]:
         with self.report.usage_perf_report.sql_parsing_timer:
-            table_info = self._parse_query_via_lineage_runner(query.query_text)
+            table_info = self._parse_query_via_sqlglot(query.query_text)
             if table_info is None and query.statement_type == QueryStatementType.SELECT:
                 with self.report.usage_perf_report.spark_sql_parsing_timer:
                     table_info = self._parse_query_via_spark_sql_plan(query.query_text)
@@ -191,26 +193,33 @@ class UnityCatalogUsageExtractor:
                     ),
                 )
 
-    def _parse_query_via_lineage_runner(self, query: str) -> Optional[StringTableInfo]:
+    def _parse_query_via_sqlglot(self, query: str) -> Optional[StringTableInfo]:
         try:
-            runner = LineageRunner(query)
+            sql_parser_in_tables = create_lineage_sql_parsed_result(
+                query=query,
+                default_db=None,
+                platform=self.platform,
+                env=self.config.env,
+                platform_instance=None,
+            )
+
             return GenericTableInfo(
                 source_tables=[
-                    self._parse_sqllineage_table(table)
-                    for table in runner.source_tables
+                    self._parse_sqlglot_table(table)
+                    for table in sql_parser_in_tables.in_tables
                 ],
                 target_tables=[
-                    self._parse_sqllineage_table(table)
-                    for table in runner.target_tables
+                    self._parse_sqlglot_table(table)
+                    for table in sql_parser_in_tables.out_tables
                 ],
             )
         except Exception as e:
-            logger.info(f"Could not parse query via lineage runner, {query}: {e!r}")
+            logger.info(f"Could not parse query via sqlglot, {query}: {e!r}")
             return None
 
     @staticmethod
-    def _parse_sqllineage_table(sqllineage_table: object) -> str:
-        full_table_name = str(sqllineage_table)
+    def _parse_sqlglot_table(table_urn: str) -> str:
+        full_table_name = DatasetUrn.from_string(table_urn).name
         default_schema = "<default>."
         if full_table_name.startswith(default_schema):
             return full_table_name[len(default_schema) :]

datahub/metadata/_schema_classes.py

@@ -23267,7 +23267,7 @@ class StructuredPropertyDefinitionClass(_Aspect):
     
     @property
     def lastModified(self) -> Union[None, "AuditStampClass"]:
-        """Created Audit stamp"""
+        """Last Modified Audit stamp"""
         return self._inner_dict.get('lastModified')  # type: ignore
     
     @lastModified.setter
@@ -23280,7 +23280,7 @@ class StructuredPropertyKeyClass(_Aspect):
 
 
     ASPECT_NAME = 'structuredPropertyKey'
-    ASPECT_INFO = {'keyForEntity': 'structuredProperty', 'entityCategory': 'core', 'entityAspects': ['propertyDefinition', 'institutionalMemory', 'status'], 'entityDoc': 'Structured Property represents a property meant for extending the core model of a logical entity'}
+    ASPECT_INFO = {'keyForEntity': 'structuredProperty', 'entityCategory': 'core', 'entityAspects': ['propertyDefinition', 'structuredPropertySettings', 'institutionalMemory', 'status'], 'entityDoc': 'Structured Property represents a property meant for extending the core model of a logical entity'}
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.structured.StructuredPropertyKey")
 
     def __init__(self,
@@ -23304,6 +23304,122 @@ class StructuredPropertyKeyClass(_Aspect):
         self._inner_dict['id'] = value
     
     
+class StructuredPropertySettingsClass(_Aspect):
+    """Settings specific to a structured property entity"""
+
+
+    ASPECT_NAME = 'structuredPropertySettings'
+    ASPECT_INFO = {}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.structured.StructuredPropertySettings")
+
+    def __init__(self,
+        isHidden: Optional[bool]=None,
+        showInSearchFilters: Optional[bool]=None,
+        showInAssetSummary: Optional[bool]=None,
+        showAsAssetBadge: Optional[bool]=None,
+        showInColumnsTable: Optional[bool]=None,
+        lastModified: Union[None, "AuditStampClass"]=None,
+    ):
+        super().__init__()
+        
+        if isHidden is None:
+            # default: False
+            self.isHidden = self.RECORD_SCHEMA.fields_dict["isHidden"].default
+        else:
+            self.isHidden = isHidden
+        if showInSearchFilters is None:
+            # default: False
+            self.showInSearchFilters = self.RECORD_SCHEMA.fields_dict["showInSearchFilters"].default
+        else:
+            self.showInSearchFilters = showInSearchFilters
+        if showInAssetSummary is None:
+            # default: False
+            self.showInAssetSummary = self.RECORD_SCHEMA.fields_dict["showInAssetSummary"].default
+        else:
+            self.showInAssetSummary = showInAssetSummary
+        if showAsAssetBadge is None:
+            # default: False
+            self.showAsAssetBadge = self.RECORD_SCHEMA.fields_dict["showAsAssetBadge"].default
+        else:
+            self.showAsAssetBadge = showAsAssetBadge
+        if showInColumnsTable is None:
+            # default: False
+            self.showInColumnsTable = self.RECORD_SCHEMA.fields_dict["showInColumnsTable"].default
+        else:
+            self.showInColumnsTable = showInColumnsTable
+        self.lastModified = lastModified
+    
+    def _restore_defaults(self) -> None:
+        self.isHidden = self.RECORD_SCHEMA.fields_dict["isHidden"].default
+        self.showInSearchFilters = self.RECORD_SCHEMA.fields_dict["showInSearchFilters"].default
+        self.showInAssetSummary = self.RECORD_SCHEMA.fields_dict["showInAssetSummary"].default
+        self.showAsAssetBadge = self.RECORD_SCHEMA.fields_dict["showAsAssetBadge"].default
+        self.showInColumnsTable = self.RECORD_SCHEMA.fields_dict["showInColumnsTable"].default
+        self.lastModified = self.RECORD_SCHEMA.fields_dict["lastModified"].default
+    
+    
+    @property
+    def isHidden(self) -> bool:
+        """Whether or not this asset should be hidden in the main application"""
+        return self._inner_dict.get('isHidden')  # type: ignore
+    
+    @isHidden.setter
+    def isHidden(self, value: bool) -> None:
+        self._inner_dict['isHidden'] = value
+    
+    
+    @property
+    def showInSearchFilters(self) -> bool:
+        """Whether or not this asset should be displayed as a search filter"""
+        return self._inner_dict.get('showInSearchFilters')  # type: ignore
+    
+    @showInSearchFilters.setter
+    def showInSearchFilters(self, value: bool) -> None:
+        self._inner_dict['showInSearchFilters'] = value
+    
+    
+    @property
+    def showInAssetSummary(self) -> bool:
+        """Whether or not this asset should be displayed in the asset sidebar"""
+        return self._inner_dict.get('showInAssetSummary')  # type: ignore
+    
+    @showInAssetSummary.setter
+    def showInAssetSummary(self, value: bool) -> None:
+        self._inner_dict['showInAssetSummary'] = value
+    
+    
+    @property
+    def showAsAssetBadge(self) -> bool:
+        """Whether or not this asset should be displayed as an asset badge on other
+    asset's headers"""
+        return self._inner_dict.get('showAsAssetBadge')  # type: ignore
+    
+    @showAsAssetBadge.setter
+    def showAsAssetBadge(self, value: bool) -> None:
+        self._inner_dict['showAsAssetBadge'] = value
+    
+    
+    @property
+    def showInColumnsTable(self) -> bool:
+        """Whether or not this asset should be displayed as a column in the schema field table
+    in a Dataset's "Columns" tab."""
+        return self._inner_dict.get('showInColumnsTable')  # type: ignore
+    
+    @showInColumnsTable.setter
+    def showInColumnsTable(self, value: bool) -> None:
+        self._inner_dict['showInColumnsTable'] = value
+    
+    
+    @property
+    def lastModified(self) -> Union[None, "AuditStampClass"]:
+        """Last Modified Audit stamp"""
+        return self._inner_dict.get('lastModified')  # type: ignore
+    
+    @lastModified.setter
+    def lastModified(self, value: Union[None, "AuditStampClass"]) -> None:
+        self._inner_dict['lastModified'] = value
+    
+    
 class StructuredPropertyValueAssignmentClass(DictWrapper):
     # No docs available.
     
@@ -24775,6 +24891,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.structured.StructuredProperties': StructuredPropertiesClass,
     'com.linkedin.pegasus2avro.structured.StructuredPropertyDefinition': StructuredPropertyDefinitionClass,
     'com.linkedin.pegasus2avro.structured.StructuredPropertyKey': StructuredPropertyKeyClass,
+    'com.linkedin.pegasus2avro.structured.StructuredPropertySettings': StructuredPropertySettingsClass,
     'com.linkedin.pegasus2avro.structured.StructuredPropertyValueAssignment': StructuredPropertyValueAssignmentClass,
     'com.linkedin.pegasus2avro.tag.TagProperties': TagPropertiesClass,
     'com.linkedin.pegasus2avro.telemetry.TelemetryClientId': TelemetryClientIdClass,
@@ -25240,6 +25357,7 @@ __SCHEMA_TYPES = {
     'StructuredProperties': StructuredPropertiesClass,
     'StructuredPropertyDefinition': StructuredPropertyDefinitionClass,
     'StructuredPropertyKey': StructuredPropertyKeyClass,
+    'StructuredPropertySettings': StructuredPropertySettingsClass,
     'StructuredPropertyValueAssignment': StructuredPropertyValueAssignmentClass,
     'TagProperties': TagPropertiesClass,
     'TelemetryClientId': TelemetryClientIdClass,
@@ -25336,6 +25454,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     QuerySubjectsClass,
     StructuredPropertyKeyClass,
     StructuredPropertyDefinitionClass,
+    StructuredPropertySettingsClass,
     StructuredPropertiesClass,
     GlobalSettingsInfoClass,
     DataHubRetentionConfigClass,
@@ -25548,6 +25667,7 @@ class AspectBag(TypedDict, total=False):
     querySubjects: QuerySubjectsClass
     structuredPropertyKey: StructuredPropertyKeyClass
     propertyDefinition: StructuredPropertyDefinitionClass
+    structuredPropertySettings: StructuredPropertySettingsClass
     structuredProperties: StructuredPropertiesClass
     globalSettingsInfo: GlobalSettingsInfoClass
     dataHubRetentionConfig: DataHubRetentionConfigClass

datahub/metadata/com/linkedin/pegasus2avro/structured/__init__.py

@@ -12,6 +12,7 @@ from .....schema_classes import PropertyValueClass
 from .....schema_classes import StructuredPropertiesClass
 from .....schema_classes import StructuredPropertyDefinitionClass
 from .....schema_classes import StructuredPropertyKeyClass
+from .....schema_classes import StructuredPropertySettingsClass
 from .....schema_classes import StructuredPropertyValueAssignmentClass
 
 
@@ -20,6 +21,7 @@ PropertyValue = PropertyValueClass
 StructuredProperties = StructuredPropertiesClass
 StructuredPropertyDefinition = StructuredPropertyDefinitionClass
 StructuredPropertyKey = StructuredPropertyKeyClass
+StructuredPropertySettings = StructuredPropertySettingsClass
 StructuredPropertyValueAssignment = StructuredPropertyValueAssignmentClass
 
 # fmt: on

datahub/metadata/schema.avsc

@@ -2995,6 +2995,7 @@
       "entityCategory": "core",
       "entityAspects": [
         "propertyDefinition",
+        "structuredPropertySettings",
         "institutionalMemory",
         "status"
       ],
@@ -3328,10 +3329,81 @@
         ],
         "name": "lastModified",
         "default": null,
-        "doc": "Created Audit stamp"
+        "doc": "Last Modified Audit stamp"
       }
     ]
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "structuredPropertySettings"
+    },
+    "name": "StructuredPropertySettings",
+    "namespace": "com.linkedin.pegasus2avro.structured",
+    "fields": [
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "isHidden",
+        "default": false,
+        "doc": "Whether or not this asset should be hidden in the main application"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInSearchFilters",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as a search filter"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInAssetSummary",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed in the asset sidebar"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showAsAssetBadge",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as an asset badge on other\nasset's headers"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInColumnsTable",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as a column in the schema field table\nin a Dataset's \"Columns\" tab."
+      },
+      {
+        "Searchable": {
+          "/time": {
+            "fieldName": "lastModifiedSettings",
+            "fieldType": "DATETIME"
+          }
+        },
+        "type": [
+          "null",
+          "com.linkedin.pegasus2avro.common.AuditStamp"
+        ],
+        "name": "lastModified",
+        "default": null,
+        "doc": "Last Modified Audit stamp"
+      }
+    ],
+    "doc": "Settings specific to a structured property entity"
+  },
   {
     "type": "record",
     "Aspect": {

datahub/metadata/schemas/StructuredPropertyDefinition.avsc

@@ -359,7 +359,7 @@
       ],
       "name": "lastModified",
       "default": null,
-      "doc": "Created Audit stamp"
+      "doc": "Last Modified Audit stamp"
     }
   ]
 }

datahub/metadata/schemas/StructuredPropertyKey.avsc

@@ -6,6 +6,7 @@
     "entityCategory": "core",
     "entityAspects": [
       "propertyDefinition",
+      "structuredPropertySettings",
       "institutionalMemory",
       "status"
     ],