acryl-datahub 1.1.0.5rc6__py3-none-any.whl → 1.1.0.5rc8__py3-none-any.whl

datahub/metadata/_internal_schema_classes.py

@@ -14127,26 +14127,39 @@ class CorpUserHomePageSettingsClass(DictWrapper):
     
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.identity.CorpUserHomePageSettings")
     def __init__(self,
-        pageTemplate: str,
+        pageTemplate: Union[None, str]=None,
+        dismissedAnnouncements: Union[None, List[str]]=None,
     ):
         super().__init__()
         
         self.pageTemplate = pageTemplate
+        self.dismissedAnnouncements = dismissedAnnouncements
     
     def _restore_defaults(self) -> None:
-        self.pageTemplate = str()
+        self.pageTemplate = self.RECORD_SCHEMA.fields_dict["pageTemplate"].default
+        self.dismissedAnnouncements = self.RECORD_SCHEMA.fields_dict["dismissedAnnouncements"].default
     
     
     @property
-    def pageTemplate(self) -> str:
+    def pageTemplate(self) -> Union[None, str]:
         """The page template that will be rendered in the UI by default for this user"""
         return self._inner_dict.get('pageTemplate')  # type: ignore
     
     @pageTemplate.setter
-    def pageTemplate(self, value: str) -> None:
+    def pageTemplate(self, value: Union[None, str]) -> None:
         self._inner_dict['pageTemplate'] = value
     
     
+    @property
+    def dismissedAnnouncements(self) -> Union[None, List[str]]:
+        """The list of announcement urns that have been dismissed by the user"""
+        return self._inner_dict.get('dismissedAnnouncements')  # type: ignore
+    
+    @dismissedAnnouncements.setter
+    def dismissedAnnouncements(self, value: Union[None, List[str]]) -> None:
+        self._inner_dict['dismissedAnnouncements'] = value
+    
+    
 class CorpUserInfoClass(_Aspect):
     """Linkedin corp user information"""
 
@@ -24394,6 +24407,59 @@ class DataHubSecretValueClass(_Aspect):
         self._inner_dict['created'] = value
     
     
+class ApplicationsSettingsClass(DictWrapper):
+    # No docs available.
+    
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.settings.global.ApplicationsSettings")
+    def __init__(self,
+        enabled: bool,
+        config: Union[None, str]=None,
+        configVersion: Union[None, str]=None,
+    ):
+        super().__init__()
+        
+        self.enabled = enabled
+        self.config = config
+        self.configVersion = configVersion
+    
+    def _restore_defaults(self) -> None:
+        self.enabled = bool()
+        self.config = self.RECORD_SCHEMA.fields_dict["config"].default
+        self.configVersion = self.RECORD_SCHEMA.fields_dict["configVersion"].default
+    
+    
+    @property
+    def enabled(self) -> bool:
+        # No docs available.
+        return self._inner_dict.get('enabled')  # type: ignore
+    
+    @enabled.setter
+    def enabled(self, value: bool) -> None:
+        self._inner_dict['enabled'] = value
+    
+    
+    @property
+    def config(self) -> Union[None, str]:
+        """The configuration for the feature, in JSON format."""
+        return self._inner_dict.get('config')  # type: ignore
+    
+    @config.setter
+    def config(self, value: Union[None, str]) -> None:
+        self._inner_dict['config'] = value
+    
+    
+    @property
+    def configVersion(self) -> Union[None, str]:
+        """The version of the configuration schema that has been used to serialize
+           the config.
+    If not provided, the version is assumed to be the latest version."""
+        return self._inner_dict.get('configVersion')  # type: ignore
+    
+    @configVersion.setter
+    def configVersion(self, value: Union[None, str]) -> None:
+        self._inner_dict['configVersion'] = value
+    
+    
 class DocPropagationFeatureSettingsClass(DictWrapper):
     # No docs available.
     
@@ -24502,6 +24568,7 @@ class GlobalSettingsInfoClass(_Aspect):
         views: Union[None, "GlobalViewsSettingsClass"]=None,
         docPropagation: Optional[Union["DocPropagationFeatureSettingsClass", None]]=None,
         homePage: Union[None, "GlobalHomePageSettingsClass"]=None,
+        applications: Union[None, "ApplicationsSettingsClass"]=None,
     ):
         super().__init__()
         
@@ -24513,12 +24580,14 @@ class GlobalSettingsInfoClass(_Aspect):
         else:
             self.docPropagation = docPropagation
         self.homePage = homePage
+        self.applications = applications
     
     def _restore_defaults(self) -> None:
         self.sso = self.RECORD_SCHEMA.fields_dict["sso"].default
         self.views = self.RECORD_SCHEMA.fields_dict["views"].default
         self.docPropagation = _json_converter.from_json_object(self.RECORD_SCHEMA.fields_dict["docPropagation"].default, writers_schema=self.RECORD_SCHEMA.fields_dict["docPropagation"].type)
         self.homePage = self.RECORD_SCHEMA.fields_dict["homePage"].default
+        self.applications = self.RECORD_SCHEMA.fields_dict["applications"].default
     
     
     @property
@@ -24561,6 +24630,16 @@ class GlobalSettingsInfoClass(_Aspect):
         self._inner_dict['homePage'] = value
     
     
+    @property
+    def applications(self) -> Union[None, "ApplicationsSettingsClass"]:
+        """Settings related to applications. If not enabled, applications won't show up in navigation"""
+        return self._inner_dict.get('applications')  # type: ignore
+    
+    @applications.setter
+    def applications(self, value: Union[None, "ApplicationsSettingsClass"]) -> None:
+        self._inner_dict['applications'] = value
+    
+    
 class GlobalViewsSettingsClass(DictWrapper):
     """Settings for DataHub Views feature."""
     
@@ -27086,6 +27165,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.schemafield.SchemaFieldAliases': SchemaFieldAliasesClass,
     'com.linkedin.pegasus2avro.schemafield.SchemaFieldInfo': SchemaFieldInfoClass,
     'com.linkedin.pegasus2avro.secret.DataHubSecretValue': DataHubSecretValueClass,
+    'com.linkedin.pegasus2avro.settings.global.ApplicationsSettings': ApplicationsSettingsClass,
     'com.linkedin.pegasus2avro.settings.global.DocPropagationFeatureSettings': DocPropagationFeatureSettingsClass,
     'com.linkedin.pegasus2avro.settings.global.GlobalHomePageSettings': GlobalHomePageSettingsClass,
     'com.linkedin.pegasus2avro.settings.global.GlobalSettingsInfo': GlobalSettingsInfoClass,
@@ -27592,6 +27672,7 @@ __SCHEMA_TYPES = {
     'SchemaFieldAliases': SchemaFieldAliasesClass,
     'SchemaFieldInfo': SchemaFieldInfoClass,
     'DataHubSecretValue': DataHubSecretValueClass,
+    'ApplicationsSettings': ApplicationsSettingsClass,
     'DocPropagationFeatureSettings': DocPropagationFeatureSettingsClass,
     'GlobalHomePageSettings': GlobalHomePageSettingsClass,
     'GlobalSettingsInfo': GlobalSettingsInfoClass,

datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py

@@ -7,6 +7,7 @@
 # pylint: skip-file
 # fmt: off
 # isort: skip_file
+from ......schema_classes import ApplicationsSettingsClass
 from ......schema_classes import DocPropagationFeatureSettingsClass
 from ......schema_classes import GlobalHomePageSettingsClass
 from ......schema_classes import GlobalSettingsInfoClass
@@ -15,6 +16,7 @@ from ......schema_classes import OidcSettingsClass
 from ......schema_classes import SsoSettingsClass
 
 
+ApplicationsSettings = ApplicationsSettingsClass
 DocPropagationFeatureSettings = DocPropagationFeatureSettingsClass
 GlobalHomePageSettings = GlobalHomePageSettingsClass
 GlobalSettingsInfo = GlobalSettingsInfoClass

datahub/metadata/schema.avsc

@@ -751,9 +751,25 @@
                   "class": "com.linkedin.pegasus2avro.common.urn.Urn"
                 },
                 "Urn": "Urn",
-                "type": "string",
+                "type": [
+                  "null",
+                  "string"
+                ],
                 "name": "pageTemplate",
+                "default": null,
                 "doc": "The page template that will be rendered in the UI by default for this user"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "array",
+                    "items": "string"
+                  }
+                ],
+                "name": "dismissedAnnouncements",
+                "default": null,
+                "doc": "The list of announcement urns that have been dismissed by the user"
               }
             ],
             "doc": "Settings related to the home page for a user"
@@ -18531,6 +18547,43 @@
         "name": "homePage",
         "default": null,
         "doc": "Global settings related to the home page for an instance"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "record",
+            "name": "ApplicationsSettings",
+            "namespace": "com.linkedin.pegasus2avro.settings.global",
+            "fields": [
+              {
+                "type": "boolean",
+                "name": "enabled"
+              },
+              {
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "config",
+                "default": null,
+                "doc": "The configuration for the feature, in JSON format."
+              },
+              {
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "configVersion",
+                "default": null,
+                "doc": "The version of the configuration schema that has been used to serialize\n       the config.\nIf not provided, the version is assumed to be the latest version."
+              }
+            ]
+          }
+        ],
+        "name": "applications",
+        "default": null,
+        "doc": "Settings related to applications. If not enabled, applications won't show up in navigation"
       }
     ],
     "doc": "DataHub Global platform settings. Careful - these should not be modified by the outside world!"

datahub/metadata/schemas/CorpUserSettings.avsc

@@ -172,10 +172,26 @@
               "java": {
                 "class": "com.linkedin.pegasus2avro.common.urn.Urn"
               },
-              "type": "string",
+              "type": [
+                "null",
+                "string"
+              ],
               "name": "pageTemplate",
+              "default": null,
               "doc": "The page template that will be rendered in the UI by default for this user",
               "Urn": "Urn"
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "array",
+                  "items": "string"
+                }
+              ],
+              "name": "dismissedAnnouncements",
+              "default": null,
+              "doc": "The list of announcement urns that have been dismissed by the user"
             }
           ],
           "doc": "Settings related to the home page for a user"

datahub/metadata/schemas/GlobalSettingsInfo.avsc

@@ -307,6 +307,43 @@
       "name": "homePage",
       "default": null,
       "doc": "Global settings related to the home page for an instance"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "ApplicationsSettings",
+          "namespace": "com.linkedin.pegasus2avro.settings.global",
+          "fields": [
+            {
+              "type": "boolean",
+              "name": "enabled"
+            },
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "config",
+              "default": null,
+              "doc": "The configuration for the feature, in JSON format."
+            },
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "configVersion",
+              "default": null,
+              "doc": "The version of the configuration schema that has been used to serialize\n       the config.\nIf not provided, the version is assumed to be the latest version."
+            }
+          ]
+        }
+      ],
+      "name": "applications",
+      "default": null,
+      "doc": "Settings related to applications. If not enabled, applications won't show up in navigation"
     }
   ],
   "doc": "DataHub Global platform settings. Careful - these should not be modified by the outside world!"

datahub/sdk/lineage_client.py

@@ -478,6 +478,7 @@ class LineageClient:
         env: str = "PROD",
         default_db: Optional[str] = None,
         default_schema: Optional[str] = None,
+        override_dialect: Optional[str] = None,
     ) -> None:
         """Add lineage by parsing a SQL query."""
         from datahub.sql_parsing.sqlglot_lineage import (
@@ -493,6 +494,7 @@ class LineageClient:
             platform_instance=platform_instance,
             env=env,
             graph=self._client._graph,
+            override_dialect=override_dialect,
         )
 
         if parsed_result.debug_info.table_error:

datahub/sql_parsing/sql_parsing_aggregator.py

@@ -1494,9 +1494,9 @@ class SqlParsingAggregator(Closeable):
             return
 
         # If a query doesn't involve any allowed tables, skip it.
-        if downstream_urn is None and not any(
-            self.is_allowed_table(urn) for urn in query.upstreams
-        ):
+        if (
+            downstream_urn is None or not self.is_allowed_table(downstream_urn)
+        ) and not any(self.is_allowed_table(urn) for urn in query.upstreams):
             self.report.num_queries_skipped_due_to_filters += 1
             return
 
@@ -1577,27 +1577,33 @@ class SqlParsingAggregator(Closeable):
 
         @dataclasses.dataclass
         class QueryLineageInfo:
-            upstreams: List[UrnStr]  # this is direct upstreams, with *no temp tables*
-            column_lineage: List[ColumnLineageInfo]
+            upstreams: OrderedSet[
+                UrnStr
+            ]  # this is direct upstreams, with *no temp tables*
+            column_lineage: OrderedSet[ColumnLineageInfo]
             confidence_score: float
 
             def _merge_lineage_from(self, other_query: "QueryLineageInfo") -> None:
-                self.upstreams += other_query.upstreams
-                self.column_lineage += other_query.column_lineage
+                self.upstreams.update(other_query.upstreams)
+                self.column_lineage.update(other_query.column_lineage)
                 self.confidence_score = min(
                     self.confidence_score, other_query.confidence_score
                 )
 
+        cache: Dict[str, QueryLineageInfo] = {}
+
         def _recurse_into_query(
             query: QueryMetadata, recursion_path: List[QueryId]
         ) -> QueryLineageInfo:
             if query.query_id in recursion_path:
                 # This is a cycle, so we just return the query as-is.
                 return QueryLineageInfo(
-                    upstreams=query.upstreams,
-                    column_lineage=query.column_lineage,
+                    upstreams=OrderedSet(query.upstreams),
+                    column_lineage=OrderedSet(query.column_lineage),
                     confidence_score=query.confidence_score,
                 )
+            if query.query_id in cache:
+                return cache[query.query_id]
             recursion_path = [*recursion_path, query.query_id]
             composed_of_queries.add(query.query_id)
 
@@ -1612,7 +1618,7 @@ class SqlParsingAggregator(Closeable):
                         upstream_query = self._query_map.get(upstream_query_id)
                         if (
                             upstream_query
-                            and upstream_query.query_id not in composed_of_queries
+                            and upstream_query.query_id not in recursion_path
                         ):
                             temp_query_lineage_info = _recurse_into_query(
                                 upstream_query, recursion_path
@@ -1672,11 +1678,14 @@ class SqlParsingAggregator(Closeable):
                 ]
             )
 
-            return QueryLineageInfo(
-                upstreams=list(new_upstreams),
-                column_lineage=new_cll,
+            ret = QueryLineageInfo(
+                upstreams=new_upstreams,
+                column_lineage=OrderedSet(new_cll),
                 confidence_score=new_confidence_score,
             )
+            cache[query.query_id] = ret
+
+            return ret
 
         resolved_lineage_info = _recurse_into_query(base_query, [])
 
@@ -1716,8 +1725,8 @@ class SqlParsingAggregator(Closeable):
             base_query,
             query_id=composite_query_id,
             formatted_query_string=merged_query_text,
-            upstreams=resolved_lineage_info.upstreams,
-            column_lineage=resolved_lineage_info.column_lineage,
+            upstreams=list(resolved_lineage_info.upstreams),
+            column_lineage=list(resolved_lineage_info.column_lineage),
             confidence_score=resolved_lineage_info.confidence_score,
         )
 

datahub/sql_parsing/sqlglot_lineage.py

@@ -56,6 +56,7 @@ from datahub.sql_parsing.sql_parsing_common import (
     QueryTypeProps,
 )
 from datahub.sql_parsing.sqlglot_utils import (
+    DialectOrStr,
     get_dialect,
     get_query_fingerprint_debug,
     is_dialect_instance,
@@ -124,6 +125,17 @@ class _DownstreamColumnRef(_ParserBaseModel):
 
 
 class DownstreamColumnRef(_ParserBaseModel):
+    """
+    TODO: Instead of implementing custom __hash__ function this class should simply inherit from _FrozenModel.
+          What stops us is that `column_type` field of type `SchemaFieldDataTypeClass` is not hashable - it's an
+          auto-generated class from .pdl model files. We need generic solution allowing us to either:
+          1. Implement hashing for .pdl model objects
+          2. Reliably provide pydantic (both v1 and v2) with information to skip particular fields from default
+             hash function - with a twist here that _FrozenModel implements its own `__lt__` function - it needs
+             to understand that instruction as well.
+          Instances of this class needs to be hashable as we store them in a set when processing lineage from queries.
+    """
+
     table: Optional[Urn] = None
     column: str
     column_type: Optional[SchemaFieldDataTypeClass] = None
@@ -139,8 +151,11 @@ class DownstreamColumnRef(_ParserBaseModel):
             return v
         return SchemaFieldDataTypeClass.from_obj(v)
 
+    def __hash__(self) -> int:
+        return hash((self.table, self.column, self.native_column_type))
 
-class ColumnTransformation(_ParserBaseModel):
+
+class ColumnTransformation(_FrozenModel):
     is_direct_copy: bool
     column_logic: str
 
@@ -153,11 +168,21 @@ class _ColumnLineageInfo(_ParserBaseModel):
 
 
 class ColumnLineageInfo(_ParserBaseModel):
+    """
+    TODO: Instead of implementing custom __hash__ function this class should simply inherit from _FrozenModel.
+          To achieve this, we need to change `upstreams` to `Tuple[ColumnRef, ...]` - along with many code lines
+          depending on it.
+          Instances of this class needs to be hashable as we store them in a set when processing lineage from queries.
+    """
+
     downstream: DownstreamColumnRef
     upstreams: List[ColumnRef]
 
     logic: Optional[ColumnTransformation] = pydantic.Field(default=None)
 
+    def __hash__(self) -> int:
+        return hash((self.downstream, tuple(self.upstreams), self.logic))
+
 
 class _JoinInfo(_ParserBaseModel):
     join_type: str
@@ -1231,12 +1256,12 @@ def _sqlglot_lineage_inner(
     schema_resolver: SchemaResolverInterface,
     default_db: Optional[str] = None,
     default_schema: Optional[str] = None,
-    default_dialect: Optional[str] = None,
+    override_dialect: Optional[DialectOrStr] = None,
 ) -> SqlParsingResult:
-    if not default_dialect:
-        dialect = get_dialect(schema_resolver.platform)
+    if override_dialect:
+        dialect = get_dialect(override_dialect)
     else:
-        dialect = get_dialect(default_dialect)
+        dialect = get_dialect(schema_resolver.platform)
 
     default_db = _normalize_db_or_schema(default_db, dialect)
     default_schema = _normalize_db_or_schema(default_schema, dialect)
@@ -1423,7 +1448,7 @@ def _sqlglot_lineage_nocache(
     schema_resolver: SchemaResolverInterface,
     default_db: Optional[str] = None,
     default_schema: Optional[str] = None,
-    default_dialect: Optional[str] = None,
+    override_dialect: Optional[DialectOrStr] = None,
 ) -> SqlParsingResult:
     """Parse a SQL statement and generate lineage information.
 
@@ -1441,8 +1466,8 @@ def _sqlglot_lineage_nocache(
     can be brittle with respect to missing schema information and complex
     SQL logic like UNNESTs.
 
-    The SQL dialect can be given as an argument called default_dialect or it can
-    be inferred from the schema_resolver's platform.
+    The SQL dialect will be inferred from the schema_resolver's platform.
+    That inference can be overridden by passing an override_dialect argument.
     The set of supported dialects is the same as sqlglot's. See their
     `documentation <https://sqlglot.com/sqlglot/dialects/dialect.html#Dialects>`_
     for the full list.
@@ -1457,7 +1482,7 @@ def _sqlglot_lineage_nocache(
         schema_resolver: The schema resolver to use for resolving table schemas.
         default_db: The default database to use for unqualified table names.
         default_schema: The default schema to use for unqualified table names.
-        default_dialect: A default dialect to override the dialect provided by 'schema_resolver'.
+        override_dialect: Override the dialect provided by 'schema_resolver'.
 
     Returns:
         A SqlParsingResult object containing the parsed lineage information.
@@ -1482,7 +1507,7 @@ def _sqlglot_lineage_nocache(
             schema_resolver=schema_resolver,
             default_db=default_db,
             default_schema=default_schema,
-            default_dialect=default_dialect,
+            override_dialect=override_dialect,
         )
     except Exception as e:
         return SqlParsingResult.make_from_error(e)
@@ -1520,15 +1545,15 @@ def sqlglot_lineage(
     schema_resolver: SchemaResolverInterface,
     default_db: Optional[str] = None,
     default_schema: Optional[str] = None,
-    default_dialect: Optional[str] = None,
+    override_dialect: Optional[DialectOrStr] = None,
 ) -> SqlParsingResult:
     if schema_resolver.includes_temp_tables():
         return _sqlglot_lineage_nocache(
-            sql, schema_resolver, default_db, default_schema, default_dialect
+            sql, schema_resolver, default_db, default_schema, override_dialect
         )
     else:
         return _sqlglot_lineage_cached(
-            sql, schema_resolver, default_db, default_schema, default_dialect
+            sql, schema_resolver, default_db, default_schema, override_dialect
         )
 
 
@@ -1580,6 +1605,7 @@ def create_lineage_sql_parsed_result(
     default_schema: Optional[str] = None,
     graph: Optional[DataHubGraph] = None,
     schema_aware: bool = True,
+    override_dialect: Optional[DialectOrStr] = None,
 ) -> SqlParsingResult:
     schema_resolver = create_schema_resolver(
         platform=platform,
@@ -1599,6 +1625,7 @@ def create_lineage_sql_parsed_result(
             schema_resolver=schema_resolver,
             default_db=default_db,
             default_schema=default_schema,
+            override_dialect=override_dialect,
         )
     except Exception as e:
         return SqlParsingResult.make_from_error(e)