acryl-datahub 1.1.0.5rc6__py3-none-any.whl → 1.1.0.5rc7__py3-none-any.whl

datahub/ingestion/source/unity/source.py

@@ -768,10 +768,11 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
 
     def gen_schema_containers(self, schema: Schema) -> Iterable[MetadataWorkUnit]:
         domain_urn = self._gen_domain_urn(f"{schema.catalog.name}.{schema.name}")
-        schema_tags = self.unity_catalog_api_proxy.get_schema_tags(
-            schema.catalog.name
-        ).get(f"{schema.catalog.name}.{schema.name}", [])
-        if schema_tags:
+        schema_tags = []
+        if self.config.include_tags:
+            schema_tags = self.unity_catalog_api_proxy.get_schema_tags(
+                schema.catalog.name
+            ).get(f"{schema.catalog.name}.{schema.name}", [])
             logger.debug(f"Schema tags for {schema.name}: {schema_tags}")
             # Generate platform resources for schema tags
             yield from self.gen_platform_resources(schema_tags)
@@ -809,10 +810,11 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
 
     def gen_catalog_containers(self, catalog: Catalog) -> Iterable[MetadataWorkUnit]:
         domain_urn = self._gen_domain_urn(catalog.name)
-        catalog_tags = self.unity_catalog_api_proxy.get_catalog_tags(catalog.name).get(
-            catalog.name, []
-        )
-        if catalog_tags:
+        catalog_tags = []
+        if self.config.include_tags:
+            catalog_tags = self.unity_catalog_api_proxy.get_catalog_tags(
+                catalog.name
+            ).get(catalog.name, [])
             logger.debug(f"Schema tags for {catalog.name}: {catalog_tags}")
             # Generate platform resources for schema tags
             yield from self.gen_platform_resources(catalog_tags)

datahub/integrations/assertion/snowflake/compiler.py

@@ -84,9 +84,10 @@ class SnowflakeAssertionCompiler(AssertionCompiler):
 
         dmf_definitions_path = self.output_dir / DMF_DEFINITIONS_FILE_NAME
         dmf_associations_path = self.output_dir / DMF_ASSOCIATIONS_FILE_NAME
-        with (dmf_definitions_path).open("w") as definitions, (
-            dmf_associations_path
-        ).open("w") as associations:
+        with (
+            (dmf_definitions_path).open("w") as definitions,
+            (dmf_associations_path).open("w") as associations,
+        ):
             for assertion_spec in assertion_config_spec.assertions:
                 result.report.num_processed += 1
                 try:

datahub/metadata/_internal_schema_classes.py

@@ -14127,26 +14127,39 @@ class CorpUserHomePageSettingsClass(DictWrapper):
     
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.identity.CorpUserHomePageSettings")
     def __init__(self,
-        pageTemplate: str,
+        pageTemplate: Union[None, str]=None,
+        dismissedAnnouncements: Union[None, List[str]]=None,
     ):
         super().__init__()
         
         self.pageTemplate = pageTemplate
+        self.dismissedAnnouncements = dismissedAnnouncements
     
     def _restore_defaults(self) -> None:
-        self.pageTemplate = str()
+        self.pageTemplate = self.RECORD_SCHEMA.fields_dict["pageTemplate"].default
+        self.dismissedAnnouncements = self.RECORD_SCHEMA.fields_dict["dismissedAnnouncements"].default
     
     
     @property
-    def pageTemplate(self) -> str:
+    def pageTemplate(self) -> Union[None, str]:
         """The page template that will be rendered in the UI by default for this user"""
         return self._inner_dict.get('pageTemplate')  # type: ignore
     
     @pageTemplate.setter
-    def pageTemplate(self, value: str) -> None:
+    def pageTemplate(self, value: Union[None, str]) -> None:
         self._inner_dict['pageTemplate'] = value
     
     
+    @property
+    def dismissedAnnouncements(self) -> Union[None, List[str]]:
+        """The list of announcement urns that have been dismissed by the user"""
+        return self._inner_dict.get('dismissedAnnouncements')  # type: ignore
+    
+    @dismissedAnnouncements.setter
+    def dismissedAnnouncements(self, value: Union[None, List[str]]) -> None:
+        self._inner_dict['dismissedAnnouncements'] = value
+    
+    
 class CorpUserInfoClass(_Aspect):
     """Linkedin corp user information"""
 
@@ -24394,6 +24407,59 @@ class DataHubSecretValueClass(_Aspect):
         self._inner_dict['created'] = value
     
     
+class ApplicationsSettingsClass(DictWrapper):
+    # No docs available.
+    
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.settings.global.ApplicationsSettings")
+    def __init__(self,
+        enabled: bool,
+        config: Union[None, str]=None,
+        configVersion: Union[None, str]=None,
+    ):
+        super().__init__()
+        
+        self.enabled = enabled
+        self.config = config
+        self.configVersion = configVersion
+    
+    def _restore_defaults(self) -> None:
+        self.enabled = bool()
+        self.config = self.RECORD_SCHEMA.fields_dict["config"].default
+        self.configVersion = self.RECORD_SCHEMA.fields_dict["configVersion"].default
+    
+    
+    @property
+    def enabled(self) -> bool:
+        # No docs available.
+        return self._inner_dict.get('enabled')  # type: ignore
+    
+    @enabled.setter
+    def enabled(self, value: bool) -> None:
+        self._inner_dict['enabled'] = value
+    
+    
+    @property
+    def config(self) -> Union[None, str]:
+        """The configuration for the feature, in JSON format."""
+        return self._inner_dict.get('config')  # type: ignore
+    
+    @config.setter
+    def config(self, value: Union[None, str]) -> None:
+        self._inner_dict['config'] = value
+    
+    
+    @property
+    def configVersion(self) -> Union[None, str]:
+        """The version of the configuration schema that has been used to serialize
+           the config.
+    If not provided, the version is assumed to be the latest version."""
+        return self._inner_dict.get('configVersion')  # type: ignore
+    
+    @configVersion.setter
+    def configVersion(self, value: Union[None, str]) -> None:
+        self._inner_dict['configVersion'] = value
+    
+    
 class DocPropagationFeatureSettingsClass(DictWrapper):
     # No docs available.
     
@@ -24502,6 +24568,7 @@ class GlobalSettingsInfoClass(_Aspect):
         views: Union[None, "GlobalViewsSettingsClass"]=None,
         docPropagation: Optional[Union["DocPropagationFeatureSettingsClass", None]]=None,
         homePage: Union[None, "GlobalHomePageSettingsClass"]=None,
+        applications: Union[None, "ApplicationsSettingsClass"]=None,
     ):
         super().__init__()
         
@@ -24513,12 +24580,14 @@ class GlobalSettingsInfoClass(_Aspect):
         else:
             self.docPropagation = docPropagation
         self.homePage = homePage
+        self.applications = applications
     
     def _restore_defaults(self) -> None:
         self.sso = self.RECORD_SCHEMA.fields_dict["sso"].default
         self.views = self.RECORD_SCHEMA.fields_dict["views"].default
         self.docPropagation = _json_converter.from_json_object(self.RECORD_SCHEMA.fields_dict["docPropagation"].default, writers_schema=self.RECORD_SCHEMA.fields_dict["docPropagation"].type)
         self.homePage = self.RECORD_SCHEMA.fields_dict["homePage"].default
+        self.applications = self.RECORD_SCHEMA.fields_dict["applications"].default
     
     
     @property
@@ -24561,6 +24630,16 @@ class GlobalSettingsInfoClass(_Aspect):
         self._inner_dict['homePage'] = value
     
     
+    @property
+    def applications(self) -> Union[None, "ApplicationsSettingsClass"]:
+        """Settings related to applications. If not enabled, applications won't show up in navigation"""
+        return self._inner_dict.get('applications')  # type: ignore
+    
+    @applications.setter
+    def applications(self, value: Union[None, "ApplicationsSettingsClass"]) -> None:
+        self._inner_dict['applications'] = value
+    
+    
 class GlobalViewsSettingsClass(DictWrapper):
     """Settings for DataHub Views feature."""
     
@@ -27086,6 +27165,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.schemafield.SchemaFieldAliases': SchemaFieldAliasesClass,
     'com.linkedin.pegasus2avro.schemafield.SchemaFieldInfo': SchemaFieldInfoClass,
     'com.linkedin.pegasus2avro.secret.DataHubSecretValue': DataHubSecretValueClass,
+    'com.linkedin.pegasus2avro.settings.global.ApplicationsSettings': ApplicationsSettingsClass,
     'com.linkedin.pegasus2avro.settings.global.DocPropagationFeatureSettings': DocPropagationFeatureSettingsClass,
     'com.linkedin.pegasus2avro.settings.global.GlobalHomePageSettings': GlobalHomePageSettingsClass,
     'com.linkedin.pegasus2avro.settings.global.GlobalSettingsInfo': GlobalSettingsInfoClass,
@@ -27592,6 +27672,7 @@ __SCHEMA_TYPES = {
     'SchemaFieldAliases': SchemaFieldAliasesClass,
     'SchemaFieldInfo': SchemaFieldInfoClass,
     'DataHubSecretValue': DataHubSecretValueClass,
+    'ApplicationsSettings': ApplicationsSettingsClass,
     'DocPropagationFeatureSettings': DocPropagationFeatureSettingsClass,
     'GlobalHomePageSettings': GlobalHomePageSettingsClass,
     'GlobalSettingsInfo': GlobalSettingsInfoClass,

datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py

@@ -7,6 +7,7 @@
 # pylint: skip-file
 # fmt: off
 # isort: skip_file
+from ......schema_classes import ApplicationsSettingsClass
 from ......schema_classes import DocPropagationFeatureSettingsClass
 from ......schema_classes import GlobalHomePageSettingsClass
 from ......schema_classes import GlobalSettingsInfoClass
@@ -15,6 +16,7 @@ from ......schema_classes import OidcSettingsClass
 from ......schema_classes import SsoSettingsClass
 
 
+ApplicationsSettings = ApplicationsSettingsClass
 DocPropagationFeatureSettings = DocPropagationFeatureSettingsClass
 GlobalHomePageSettings = GlobalHomePageSettingsClass
 GlobalSettingsInfo = GlobalSettingsInfoClass

datahub/metadata/schema.avsc

@@ -751,9 +751,25 @@
                   "class": "com.linkedin.pegasus2avro.common.urn.Urn"
                 },
                 "Urn": "Urn",
-                "type": "string",
+                "type": [
+                  "null",
+                  "string"
+                ],
                 "name": "pageTemplate",
+                "default": null,
                 "doc": "The page template that will be rendered in the UI by default for this user"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "array",
+                    "items": "string"
+                  }
+                ],
+                "name": "dismissedAnnouncements",
+                "default": null,
+                "doc": "The list of announcement urns that have been dismissed by the user"
               }
             ],
             "doc": "Settings related to the home page for a user"
@@ -18531,6 +18547,43 @@
         "name": "homePage",
         "default": null,
         "doc": "Global settings related to the home page for an instance"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "record",
+            "name": "ApplicationsSettings",
+            "namespace": "com.linkedin.pegasus2avro.settings.global",
+            "fields": [
+              {
+                "type": "boolean",
+                "name": "enabled"
+              },
+              {
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "config",
+                "default": null,
+                "doc": "The configuration for the feature, in JSON format."
+              },
+              {
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "configVersion",
+                "default": null,
+                "doc": "The version of the configuration schema that has been used to serialize\n       the config.\nIf not provided, the version is assumed to be the latest version."
+              }
+            ]
+          }
+        ],
+        "name": "applications",
+        "default": null,
+        "doc": "Settings related to applications. If not enabled, applications won't show up in navigation"
       }
     ],
     "doc": "DataHub Global platform settings. Careful - these should not be modified by the outside world!"

datahub/metadata/schemas/CorpUserSettings.avsc

@@ -172,10 +172,26 @@
               "java": {
                 "class": "com.linkedin.pegasus2avro.common.urn.Urn"
               },
-              "type": "string",
+              "type": [
+                "null",
+                "string"
+              ],
               "name": "pageTemplate",
+              "default": null,
               "doc": "The page template that will be rendered in the UI by default for this user",
               "Urn": "Urn"
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "array",
+                  "items": "string"
+                }
+              ],
+              "name": "dismissedAnnouncements",
+              "default": null,
+              "doc": "The list of announcement urns that have been dismissed by the user"
             }
           ],
           "doc": "Settings related to the home page for a user"

datahub/metadata/schemas/GlobalSettingsInfo.avsc

@@ -307,6 +307,43 @@
       "name": "homePage",
       "default": null,
       "doc": "Global settings related to the home page for an instance"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "ApplicationsSettings",
+          "namespace": "com.linkedin.pegasus2avro.settings.global",
+          "fields": [
+            {
+              "type": "boolean",
+              "name": "enabled"
+            },
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "config",
+              "default": null,
+              "doc": "The configuration for the feature, in JSON format."
+            },
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "configVersion",
+              "default": null,
+              "doc": "The version of the configuration schema that has been used to serialize\n       the config.\nIf not provided, the version is assumed to be the latest version."
+            }
+          ]
+        }
+      ],
+      "name": "applications",
+      "default": null,
+      "doc": "Settings related to applications. If not enabled, applications won't show up in navigation"
     }
   ],
   "doc": "DataHub Global platform settings. Careful - these should not be modified by the outside world!"

datahub/sdk/lineage_client.py

@@ -478,6 +478,7 @@ class LineageClient:
         env: str = "PROD",
         default_db: Optional[str] = None,
         default_schema: Optional[str] = None,
+        default_dialect: Optional[str] = None,
     ) -> None:
         """Add lineage by parsing a SQL query."""
         from datahub.sql_parsing.sqlglot_lineage import (
@@ -493,6 +494,7 @@ class LineageClient:
             platform_instance=platform_instance,
             env=env,
             graph=self._client._graph,
+            default_dialect=default_dialect,
         )
 
         if parsed_result.debug_info.table_error:

datahub/sql_parsing/sql_parsing_aggregator.py

@@ -1494,9 +1494,9 @@ class SqlParsingAggregator(Closeable):
             return
 
         # If a query doesn't involve any allowed tables, skip it.
-        if downstream_urn is None and not any(
-            self.is_allowed_table(urn) for urn in query.upstreams
-        ):
+        if (
+            downstream_urn is None or not self.is_allowed_table(downstream_urn)
+        ) and not any(self.is_allowed_table(urn) for urn in query.upstreams):
             self.report.num_queries_skipped_due_to_filters += 1
             return
 

datahub/sql_parsing/sqlglot_lineage.py

@@ -1580,6 +1580,7 @@ def create_lineage_sql_parsed_result(
     default_schema: Optional[str] = None,
     graph: Optional[DataHubGraph] = None,
     schema_aware: bool = True,
+    default_dialect: Optional[str] = None,
 ) -> SqlParsingResult:
     schema_resolver = create_schema_resolver(
         platform=platform,
@@ -1599,6 +1600,7 @@ def create_lineage_sql_parsed_result(
             schema_resolver=schema_resolver,
             default_db=default_db,
             default_schema=default_schema,
+            default_dialect=default_dialect,
         )
     except Exception as e:
         return SqlParsingResult.make_from_error(e)

datahub/utilities/sqlalchemy_query_combiner.py

@@ -272,8 +272,11 @@ class SQLAlchemyQueryCombiner:
                     self.report.uncombined_queries_issued += 1
                     return _sa_execute_underlying_method(conn, query, *args, **kwargs)
 
-        with _sa_execute_method_patching_lock, unittest.mock.patch(
-            "sqlalchemy.engine.Connection.execute", _sa_execute_fake
+        with (
+            _sa_execute_method_patching_lock,
+            unittest.mock.patch(
+                "sqlalchemy.engine.Connection.execute", _sa_execute_fake
+            ),
         ):
             yield self