acryl-datahub 1.0.0rc2__py3-none-any.whl → 1.0.0rc3__py3-none-any.whl

datahub/ingestion/source/tableau/tableau.py

@@ -2190,6 +2190,10 @@ class TableauSiteSource:
                 dataset_snapshot.aspects.append(browse_paths)
             else:
                 logger.debug(f"Browse path not set for Custom SQL table {csql_id}")
+                logger.warning(
+                    f"Skipping Custom SQL table {csql_id} due to filtered downstream"
+                )
+                continue
 
             dataset_properties = DatasetPropertiesClass(
                 name=csql.get(c.NAME),
@@ -2628,6 +2632,15 @@ class TableauSiteSource:
             datasource_info = datasource
 
         browse_path = self._get_project_browse_path_name(datasource)
+        if (
+            not is_embedded_ds
+            and self._get_published_datasource_project_luid(datasource) is None
+        ):
+            logger.warning(
+                f"Skip ingesting published datasource {datasource.get(c.NAME)} because of filtered project"
+            )
+            return
+
         logger.debug(f"datasource {datasource.get(c.NAME)} browse-path {browse_path}")
         datasource_id = datasource[c.ID]
         datasource_urn = builder.make_dataset_urn_with_platform_instance(
@@ -2851,6 +2864,11 @@ class TableauSiteSource:
             query_filter=tables_filter,
             page_size=self.config.effective_database_table_page_size,
         ):
+            if tableau_database_table_id_to_urn_map.get(tableau_table[c.ID]) is None:
+                logger.warning(
+                    f"Skipping table {tableau_table[c.ID]} due to filtered out published datasource"
+                )
+                continue
             database_table = self.database_tables[
                 tableau_database_table_id_to_urn_map[tableau_table[c.ID]]
             ]
@@ -2905,6 +2923,7 @@ class TableauSiteSource:
             dataset_snapshot.aspects.append(browse_paths)
         else:
             logger.debug(f"Browse path not set for table {database_table.urn}")
+            return
 
         schema_metadata = self.get_schema_metadata_for_table(
             tableau_columns, database_table.parsed_columns

datahub/metadata/schema.avsc

@@ -3504,7 +3504,9 @@
                   "fieldName": "glossaryTerms",
                   "fieldType": "URN",
                   "filterNameOverride": "Glossary Term",
-                  "hasValuesFieldName": "hasGlossaryTerms"
+                  "hasValuesFieldName": "hasGlossaryTerms",
+                  "includeSystemModifiedAt": true,
+                  "systemModifiedAtFieldName": "termsModifiedAt"
                 },
                 "java": {
                   "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"
@@ -10339,7 +10341,9 @@
           "/terms/*/urn": {
             "boostScore": 0.5,
             "fieldName": "editedFieldGlossaryTerms",
-            "fieldType": "URN"
+            "fieldType": "URN",
+            "includeSystemModifiedAt": true,
+            "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
           }
         },
         "type": [
@@ -10477,6 +10481,12 @@
             "namespace": "com.linkedin.pegasus2avro.businessattribute",
             "fields": [
               {
+                "Searchable": {
+                  "fieldName": "schemaFieldBusinessAttribute",
+                  "includeSystemModifiedAt": true,
+                  "queryByDefault": false,
+                  "systemModifiedAtFieldName": "schemaFieldBusinessAttributeModifiedAt"
+                },
                 "java": {
                   "class": "com.linkedin.pegasus2avro.common.urn.BusinessAttributeUrn"
                 },
@@ -15077,6 +15087,7 @@
       },
       {
         "Searchable": {
+          "boostScore": 10.0,
           "enableAutocomplete": true,
           "fieldNameAliases": [
             "_entityName"
@@ -16546,7 +16557,9 @@
                                     "/terms/*/urn": {
                                       "boostScore": 0.5,
                                       "fieldName": "editedFieldGlossaryTerms",
-                                      "fieldType": "URN"
+                                      "fieldType": "URN",
+                                      "includeSystemModifiedAt": true,
+                                      "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
                                     }
                                   },
                                   "type": [
@@ -19423,6 +19436,13 @@
         "doc": "The display name of the property. This is the name that will be shown in the UI and can be used to look up the property id."
       },
       {
+        "UrnValidation": {
+          "entityTypes": [
+            "dataType"
+          ],
+          "exist": true,
+          "strict": true
+        },
         "java": {
           "class": "com.linkedin.pegasus2avro.common.urn.Urn"
         },
@@ -19511,6 +19531,13 @@
             "fieldName": "entityTypes"
           }
         },
+        "UrnValidation": {
+          "entityTypes": [
+            "entityType"
+          ],
+          "exist": true,
+          "strict": true
+        },
         "Urn": "Urn",
         "urn_is_array": true,
         "type": {

datahub/metadata/schemas/AssertionInfo.avsc

@@ -2010,7 +2010,9 @@
                                               "fieldName": "glossaryTerms",
                                               "fieldType": "URN",
                                               "filterNameOverride": "Glossary Term",
-                                              "hasValuesFieldName": "hasGlossaryTerms"
+                                              "hasValuesFieldName": "hasGlossaryTerms",
+                                              "includeSystemModifiedAt": true,
+                                              "systemModifiedAtFieldName": "termsModifiedAt"
                                             },
                                             "java": {
                                               "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/BusinessAttributeInfo.avsc

@@ -221,7 +221,9 @@
         "/terms/*/urn": {
           "boostScore": 0.5,
           "fieldName": "editedFieldGlossaryTerms",
-          "fieldType": "URN"
+          "fieldType": "URN",
+          "includeSystemModifiedAt": true,
+          "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
         }
       },
       "type": [
@@ -254,7 +256,9 @@
                         "fieldName": "glossaryTerms",
                         "fieldType": "URN",
                         "filterNameOverride": "Glossary Term",
-                        "hasValuesFieldName": "hasGlossaryTerms"
+                        "hasValuesFieldName": "hasGlossaryTerms",
+                        "includeSystemModifiedAt": true,
+                        "systemModifiedAtFieldName": "termsModifiedAt"
                       },
                       "java": {
                         "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/BusinessAttributes.avsc

@@ -31,6 +31,12 @@
           "namespace": "com.linkedin.pegasus2avro.businessattribute",
           "fields": [
             {
+              "Searchable": {
+                "fieldName": "schemaFieldBusinessAttribute",
+                "includeSystemModifiedAt": true,
+                "queryByDefault": false,
+                "systemModifiedAtFieldName": "schemaFieldBusinessAttributeModifiedAt"
+              },
               "java": {
                 "class": "com.linkedin.pegasus2avro.common.urn.BusinessAttributeUrn"
               },

datahub/metadata/schemas/ChartInfo.avsc

@@ -39,6 +39,7 @@
     },
     {
       "Searchable": {
+        "boostScore": 10.0,
         "enableAutocomplete": true,
         "fieldNameAliases": [
           "_entityName"

datahub/metadata/schemas/EditableSchemaMetadata.avsc

@@ -303,7 +303,9 @@
                 "/terms/*/urn": {
                   "boostScore": 0.5,
                   "fieldName": "editedFieldGlossaryTerms",
-                  "fieldType": "URN"
+                  "fieldType": "URN",
+                  "includeSystemModifiedAt": true,
+                  "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
                 }
               },
               "type": [
@@ -336,7 +338,9 @@
                                 "fieldName": "glossaryTerms",
                                 "fieldType": "URN",
                                 "filterNameOverride": "Glossary Term",
-                                "hasValuesFieldName": "hasGlossaryTerms"
+                                "hasValuesFieldName": "hasGlossaryTerms",
+                                "includeSystemModifiedAt": true,
+                                "systemModifiedAtFieldName": "termsModifiedAt"
                               },
                               "java": {
                                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/GlossaryTerms.avsc

@@ -26,7 +26,9 @@
                 "fieldName": "glossaryTerms",
                 "fieldType": "URN",
                 "filterNameOverride": "Glossary Term",
-                "hasValuesFieldName": "hasGlossaryTerms"
+                "hasValuesFieldName": "hasGlossaryTerms",
+                "includeSystemModifiedAt": true,
+                "systemModifiedAtFieldName": "termsModifiedAt"
               },
               "java": {
                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/InputFields.avsc

@@ -553,7 +553,9 @@
                                         "fieldName": "glossaryTerms",
                                         "fieldType": "URN",
                                         "filterNameOverride": "Glossary Term",
-                                        "hasValuesFieldName": "hasGlossaryTerms"
+                                        "hasValuesFieldName": "hasGlossaryTerms",
+                                        "includeSystemModifiedAt": true,
+                                        "systemModifiedAtFieldName": "termsModifiedAt"
                                       },
                                       "java": {
                                         "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/MetadataChangeEvent.avsc

@@ -183,6 +183,7 @@
                       },
                       {
                         "Searchable": {
+                          "boostScore": 10.0,
                           "enableAutocomplete": true,
                           "fieldNameAliases": [
                             "_entityName"
@@ -994,7 +995,9 @@
                                   "fieldName": "glossaryTerms",
                                   "fieldType": "URN",
                                   "filterNameOverride": "Glossary Term",
-                                  "hasValuesFieldName": "hasGlossaryTerms"
+                                  "hasValuesFieldName": "hasGlossaryTerms",
+                                  "includeSystemModifiedAt": true,
+                                  "systemModifiedAtFieldName": "termsModifiedAt"
                                 },
                                 "java": {
                                   "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"
@@ -4644,7 +4647,9 @@
                                   "/terms/*/urn": {
                                     "boostScore": 0.5,
                                     "fieldName": "editedFieldGlossaryTerms",
-                                    "fieldType": "URN"
+                                    "fieldType": "URN",
+                                    "includeSystemModifiedAt": true,
+                                    "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
                                   }
                                 },
                                 "type": [

datahub/metadata/schemas/SchemaMetadata.avsc

@@ -777,7 +777,9 @@
                                 "fieldName": "glossaryTerms",
                                 "fieldType": "URN",
                                 "filterNameOverride": "Glossary Term",
-                                "hasValuesFieldName": "hasGlossaryTerms"
+                                "hasValuesFieldName": "hasGlossaryTerms",
+                                "includeSystemModifiedAt": true,
+                                "systemModifiedAtFieldName": "termsModifiedAt"
                               },
                               "java": {
                                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/StructuredPropertyDefinition.avsc

@@ -23,6 +23,13 @@
       "doc": "The display name of the property. This is the name that will be shown in the UI and can be used to look up the property id."
     },
     {
+      "UrnValidation": {
+        "entityTypes": [
+          "dataType"
+        ],
+        "exist": true,
+        "strict": true
+      },
       "java": {
         "class": "com.linkedin.pegasus2avro.common.urn.Urn"
       },
@@ -111,6 +118,13 @@
           "fieldName": "entityTypes"
         }
       },
+      "UrnValidation": {
+        "entityTypes": [
+          "entityType"
+        ],
+        "exist": true,
+        "strict": true
+      },
       "type": {
         "type": "array",
         "items": "string"

datahub/sql_parsing/schema_resolver.py

@@ -13,7 +13,7 @@ from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier
 from datahub.metadata.schema_classes import SchemaFieldClass, SchemaMetadataClass
 from datahub.metadata.urns import DataPlatformUrn
-from datahub.sql_parsing._models import _TableName as _TableName  # noqa: I250
+from datahub.sql_parsing._models import _TableName as _TableName
 from datahub.sql_parsing.sql_parsing_common import PLATFORMS_WITH_CASE_SENSITIVE_TABLES
 from datahub.utilities.file_backed_collections import ConnectionWrapper, FileBackedDict
 from datahub.utilities.urns.field_paths import get_simple_field_path_from_v2_field_path

datahub/sql_parsing/sqlglot_lineage.py

@@ -473,7 +473,7 @@ def _create_table_ddl_cll(
     return column_lineage
 
 
-def _select_statement_cll(  # noqa: C901
+def _select_statement_cll(
     statement: _SupportedColumnLineageTypes,
     dialect: sqlglot.Dialect,
     root_scope: sqlglot.optimizer.Scope,

datahub/upgrade/upgrade.py

@@ -293,9 +293,9 @@ def is_client_server_compatible(client: VersionStats, server: VersionStats) -> i
         return server.version.micro - client.version.micro
 
 
-def _maybe_print_upgrade_message(  # noqa: C901
+def _maybe_print_upgrade_message(
     version_stats: Optional[DataHubVersionStats],
-) -> None:  # noqa: C901
+) -> None:
     days_before_cli_stale = 7
     days_before_quickstart_stale = 7
 

datahub/utilities/mapping.py

@@ -171,7 +171,7 @@ class OperationProcessor:
         self.owner_source_type = owner_source_type
         self.match_nested_props = match_nested_props
 
-    def process(self, raw_props: Mapping[str, Any]) -> Dict[str, Any]:  # noqa: C901
+    def process(self, raw_props: Mapping[str, Any]) -> Dict[str, Any]:
         # Defining the following local variables -
         # operations_map - the final resulting map when operations are processed.
         # Against each operation the values to be applied are stored.