acryl-datahub 0.15.0rc16__py3-none-any.whl → 0.15.0rc17__py3-none-any.whl

datahub/metadata/schema.avsc

@@ -2995,6 +2995,7 @@
       "entityCategory": "core",
       "entityAspects": [
         "propertyDefinition",
+        "structuredPropertySettings",
         "institutionalMemory",
         "status"
       ],
@@ -3328,10 +3329,81 @@
         ],
         "name": "lastModified",
         "default": null,
-        "doc": "Created Audit stamp"
+        "doc": "Last Modified Audit stamp"
       }
     ]
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "structuredPropertySettings"
+    },
+    "name": "StructuredPropertySettings",
+    "namespace": "com.linkedin.pegasus2avro.structured",
+    "fields": [
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "isHidden",
+        "default": false,
+        "doc": "Whether or not this asset should be hidden in the main application"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInSearchFilters",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as a search filter"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInAssetSummary",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed in the asset sidebar"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showAsAssetBadge",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as an asset badge on other\nasset's headers"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInColumnsTable",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as a column in the schema field table\nin a Dataset's \"Columns\" tab."
+      },
+      {
+        "Searchable": {
+          "/time": {
+            "fieldName": "lastModifiedSettings",
+            "fieldType": "DATETIME"
+          }
+        },
+        "type": [
+          "null",
+          "com.linkedin.pegasus2avro.common.AuditStamp"
+        ],
+        "name": "lastModified",
+        "default": null,
+        "doc": "Last Modified Audit stamp"
+      }
+    ],
+    "doc": "Settings specific to a structured property entity"
+  },
   {
     "type": "record",
     "Aspect": {

datahub/metadata/schemas/StructuredPropertyDefinition.avsc

@@ -359,7 +359,7 @@
       ],
       "name": "lastModified",
       "default": null,
-      "doc": "Created Audit stamp"
+      "doc": "Last Modified Audit stamp"
     }
   ]
 }

datahub/metadata/schemas/StructuredPropertyKey.avsc

@@ -6,6 +6,7 @@
     "entityCategory": "core",
     "entityAspects": [
       "propertyDefinition",
+      "structuredPropertySettings",
       "institutionalMemory",
       "status"
     ],

datahub/metadata/schemas/StructuredPropertySettings.avsc

@@ -0,0 +1,114 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "structuredPropertySettings"
+  },
+  "name": "StructuredPropertySettings",
+  "namespace": "com.linkedin.pegasus2avro.structured",
+  "fields": [
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "isHidden",
+      "default": false,
+      "doc": "Whether or not this asset should be hidden in the main application"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "showInSearchFilters",
+      "default": false,
+      "doc": "Whether or not this asset should be displayed as a search filter"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "showInAssetSummary",
+      "default": false,
+      "doc": "Whether or not this asset should be displayed in the asset sidebar"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "showAsAssetBadge",
+      "default": false,
+      "doc": "Whether or not this asset should be displayed as an asset badge on other\nasset's headers"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "showInColumnsTable",
+      "default": false,
+      "doc": "Whether or not this asset should be displayed as a column in the schema field table\nin a Dataset's \"Columns\" tab."
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "lastModifiedSettings",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "AuditStamp",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": "long",
+              "name": "time",
+              "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": "string",
+              "name": "actor",
+              "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+              "Urn": "Urn"
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "impersonator",
+              "default": null,
+              "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+              "Urn": "Urn"
+            },
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "message",
+              "default": null,
+              "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+            }
+          ],
+          "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+        }
+      ],
+      "name": "lastModified",
+      "default": null,
+      "doc": "Last Modified Audit stamp"
+    }
+  ],
+  "doc": "Settings specific to a structured property entity"
+}

datahub/sql_parsing/schema_resolver.py

@@ -123,6 +123,13 @@ class SchemaResolver(Closeable, SchemaResolverInterface):
         )
         return urn
 
+    def resolve_urn(self, urn: str) -> Tuple[str, Optional[SchemaInfo]]:
+        schema_info = self._resolve_schema_info(urn)
+        if schema_info:
+            return urn, schema_info
+
+        return urn, None
+
     def resolve_table(self, table: _TableName) -> Tuple[str, Optional[SchemaInfo]]:
         urn = self.get_urn_for_table(table)
 
@@ -293,3 +300,19 @@ def _convert_schema_field_list_to_info(
 
 def _convert_schema_aspect_to_info(schema_metadata: SchemaMetadataClass) -> SchemaInfo:
     return _convert_schema_field_list_to_info(schema_metadata.fields)
+
+
+def match_columns_to_schema(
+    schema_info: SchemaInfo, input_columns: List[str]
+) -> List[str]:
+    column_from_gms: List[str] = list(schema_info.keys())  # list() to silent lint
+
+    gms_column_map: Dict[str, str] = {
+        column.lower(): column for column in column_from_gms
+    }
+
+    output_columns: List[str] = [
+        gms_column_map.get(column.lower(), column) for column in input_columns
+    ]
+
+    return output_columns

datahub/sql_parsing/sqlglot_lineage.py

@@ -1181,6 +1181,45 @@ def sqlglot_lineage(
         )
 
 
+@functools.lru_cache(maxsize=128)
+def create_and_cache_schema_resolver(
+    platform: str,
+    env: str,
+    graph: Optional[DataHubGraph] = None,
+    platform_instance: Optional[str] = None,
+    schema_aware: bool = True,
+) -> SchemaResolver:
+    return create_schema_resolver(
+        platform=platform,
+        env=env,
+        graph=graph,
+        platform_instance=platform_instance,
+        schema_aware=schema_aware,
+    )
+
+
+def create_schema_resolver(
+    platform: str,
+    env: str,
+    graph: Optional[DataHubGraph] = None,
+    platform_instance: Optional[str] = None,
+    schema_aware: bool = True,
+) -> SchemaResolver:
+    if graph and schema_aware:
+        return graph._make_schema_resolver(
+            platform=platform,
+            platform_instance=platform_instance,
+            env=env,
+        )
+
+    return SchemaResolver(
+        platform=platform,
+        platform_instance=platform_instance,
+        env=env,
+        graph=None,
+    )
+
+
 def create_lineage_sql_parsed_result(
     query: str,
     default_db: Optional[str],
@@ -1191,21 +1230,17 @@ def create_lineage_sql_parsed_result(
     graph: Optional[DataHubGraph] = None,
     schema_aware: bool = True,
 ) -> SqlParsingResult:
+    schema_resolver = create_schema_resolver(
+        platform=platform,
+        platform_instance=platform_instance,
+        env=env,
+        schema_aware=schema_aware,
+        graph=graph,
+    )
+
+    needs_close: bool = True
     if graph and schema_aware:
         needs_close = False
-        schema_resolver = graph._make_schema_resolver(
-            platform=platform,
-            platform_instance=platform_instance,
-            env=env,
-        )
-    else:
-        needs_close = True
-        schema_resolver = SchemaResolver(
-            platform=platform,
-            platform_instance=platform_instance,
-            env=env,
-            graph=None,
-        )
 
     try:
         return sqlglot_lineage(

datahub/testing/doctest.py

@@ -0,0 +1,12 @@
+import doctest
+from types import ModuleType
+
+
+def assert_doctest(module: ModuleType) -> None:
+    result = doctest.testmod(
+        module,
+        raise_on_error=True,
+        verbose=True,
+    )
+    if result.attempted == 0:
+        raise ValueError(f"No doctests found in {module.__name__}")