acryl-datahub 0.15.0rc16__py3-none-any.whl → 0.15.0rc18__py3-none-any.whl

datahub/metadata/_schema_classes.py

@@ -23267,7 +23267,7 @@ class StructuredPropertyDefinitionClass(_Aspect):
     
     @property
     def lastModified(self) -> Union[None, "AuditStampClass"]:
-        """Created Audit stamp"""
+        """Last Modified Audit stamp"""
         return self._inner_dict.get('lastModified')  # type: ignore
     
     @lastModified.setter
@@ -23280,7 +23280,7 @@ class StructuredPropertyKeyClass(_Aspect):
 
 
     ASPECT_NAME = 'structuredPropertyKey'
-    ASPECT_INFO = {'keyForEntity': 'structuredProperty', 'entityCategory': 'core', 'entityAspects': ['propertyDefinition', 'institutionalMemory', 'status'], 'entityDoc': 'Structured Property represents a property meant for extending the core model of a logical entity'}
+    ASPECT_INFO = {'keyForEntity': 'structuredProperty', 'entityCategory': 'core', 'entityAspects': ['propertyDefinition', 'structuredPropertySettings', 'institutionalMemory', 'status'], 'entityDoc': 'Structured Property represents a property meant for extending the core model of a logical entity'}
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.structured.StructuredPropertyKey")
 
     def __init__(self,
@@ -23304,6 +23304,122 @@ class StructuredPropertyKeyClass(_Aspect):
         self._inner_dict['id'] = value
     
     
+class StructuredPropertySettingsClass(_Aspect):
+    """Settings specific to a structured property entity"""
+
+
+    ASPECT_NAME = 'structuredPropertySettings'
+    ASPECT_INFO = {}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.structured.StructuredPropertySettings")
+
+    def __init__(self,
+        isHidden: Optional[bool]=None,
+        showInSearchFilters: Optional[bool]=None,
+        showInAssetSummary: Optional[bool]=None,
+        showAsAssetBadge: Optional[bool]=None,
+        showInColumnsTable: Optional[bool]=None,
+        lastModified: Union[None, "AuditStampClass"]=None,
+    ):
+        super().__init__()
+        
+        if isHidden is None:
+            # default: False
+            self.isHidden = self.RECORD_SCHEMA.fields_dict["isHidden"].default
+        else:
+            self.isHidden = isHidden
+        if showInSearchFilters is None:
+            # default: False
+            self.showInSearchFilters = self.RECORD_SCHEMA.fields_dict["showInSearchFilters"].default
+        else:
+            self.showInSearchFilters = showInSearchFilters
+        if showInAssetSummary is None:
+            # default: False
+            self.showInAssetSummary = self.RECORD_SCHEMA.fields_dict["showInAssetSummary"].default
+        else:
+            self.showInAssetSummary = showInAssetSummary
+        if showAsAssetBadge is None:
+            # default: False
+            self.showAsAssetBadge = self.RECORD_SCHEMA.fields_dict["showAsAssetBadge"].default
+        else:
+            self.showAsAssetBadge = showAsAssetBadge
+        if showInColumnsTable is None:
+            # default: False
+            self.showInColumnsTable = self.RECORD_SCHEMA.fields_dict["showInColumnsTable"].default
+        else:
+            self.showInColumnsTable = showInColumnsTable
+        self.lastModified = lastModified
+    
+    def _restore_defaults(self) -> None:
+        self.isHidden = self.RECORD_SCHEMA.fields_dict["isHidden"].default
+        self.showInSearchFilters = self.RECORD_SCHEMA.fields_dict["showInSearchFilters"].default
+        self.showInAssetSummary = self.RECORD_SCHEMA.fields_dict["showInAssetSummary"].default
+        self.showAsAssetBadge = self.RECORD_SCHEMA.fields_dict["showAsAssetBadge"].default
+        self.showInColumnsTable = self.RECORD_SCHEMA.fields_dict["showInColumnsTable"].default
+        self.lastModified = self.RECORD_SCHEMA.fields_dict["lastModified"].default
+    
+    
+    @property
+    def isHidden(self) -> bool:
+        """Whether or not this asset should be hidden in the main application"""
+        return self._inner_dict.get('isHidden')  # type: ignore
+    
+    @isHidden.setter
+    def isHidden(self, value: bool) -> None:
+        self._inner_dict['isHidden'] = value
+    
+    
+    @property
+    def showInSearchFilters(self) -> bool:
+        """Whether or not this asset should be displayed as a search filter"""
+        return self._inner_dict.get('showInSearchFilters')  # type: ignore
+    
+    @showInSearchFilters.setter
+    def showInSearchFilters(self, value: bool) -> None:
+        self._inner_dict['showInSearchFilters'] = value
+    
+    
+    @property
+    def showInAssetSummary(self) -> bool:
+        """Whether or not this asset should be displayed in the asset sidebar"""
+        return self._inner_dict.get('showInAssetSummary')  # type: ignore
+    
+    @showInAssetSummary.setter
+    def showInAssetSummary(self, value: bool) -> None:
+        self._inner_dict['showInAssetSummary'] = value
+    
+    
+    @property
+    def showAsAssetBadge(self) -> bool:
+        """Whether or not this asset should be displayed as an asset badge on other
+    asset's headers"""
+        return self._inner_dict.get('showAsAssetBadge')  # type: ignore
+    
+    @showAsAssetBadge.setter
+    def showAsAssetBadge(self, value: bool) -> None:
+        self._inner_dict['showAsAssetBadge'] = value
+    
+    
+    @property
+    def showInColumnsTable(self) -> bool:
+        """Whether or not this asset should be displayed as a column in the schema field table
+    in a Dataset's "Columns" tab."""
+        return self._inner_dict.get('showInColumnsTable')  # type: ignore
+    
+    @showInColumnsTable.setter
+    def showInColumnsTable(self, value: bool) -> None:
+        self._inner_dict['showInColumnsTable'] = value
+    
+    
+    @property
+    def lastModified(self) -> Union[None, "AuditStampClass"]:
+        """Last Modified Audit stamp"""
+        return self._inner_dict.get('lastModified')  # type: ignore
+    
+    @lastModified.setter
+    def lastModified(self, value: Union[None, "AuditStampClass"]) -> None:
+        self._inner_dict['lastModified'] = value
+    
+    
 class StructuredPropertyValueAssignmentClass(DictWrapper):
     # No docs available.
     
@@ -24775,6 +24891,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.structured.StructuredProperties': StructuredPropertiesClass,
     'com.linkedin.pegasus2avro.structured.StructuredPropertyDefinition': StructuredPropertyDefinitionClass,
     'com.linkedin.pegasus2avro.structured.StructuredPropertyKey': StructuredPropertyKeyClass,
+    'com.linkedin.pegasus2avro.structured.StructuredPropertySettings': StructuredPropertySettingsClass,
     'com.linkedin.pegasus2avro.structured.StructuredPropertyValueAssignment': StructuredPropertyValueAssignmentClass,
     'com.linkedin.pegasus2avro.tag.TagProperties': TagPropertiesClass,
     'com.linkedin.pegasus2avro.telemetry.TelemetryClientId': TelemetryClientIdClass,
@@ -25240,6 +25357,7 @@ __SCHEMA_TYPES = {
     'StructuredProperties': StructuredPropertiesClass,
     'StructuredPropertyDefinition': StructuredPropertyDefinitionClass,
     'StructuredPropertyKey': StructuredPropertyKeyClass,
+    'StructuredPropertySettings': StructuredPropertySettingsClass,
     'StructuredPropertyValueAssignment': StructuredPropertyValueAssignmentClass,
     'TagProperties': TagPropertiesClass,
     'TelemetryClientId': TelemetryClientIdClass,
@@ -25336,6 +25454,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     QuerySubjectsClass,
     StructuredPropertyKeyClass,
     StructuredPropertyDefinitionClass,
+    StructuredPropertySettingsClass,
     StructuredPropertiesClass,
     GlobalSettingsInfoClass,
     DataHubRetentionConfigClass,
@@ -25548,6 +25667,7 @@ class AspectBag(TypedDict, total=False):
     querySubjects: QuerySubjectsClass
     structuredPropertyKey: StructuredPropertyKeyClass
     propertyDefinition: StructuredPropertyDefinitionClass
+    structuredPropertySettings: StructuredPropertySettingsClass
     structuredProperties: StructuredPropertiesClass
     globalSettingsInfo: GlobalSettingsInfoClass
     dataHubRetentionConfig: DataHubRetentionConfigClass

datahub/metadata/com/linkedin/pegasus2avro/structured/__init__.py

@@ -12,6 +12,7 @@ from .....schema_classes import PropertyValueClass
 from .....schema_classes import StructuredPropertiesClass
 from .....schema_classes import StructuredPropertyDefinitionClass
 from .....schema_classes import StructuredPropertyKeyClass
+from .....schema_classes import StructuredPropertySettingsClass
 from .....schema_classes import StructuredPropertyValueAssignmentClass
 
 
@@ -20,6 +21,7 @@ PropertyValue = PropertyValueClass
 StructuredProperties = StructuredPropertiesClass
 StructuredPropertyDefinition = StructuredPropertyDefinitionClass
 StructuredPropertyKey = StructuredPropertyKeyClass
+StructuredPropertySettings = StructuredPropertySettingsClass
 StructuredPropertyValueAssignment = StructuredPropertyValueAssignmentClass
 
 # fmt: on

datahub/metadata/schema.avsc

@@ -2995,6 +2995,7 @@
       "entityCategory": "core",
       "entityAspects": [
         "propertyDefinition",
+        "structuredPropertySettings",
         "institutionalMemory",
         "status"
       ],
@@ -3328,10 +3329,81 @@
         ],
         "name": "lastModified",
         "default": null,
-        "doc": "Created Audit stamp"
+        "doc": "Last Modified Audit stamp"
       }
     ]
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "structuredPropertySettings"
+    },
+    "name": "StructuredPropertySettings",
+    "namespace": "com.linkedin.pegasus2avro.structured",
+    "fields": [
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "isHidden",
+        "default": false,
+        "doc": "Whether or not this asset should be hidden in the main application"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInSearchFilters",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as a search filter"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInAssetSummary",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed in the asset sidebar"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showAsAssetBadge",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as an asset badge on other\nasset's headers"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInColumnsTable",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as a column in the schema field table\nin a Dataset's \"Columns\" tab."
+      },
+      {
+        "Searchable": {
+          "/time": {
+            "fieldName": "lastModifiedSettings",
+            "fieldType": "DATETIME"
+          }
+        },
+        "type": [
+          "null",
+          "com.linkedin.pegasus2avro.common.AuditStamp"
+        ],
+        "name": "lastModified",
+        "default": null,
+        "doc": "Last Modified Audit stamp"
+      }
+    ],
+    "doc": "Settings specific to a structured property entity"
+  },
   {
     "type": "record",
     "Aspect": {

datahub/metadata/schemas/StructuredPropertyDefinition.avsc

@@ -359,7 +359,7 @@
       ],
       "name": "lastModified",
       "default": null,
-      "doc": "Created Audit stamp"
+      "doc": "Last Modified Audit stamp"
     }
   ]
 }

datahub/metadata/schemas/StructuredPropertyKey.avsc

@@ -6,6 +6,7 @@
     "entityCategory": "core",
     "entityAspects": [
       "propertyDefinition",
+      "structuredPropertySettings",
       "institutionalMemory",
       "status"
     ],

datahub/metadata/schemas/StructuredPropertySettings.avsc

@@ -0,0 +1,114 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "structuredPropertySettings"
+  },
+  "name": "StructuredPropertySettings",
+  "namespace": "com.linkedin.pegasus2avro.structured",
+  "fields": [
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "isHidden",
+      "default": false,
+      "doc": "Whether or not this asset should be hidden in the main application"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "showInSearchFilters",
+      "default": false,
+      "doc": "Whether or not this asset should be displayed as a search filter"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "showInAssetSummary",
+      "default": false,
+      "doc": "Whether or not this asset should be displayed in the asset sidebar"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "showAsAssetBadge",
+      "default": false,
+      "doc": "Whether or not this asset should be displayed as an asset badge on other\nasset's headers"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "showInColumnsTable",
+      "default": false,
+      "doc": "Whether or not this asset should be displayed as a column in the schema field table\nin a Dataset's \"Columns\" tab."
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "lastModifiedSettings",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "AuditStamp",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": "long",
+              "name": "time",
+              "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": "string",
+              "name": "actor",
+              "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+              "Urn": "Urn"
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "impersonator",
+              "default": null,
+              "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+              "Urn": "Urn"
+            },
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "message",
+              "default": null,
+              "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+            }
+          ],
+          "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+        }
+      ],
+      "name": "lastModified",
+      "default": null,
+      "doc": "Last Modified Audit stamp"
+    }
+  ],
+  "doc": "Settings specific to a structured property entity"
+}

datahub/sql_parsing/schema_resolver.py

@@ -123,6 +123,13 @@ class SchemaResolver(Closeable, SchemaResolverInterface):
         )
         return urn
 
+    def resolve_urn(self, urn: str) -> Tuple[str, Optional[SchemaInfo]]:
+        schema_info = self._resolve_schema_info(urn)
+        if schema_info:
+            return urn, schema_info
+
+        return urn, None
+
     def resolve_table(self, table: _TableName) -> Tuple[str, Optional[SchemaInfo]]:
         urn = self.get_urn_for_table(table)
 
@@ -293,3 +300,19 @@ def _convert_schema_field_list_to_info(
 
 def _convert_schema_aspect_to_info(schema_metadata: SchemaMetadataClass) -> SchemaInfo:
     return _convert_schema_field_list_to_info(schema_metadata.fields)
+
+
+def match_columns_to_schema(
+    schema_info: SchemaInfo, input_columns: List[str]
+) -> List[str]:
+    column_from_gms: List[str] = list(schema_info.keys())  # list() to silent lint
+
+    gms_column_map: Dict[str, str] = {
+        column.lower(): column for column in column_from_gms
+    }
+
+    output_columns: List[str] = [
+        gms_column_map.get(column.lower(), column) for column in input_columns
+    ]
+
+    return output_columns

datahub/sql_parsing/sqlglot_lineage.py

@@ -1181,6 +1181,45 @@ def sqlglot_lineage(
         )
 
 
+@functools.lru_cache(maxsize=128)
+def create_and_cache_schema_resolver(
+    platform: str,
+    env: str,
+    graph: Optional[DataHubGraph] = None,
+    platform_instance: Optional[str] = None,
+    schema_aware: bool = True,
+) -> SchemaResolver:
+    return create_schema_resolver(
+        platform=platform,
+        env=env,
+        graph=graph,
+        platform_instance=platform_instance,
+        schema_aware=schema_aware,
+    )
+
+
+def create_schema_resolver(
+    platform: str,
+    env: str,
+    graph: Optional[DataHubGraph] = None,
+    platform_instance: Optional[str] = None,
+    schema_aware: bool = True,
+) -> SchemaResolver:
+    if graph and schema_aware:
+        return graph._make_schema_resolver(
+            platform=platform,
+            platform_instance=platform_instance,
+            env=env,
+        )
+
+    return SchemaResolver(
+        platform=platform,
+        platform_instance=platform_instance,
+        env=env,
+        graph=None,
+    )
+
+
 def create_lineage_sql_parsed_result(
     query: str,
     default_db: Optional[str],
@@ -1191,21 +1230,17 @@ def create_lineage_sql_parsed_result(
     graph: Optional[DataHubGraph] = None,
     schema_aware: bool = True,
 ) -> SqlParsingResult:
+    schema_resolver = create_schema_resolver(
+        platform=platform,
+        platform_instance=platform_instance,
+        env=env,
+        schema_aware=schema_aware,
+        graph=graph,
+    )
+
+    needs_close: bool = True
     if graph and schema_aware:
         needs_close = False
-        schema_resolver = graph._make_schema_resolver(
-            platform=platform,
-            platform_instance=platform_instance,
-            env=env,
-        )
-    else:
-        needs_close = True
-        schema_resolver = SchemaResolver(
-            platform=platform,
-            platform_instance=platform_instance,
-            env=env,
-            graph=None,
-        )
 
     try:
         return sqlglot_lineage(

datahub/testing/doctest.py

@@ -0,0 +1,12 @@
+import doctest
+from types import ModuleType
+
+
+def assert_doctest(module: ModuleType) -> None:
+    result = doctest.testmod(
+        module,
+        raise_on_error=True,
+        verbose=True,
+    )
+    if result.attempted == 0:
+        raise ValueError(f"No doctests found in {module.__name__}")