acryl-datahub 1.1.0.5rc11__py3-none-any.whl → 1.1.0.5rc13__py3-none-any.whl

datahub/ingestion/source/snowflake/snowflake_queries.py

@@ -787,6 +787,11 @@ class QueryLogQueryBuilder:
         would be incorrectly filtered out because they don't populate the DML arrays, causing missing lineage
         and operational metadata.
 
+        Filtering Logic:
+        A query is included if it matches:
+        - Any database name in additional_database_names (exact match), OR
+        - Any database pattern in database_pattern.allow AND NOT any pattern in database_pattern.deny
+
         Args:
             database_pattern: The AllowDenyPattern configuration for database filtering
             additional_database_names: Additional database names to always include (no pattern matching)
@@ -797,14 +802,31 @@ class QueryLogQueryBuilder:
         if not database_pattern and not additional_database_names:
             return "TRUE"
 
-        # Build the database filter conditions for pattern matching
+        # Build the database filter conditions
+        # Logic: Allow if (matches additional_database_names_allowlist) OR (matches database_pattern.allow AND NOT matches database_pattern.deny)
         # Note: Using UPPER() + RLIKE for case-insensitive matching is more performant than REGEXP_LIKE with 'i' flag
-        database_filter_parts = []
 
+        # Build additional database names condition (exact matches) - these always get included
+        additional_db_condition = None
+        if additional_database_names:
+            additional_db_conditions = []
+            for db_name in additional_database_names:
+                # Escape single quotes
+                escaped_db_name = db_name.replace("'", "''")
+                additional_db_conditions.append(
+                    f"SPLIT_PART(UPPER(o:objectName), '.', 1) = '{escaped_db_name.upper()}'"
+                )
+            if additional_db_conditions:
+                additional_db_condition = " OR ".join(additional_db_conditions)
+
+        # Build database pattern condition (allow AND NOT deny)
+        database_pattern_condition = None
         if database_pattern:
             allow_patterns = database_pattern.allow
             deny_patterns = database_pattern.deny
 
+            pattern_parts = []
+
             # Add allow patterns (if not the default "allow all")
             if allow_patterns and allow_patterns != [".*"]:
                 allow_conditions = []
@@ -815,7 +837,11 @@ class QueryLogQueryBuilder:
                         f"SPLIT_PART(UPPER(o:objectName), '.', 1) RLIKE '{escaped_pattern}'"
                     )
                 if allow_conditions:
-                    database_filter_parts.append(f"({' OR '.join(allow_conditions)})")
+                    pattern_parts.append(
+                        allow_conditions[0]
+                        if len(allow_conditions) == 1
+                        else f"({' OR '.join(allow_conditions)})"
+                    )
 
             # Add deny patterns
             if deny_patterns:
@@ -827,24 +853,36 @@ class QueryLogQueryBuilder:
                         f"SPLIT_PART(UPPER(o:objectName), '.', 1) NOT RLIKE '{escaped_pattern}'"
                     )
                 if deny_conditions:
-                    database_filter_parts.append(f"({' AND '.join(deny_conditions)})")
+                    pattern_parts.append(
+                        deny_conditions[0]
+                        if len(deny_conditions) == 1
+                        else f"({' AND '.join(deny_conditions)})"
+                    )
 
-        # Add additional database names (exact matches)
-        if additional_database_names:
-            additional_db_conditions = []
-            for db_name in additional_database_names:
-                # Escape single quotes
-                escaped_db_name = db_name.replace("'", "''")
-                additional_db_conditions.append(
-                    f"SPLIT_PART(UPPER(o:objectName), '.', 1) = '{escaped_db_name.upper()}'"
-                )
-            if additional_db_conditions:
-                database_filter_parts.append(
-                    f"({' OR '.join(additional_db_conditions)})"
-                )
+            if pattern_parts:
+                database_pattern_condition = " AND ".join(pattern_parts)
 
-        if database_filter_parts:
-            database_filter_condition = " AND ".join(database_filter_parts)
+        # Combine conditions: additional_database_names OR database_pattern
+        filter_conditions = []
+        if additional_db_condition:
+            filter_conditions.append(
+                f"({additional_db_condition})"
+                if len(additional_db_condition.split(" OR ")) > 1
+                else additional_db_condition
+            )
+        if database_pattern_condition:
+            filter_conditions.append(
+                f"({database_pattern_condition})"
+                if len(database_pattern_condition.split(" AND ")) > 1
+                else database_pattern_condition
+            )
+
+        if filter_conditions:
+            database_filter_condition = (
+                filter_conditions[0]
+                if len(filter_conditions) == 1
+                else " OR ".join(filter_conditions)
+            )
 
             # Build a condition that checks if any objects in the arrays match the database pattern
             # This implements "at least one" matching behavior: queries are allowed if they touch

datahub/ingestion/source/snowflake/snowflake_v2.py

@@ -601,6 +601,8 @@ class SnowflakeV2Source(
                         user_email_pattern=self.config.user_email_pattern,
                         pushdown_deny_usernames=self.config.pushdown_deny_usernames,
                         query_dedup_strategy=self.config.query_dedup_strategy,
+                        push_down_database_pattern_access_history=self.config.push_down_database_pattern_access_history,
+                        additional_database_names_allowlist=self.config.additional_database_names_allowlist,
                     ),
                     structured_report=self.report,
                     filters=self.filters,

datahub/ingestion/source/sql/athena.py

@@ -303,6 +303,11 @@ class AthenaConfig(SQLCommonConfig):
         print_warning=True,
     )
 
+    emit_schema_fieldpaths_as_v1: bool = pydantic.Field(
+        default=False,
+        description="Convert simple field paths to DataHub field path v1 format. Simple column paths are those that do not contain any nested fields.",
+    )
+
     profiling: AthenaProfilingConfig = AthenaProfilingConfig()
 
     def get_sql_alchemy_url(self):
@@ -641,6 +646,11 @@ class AthenaSource(SQLAlchemySource):
                 partition_keys is not None and column["name"] in partition_keys
             ),
         )
+
+        # Keeping it as individual check to make it more explicit and easier to understand
+        if not self.config.emit_schema_fieldpaths_as_v1:
+            return fields
+
         if isinstance(
             fields[0].type.type, (RecordTypeClass, MapTypeClass, ArrayTypeClass)
         ):

datahub/ingestion/source/sql/oracle.py

@@ -441,7 +441,7 @@ class OracleInspectorObjectWrapper:
             "\nac.constraint_name,"
             "\nac.constraint_type,"
             "\nacc.column_name AS local_column,"
-            "\nac.r_table_name AS remote_table,"
+            "\nac.table_name AS remote_table,"
             "\nrcc.column_name AS remote_column,"
             "\nac.r_owner AS remote_owner,"
             "\nacc.position AS loc_pos,"

datahub/ingestion/source/tableau/tableau_common.py

@@ -579,10 +579,12 @@ def get_platform(connection_type: str) -> str:
         platform = "oracle"
     elif connection_type in ("tbio", "teradata"):
         platform = "teradata"
-    elif connection_type in ("sqlserver"):
+    elif connection_type in ("sqlserver",):
         platform = "mssql"
-    elif connection_type in ("athena"):
+    elif connection_type in ("athena",):
         platform = "athena"
+    elif connection_type in ("googlebigquery",):
+        platform = "bigquery"
     elif connection_type.endswith("_jdbc"):
         # e.g. convert trino_jdbc -> trino
         platform = connection_type[: -len("_jdbc")]

datahub/metadata/_internal_schema_classes.py

@@ -15248,6 +15248,35 @@ class DataHubIngestionSourceSourceTypeClass(object):
     
     
     
+class LogicalParentClass(_Aspect):
+    # No docs available.
+
+
+    ASPECT_NAME = 'logicalParent'
+    ASPECT_INFO = {}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.logical.LogicalParent")
+
+    def __init__(self,
+        parent: "EdgeClass",
+    ):
+        super().__init__()
+        
+        self.parent = parent
+    
+    def _restore_defaults(self) -> None:
+        self.parent = EdgeClass._construct_with_defaults()
+    
+    
+    @property
+    def parent(self) -> "EdgeClass":
+        # No docs available.
+        return self._inner_dict.get('parent')  # type: ignore
+    
+    @parent.setter
+    def parent(self, value: "EdgeClass") -> None:
+        self._inner_dict['parent'] = value
+    
+    
 class AssertionKeyClass(_Aspect):
     """Key for a Assertion"""
 
@@ -16183,7 +16212,7 @@ class DatasetKeyClass(_Aspect):
 
 
     ASPECT_NAME = 'datasetKey'
-    ASPECT_INFO = {'keyForEntity': 'dataset', 'entityCategory': 'core', 'entityAspects': ['viewProperties', 'subTypes', 'datasetProfile', 'datasetUsageStatistics', 'operation', 'domains', 'applications', 'schemaMetadata', 'status', 'container', 'deprecation', 'testResults', 'siblings', 'embed', 'incidentsSummary', 'datasetProperties', 'editableDatasetProperties', 'datasetDeprecation', 'datasetUpstreamLineage', 'upstreamLineage', 'institutionalMemory', 'ownership', 'editableSchemaMetadata', 'globalTags', 'glossaryTerms', 'browsePaths', 'dataPlatformInstance', 'browsePathsV2', 'access', 'structuredProperties', 'forms', 'partitionsSummary', 'versionProperties', 'icebergCatalogInfo'], 'entityDoc': 'Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets.'}
+    ASPECT_INFO = {'keyForEntity': 'dataset', 'entityCategory': 'core', 'entityAspects': ['viewProperties', 'subTypes', 'datasetProfile', 'datasetUsageStatistics', 'operation', 'domains', 'applications', 'schemaMetadata', 'status', 'container', 'deprecation', 'testResults', 'siblings', 'embed', 'incidentsSummary', 'datasetProperties', 'editableDatasetProperties', 'datasetDeprecation', 'datasetUpstreamLineage', 'upstreamLineage', 'institutionalMemory', 'ownership', 'editableSchemaMetadata', 'globalTags', 'glossaryTerms', 'browsePaths', 'dataPlatformInstance', 'browsePathsV2', 'access', 'structuredProperties', 'forms', 'partitionsSummary', 'versionProperties', 'icebergCatalogInfo', 'logicalParent'], 'entityDoc': 'Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets.'}
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DatasetKey")
 
     def __init__(self,
@@ -16949,7 +16978,7 @@ class SchemaFieldKeyClass(_Aspect):
 
 
     ASPECT_NAME = 'schemaFieldKey'
-    ASPECT_INFO = {'keyForEntity': 'schemaField', 'entityCategory': 'core', 'entityAspects': ['schemafieldInfo', 'structuredProperties', 'forms', 'businessAttributes', 'status', 'schemaFieldAliases', 'documentation', 'testResults', 'deprecation', 'subTypes']}
+    ASPECT_INFO = {'keyForEntity': 'schemaField', 'entityCategory': 'core', 'entityAspects': ['schemafieldInfo', 'structuredProperties', 'forms', 'businessAttributes', 'status', 'schemaFieldAliases', 'documentation', 'testResults', 'deprecation', 'subTypes', 'logicalParent']}
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.SchemaFieldKey")
 
     def __init__(self,
@@ -26963,6 +26992,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.ingestion.DataHubIngestionSourceSchedule': DataHubIngestionSourceScheduleClass,
     'com.linkedin.pegasus2avro.ingestion.DataHubIngestionSourceSource': DataHubIngestionSourceSourceClass,
     'com.linkedin.pegasus2avro.ingestion.DataHubIngestionSourceSourceType': DataHubIngestionSourceSourceTypeClass,
+    'com.linkedin.pegasus2avro.logical.LogicalParent': LogicalParentClass,
     'com.linkedin.pegasus2avro.metadata.key.AssertionKey': AssertionKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.ChartKey': ChartKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.ContainerKey': ContainerKeyClass,
@@ -27470,6 +27500,7 @@ __SCHEMA_TYPES = {
     'DataHubIngestionSourceSchedule': DataHubIngestionSourceScheduleClass,
     'DataHubIngestionSourceSource': DataHubIngestionSourceSourceClass,
     'DataHubIngestionSourceSourceType': DataHubIngestionSourceSourceTypeClass,
+    'LogicalParent': LogicalParentClass,
     'AssertionKey': AssertionKeyClass,
     'ChartKey': ChartKeyClass,
     'ContainerKey': ContainerKeyClass,
@@ -27792,6 +27823,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     ERModelRelationshipPropertiesClass,
     EditableSchemaMetadataClass,
     SchemaMetadataClass,
+    LogicalParentClass,
     ChartInfoClass,
     ChartUsageStatisticsClass,
     ChartQueryClass,
@@ -28024,6 +28056,7 @@ class AspectBag(TypedDict, total=False):
     erModelRelationshipProperties: ERModelRelationshipPropertiesClass
     editableSchemaMetadata: EditableSchemaMetadataClass
     schemaMetadata: SchemaMetadataClass
+    logicalParent: LogicalParentClass
     chartInfo: ChartInfoClass
     chartUsageStatistics: ChartUsageStatisticsClass
     chartQuery: ChartQueryClass

datahub/metadata/com/linkedin/pegasus2avro/logical/__init__.py

@@ -0,0 +1,15 @@
+# mypy: ignore-errors
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# pylint: skip-file
+# fmt: off
+# isort: skip_file
+from .....schema_classes import LogicalParentClass
+
+
+LogicalParent = LogicalParentClass
+
+# fmt: on

datahub/metadata/schema.avsc

@@ -9164,6 +9164,44 @@
     "doc": "EditableSchemaMetadata stores editable changes made to schema metadata. This separates changes made from\ningestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines."
   },
   "com.linkedin.pegasus2avro.schema.SchemaMetadata",
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "logicalParent"
+    },
+    "name": "LogicalParent",
+    "namespace": "com.linkedin.pegasus2avro.logical",
+    "fields": [
+      {
+        "Relationship": {
+          "/destinationUrn": {
+            "createdActor": "parent/created/actor",
+            "createdOn": "parent/created/time",
+            "entityTypes": [
+              "dataset",
+              "schemaField"
+            ],
+            "name": "PhysicalInstanceOf",
+            "properties": "parent/properties",
+            "updatedActor": "parent/lastModified/actor",
+            "updatedOn": "parent/lastModified/time"
+          }
+        },
+        "Searchable": {
+          "/destinationUrn": {
+            "addToFilters": true,
+            "fieldName": "logicalParent",
+            "fieldType": "URN",
+            "filterNameOverride": "Physical Instance Of",
+            "hasValuesFieldName": "hasLogicalParent",
+            "queryByDefault": false
+          }
+        },
+        "type": "com.linkedin.pegasus2avro.common.Edge",
+        "name": "parent"
+      }
+    ]
+  },
   {
     "type": "record",
     "Aspect": {
@@ -10853,13 +10891,6 @@
             "namespace": "com.linkedin.pegasus2avro.query",
             "fields": [
               {
-                "Relationship": {
-                  "entityTypes": [
-                    "dataset",
-                    "schemaField"
-                  ],
-                  "name": "IsAssociatedWith"
-                },
                 "Searchable": {
                   "fieldName": "entities",
                   "fieldType": "URN"
@@ -10868,10 +10899,6 @@
                   "class": "com.linkedin.pegasus2avro.common.urn.Urn"
                 },
                 "Urn": "Urn",
-                "entityTypes": [
-                  "dataset",
-                  "schemaField"
-                ],
                 "type": "string",
                 "name": "entity",
                 "doc": "An entity which is the subject of a query."

datahub/metadata/schemas/DatasetKey.avsc

@@ -38,7 +38,8 @@
       "forms",
       "partitionsSummary",
       "versionProperties",
-      "icebergCatalogInfo"
+      "icebergCatalogInfo",
+      "logicalParent"
     ],
     "entityDoc": "Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets."
   },

datahub/metadata/schemas/LogicalParent.avsc

@@ -0,0 +1,140 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "logicalParent"
+  },
+  "name": "LogicalParent",
+  "namespace": "com.linkedin.pegasus2avro.logical",
+  "fields": [
+    {
+      "Relationship": {
+        "/destinationUrn": {
+          "createdActor": "parent/created/actor",
+          "createdOn": "parent/created/time",
+          "entityTypes": [
+            "dataset",
+            "schemaField"
+          ],
+          "name": "PhysicalInstanceOf",
+          "properties": "parent/properties",
+          "updatedActor": "parent/lastModified/actor",
+          "updatedOn": "parent/lastModified/time"
+        }
+      },
+      "Searchable": {
+        "/destinationUrn": {
+          "addToFilters": true,
+          "fieldName": "logicalParent",
+          "fieldType": "URN",
+          "filterNameOverride": "Physical Instance Of",
+          "hasValuesFieldName": "hasLogicalParent",
+          "queryByDefault": false
+        }
+      },
+      "type": {
+        "type": "record",
+        "name": "Edge",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "fields": [
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "sourceUrn",
+            "default": null,
+            "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to.",
+            "Urn": "Urn"
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": "string",
+            "name": "destinationUrn",
+            "doc": "Urn of the destination of this relationship edge.",
+            "Urn": "Urn"
+          },
+          {
+            "type": [
+              "null",
+              {
+                "type": "record",
+                "name": "AuditStamp",
+                "namespace": "com.linkedin.pegasus2avro.common",
+                "fields": [
+                  {
+                    "type": "long",
+                    "name": "time",
+                    "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+                  },
+                  {
+                    "java": {
+                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                    },
+                    "type": "string",
+                    "name": "actor",
+                    "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+                    "Urn": "Urn"
+                  },
+                  {
+                    "java": {
+                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                    },
+                    "type": [
+                      "null",
+                      "string"
+                    ],
+                    "name": "impersonator",
+                    "default": null,
+                    "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+                    "Urn": "Urn"
+                  },
+                  {
+                    "type": [
+                      "null",
+                      "string"
+                    ],
+                    "name": "message",
+                    "default": null,
+                    "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+                  }
+                ],
+                "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+              }
+            ],
+            "name": "created",
+            "default": null,
+            "doc": "Audit stamp containing who created this relationship edge and when"
+          },
+          {
+            "type": [
+              "null",
+              "com.linkedin.pegasus2avro.common.AuditStamp"
+            ],
+            "name": "lastModified",
+            "default": null,
+            "doc": "Audit stamp containing who last modified this relationship edge and when"
+          },
+          {
+            "type": [
+              "null",
+              {
+                "type": "map",
+                "values": "string"
+              }
+            ],
+            "name": "properties",
+            "default": null,
+            "doc": "A generic properties bag that allows us to store specific information on this graph edge."
+          }
+        ],
+        "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
+      },
+      "name": "parent"
+    }
+  ]
+}

datahub/metadata/schemas/QuerySubjects.avsc

@@ -15,13 +15,6 @@
           "namespace": "com.linkedin.pegasus2avro.query",
           "fields": [
             {
-              "Relationship": {
-                "entityTypes": [
-                  "dataset",
-                  "schemaField"
-                ],
-                "name": "IsAssociatedWith"
-              },
               "Searchable": {
                 "fieldName": "entities",
                 "fieldType": "URN"
@@ -32,11 +25,7 @@
               "type": "string",
               "name": "entity",
               "doc": "An entity which is the subject of a query.",
-              "Urn": "Urn",
-              "entityTypes": [
-                "dataset",
-                "schemaField"
-              ]
+              "Urn": "Urn"
             }
           ],
           "doc": "A single subject of a particular query.\nIn the future, we may evolve this model to include richer details\nabout the Query Subject in relation to the query."

datahub/metadata/schemas/SchemaFieldKey.avsc

@@ -14,7 +14,8 @@
       "documentation",
       "testResults",
       "deprecation",
-      "subTypes"
+      "subTypes",
+      "logicalParent"
     ]
   },
   "name": "SchemaFieldKey",