acryl-datahub-cloud 0.3.8rc4__py3-none-any.whl → 0.3.8rc6__py3-none-any.whl

acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.8rc4",
+  "version": "0.3.8rc6",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

acryl_datahub_cloud/datahub_metadata_sharing/scroll_shared_entities.gql

@@ -0,0 +1,204 @@
+fragment shareDetails on Share {
+  lastShareResults {
+    destination {
+      urn
+    }
+    implicitShareEntity {
+      urn
+      type
+    }
+    created {
+      time
+      actor {
+          urn
+      }
+    }
+    lastAttempt {
+      time
+      actor {
+        urn
+      }
+    }
+    lastSuccess {
+      time
+      actor {
+        urn
+      }
+    }
+    status
+    message
+    shareConfig {
+      enableUpstreamLineage
+      enableDownstreamLineage
+    }
+    statusLastUpdated
+  }
+  lastUnshareResults {
+    destination {
+      urn
+    }
+    implicitShareEntity {
+      urn
+      type
+    }
+    created {
+      time
+      actor {
+        urn
+      }
+    }
+    lastAttempt {
+      time
+      actor {
+        urn
+      }
+    }
+    lastSuccess {
+      time
+      actor {
+        urn
+      }
+    }
+    status
+    message
+    shareConfig {
+      enableUpstreamLineage
+      enableDownstreamLineage
+    }
+    statusLastUpdated
+  }
+}
+
+
+query scrollSharedEntities($scrollId: String, $count: Int!) {
+  scrollAcrossEntities(input: {
+    types: [],
+    query: "*",
+    scrollId: $scrollId,
+    count: $count,
+    orFilters: [
+        {
+            and: [
+                {
+                    field: "shareDestinations",
+                    condition: EXISTS,
+                    values: []
+                }
+            ]
+        }
+    ]
+  }) {
+    nextScrollId
+    count
+    total
+    searchResults {
+        entity {
+            urn 
+            type
+            ...on Dataset {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Container {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Dashboard {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Chart {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on DataJob {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on DataFlow {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on MLModel {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on MLModelGroup {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on MLPrimaryKey {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on MLFeature {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on MLFeatureTable {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Domain {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on DataProduct {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Tag {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on CorpUser {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on CorpGroup {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on GlossaryTerm {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on GlossaryNode {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Notebook {
+                share {
+                    ...shareDetails
+                }
+            }
+        }
+        matchedFields {
+            name
+            value
+        }
+        insights {
+            text
+            icon
+        }
+    }
+  }
+}

acryl_datahub_cloud/datahub_metadata_sharing/share_entity.gql

@@ -0,0 +1,9 @@
+mutation shareEntity($entityUrn: String!, $destinationUrn: String!, $lineageDirection: ShareLineageDirection) {
+    shareEntity(input: {
+        entityUrn: $entityUrn,
+        connectionUrn: $destinationUrn,
+        lineageDirection: $lineageDirection
+    }) {
+        succeeded
+    }
+}

acryl_datahub_cloud/datahub_usage_reporting/query_builder.py

@@ -3,24 +3,31 @@ from typing import Dict
 
 class QueryBuilder:
     @staticmethod
-    def get_soft_deleted_entities_query() -> Dict:
+    def get_dataset_entities_query() -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}],
+            "_source": {
+                "includes": [
+                    "urn",
+                    "lastModifiedAt",
+                    "removed",
+                    "siblings",
+                    "typeNames",
+                ]
+            },
         }
 
     @staticmethod
     def get_query_entities_query() -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}],
+            "_source": {"includes": ["urn", "lastModifiedAt", "platform", "removed"]},
             "query": {
                 "bool": {
-                    "filter": {
-                        "bool": {
-                            "must_not": [
-                                {"term": {"source": "MANUAL"}},
-                            ]
-                        }
-                    }
+                    "filter": [
+                        {"bool": {"must_not": [{"term": {"source": "MANUAL"}}]}},
+                        {"exists": {"field": "platform"}},
+                    ]
                 }
             },
         }
@@ -29,6 +36,7 @@ class QueryBuilder:
     def get_upstreams_query() -> Dict:
         return {
             "sort": [{"destination.urn": {"order": "asc"}}],
+            "_source": {"includes": ["source.urn", "destination.urn"]},
             "query": {
                 "bool": {
                     "must": [
@@ -43,23 +51,27 @@ class QueryBuilder:
     def get_dashboard_usage_query(days: int) -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}],
+            "_source": {
+                "includes": [
+                    "timestampMillis",
+                    "systemMetadata.lastObserved",
+                    "urn",
+                    "eventGranularity",
+                    "viewsCount",
+                    "uniqueUserCount",
+                    "event.userCounts",
+                ]
+            },
             "query": {
                 "bool": {
-                    "filter": {
-                        "bool": {
-                            "must": [
-                                {
-                                    "range": {
-                                        "@timestamp": {
-                                            "gte": f"now-{days}d",
-                                            "lt": "now/d",
-                                        }
-                                    }
-                                },
-                                {"term": {"isExploded": False}},
-                            ]
-                        }
-                    }
+                    "filter": [
+                        {
+                            "range": {
+                                "@timestamp": {"gte": f"now-{days}d", "lt": "now/d"}
+                            }
+                        },
+                        {"term": {"isExploded": False}},
+                    ]
                 }
             },
         }
@@ -68,24 +80,28 @@ class QueryBuilder:
     def get_dataset_usage_query(days: int) -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}],
+            "_source": {
+                "includes": [
+                    "timestampMillis",
+                    "urn",
+                    "eventGranularity",
+                    "totalSqlQueries",
+                    "uniqueUserCount",
+                    "event.userCounts",
+                    "platform",
+                ]
+            },
             "query": {
                 "bool": {
-                    "filter": {
-                        "bool": {
-                            "must": [
-                                {
-                                    "range": {
-                                        "@timestamp": {
-                                            "gte": f"now-{days}d/d",
-                                            "lt": "now/d",
-                                        }
-                                    }
-                                },
-                                {"term": {"isExploded": False}},
-                                {"range": {"totalSqlQueries": {"gt": 0}}},
-                            ]
-                        }
-                    }
+                    "filter": [
+                        {
+                            "range": {
+                                "@timestamp": {"gte": f"now-{days}d/d", "lt": "now/d"}
+                            }
+                        },
+                        {"term": {"isExploded": False}},
+                        {"range": {"totalSqlQueries": {"gt": 0}}},
+                    ]
                 }
             },
         }
@@ -94,6 +110,11 @@ class QueryBuilder:
     def get_dataset_write_usage_raw_query(days: int) -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}, {"@timestamp": {"order": "asc"}}],
+            "_source": {
+                "includes": [
+                    "urn"  # Only field needed for platform extraction via regex
+                ]
+            },
             "query": {
                 "bool": {
                     "must": [
@@ -106,9 +127,6 @@ class QueryBuilder:
                     ]
                 }
             },
-            "_source": {
-                "includes": ["urn", "@timestamp"],
-            },
         }
 
     @staticmethod
@@ -141,23 +159,27 @@ class QueryBuilder:
     def get_query_usage_query(days: int) -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}],
+            "_source": {
+                "includes": [
+                    "timestampMillis",
+                    "systemMetadata.lastObserved",
+                    "urn",
+                    "eventGranularity",
+                    "queryCount",
+                    "uniqueUserCount",
+                    "event.userCounts",
+                ]
+            },
             "query": {
                 "bool": {
-                    "filter": {
-                        "bool": {
-                            "must": [
-                                {
-                                    "range": {
-                                        "@timestamp": {
-                                            "gte": f"now-{days}d/d",
-                                            "lt": "now/d",
-                                        }
-                                    }
-                                },
-                                {"term": {"isExploded": False}},
-                            ]
-                        }
-                    }
+                    "filter": [
+                        {
+                            "range": {
+                                "@timestamp": {"gte": f"now-{days}d/d", "lt": "now/d"}
+                            }
+                        },
+                        {"term": {"isExploded": False}},
+                    ]
                 }
             },
         }