acryl-datahub-cloud 0.3.8rc4__tar.gz → 0.3.8rc6__tar.gz

{acryl_datahub_cloud-0.3.8rc4 → acryl_datahub_cloud-0.3.8rc6}/PKG-INFO

@@ -1,86 +1,86 @@
 Metadata-Version: 2.2
 Name: acryl-datahub-cloud
-Version: 0.3.8rc4
+Version: 0.3.8rc6
 Requires-Dist: avro-gen3==0.7.16
 Requires-Dist: acryl-datahub
 Provides-Extra: datahub-lineage-features
-Requires-Dist: pandas; extra == "datahub-lineage-features"
-Requires-Dist: duckdb; extra == "datahub-lineage-features"
 Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
 Requires-Dist: pyarrow; extra == "datahub-lineage-features"
+Requires-Dist: duckdb; extra == "datahub-lineage-features"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
+Requires-Dist: pandas; extra == "datahub-lineage-features"
 Provides-Extra: datahub-reporting-forms
-Requires-Dist: pandas; extra == "datahub-reporting-forms"
-Requires-Dist: boto3; extra == "datahub-reporting-forms"
-Requires-Dist: duckdb; extra == "datahub-reporting-forms"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
 Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
+Requires-Dist: duckdb; extra == "datahub-reporting-forms"
+Requires-Dist: boto3; extra == "datahub-reporting-forms"
+Requires-Dist: pandas; extra == "datahub-reporting-forms"
 Provides-Extra: datahub-reporting-extract-graph
-Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
-Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
-Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
+Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
 Provides-Extra: datahub-reporting-extract-sql
-Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
-Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
-Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
+Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
 Provides-Extra: datahub-usage-feature-reporting
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-feature-reporting"
-Requires-Dist: numpy<2; extra == "datahub-usage-feature-reporting"
+Requires-Dist: pydantic<2; extra == "datahub-usage-feature-reporting"
 Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-feature-reporting"
-Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-feature-reporting"
-Requires-Dist: polars<=1.16.0; extra == "datahub-usage-feature-reporting"
-Requires-Dist: pandas; extra == "datahub-usage-feature-reporting"
-Requires-Dist: boto3; extra == "datahub-usage-feature-reporting"
 Requires-Dist: duckdb; extra == "datahub-usage-feature-reporting"
+Requires-Dist: boto3; extra == "datahub-usage-feature-reporting"
 Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-feature-reporting"
-Requires-Dist: pydantic<2; extra == "datahub-usage-feature-reporting"
+Requires-Dist: pandas; extra == "datahub-usage-feature-reporting"
+Requires-Dist: polars==1.19.0; extra == "datahub-usage-feature-reporting"
 Requires-Dist: pyarrow; extra == "datahub-usage-feature-reporting"
+Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-feature-reporting"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-feature-reporting"
+Requires-Dist: numpy<2; extra == "datahub-usage-feature-reporting"
 Provides-Extra: datahub-metadata-sharing
 Requires-Dist: tenacity; extra == "datahub-metadata-sharing"
 Provides-Extra: acryl-cs-issues
-Requires-Dist: jinja2; extra == "acryl-cs-issues"
 Requires-Dist: openai; extra == "acryl-cs-issues"
 Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
+Requires-Dist: jinja2; extra == "acryl-cs-issues"
 Requires-Dist: zenpy; extra == "acryl-cs-issues"
 Provides-Extra: all
-Requires-Dist: tenacity; extra == "all"
-Requires-Dist: zenpy; extra == "all"
-Requires-Dist: opensearch-py==2.4.2; extra == "all"
-Requires-Dist: numpy<2; extra == "all"
-Requires-Dist: openai; extra == "all"
+Requires-Dist: pydantic<2; extra == "all"
 Requires-Dist: scipy<=1.14.1; extra == "all"
-Requires-Dist: pyarrow<=18.0.0; extra == "all"
 Requires-Dist: jinja2; extra == "all"
-Requires-Dist: polars<=1.16.0; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: slack-sdk; extra == "all"
-Requires-Dist: boto3; extra == "all"
 Requires-Dist: duckdb; extra == "all"
+Requires-Dist: boto3; extra == "all"
+Requires-Dist: tenacity; extra == "all"
+Requires-Dist: openai; extra == "all"
 Requires-Dist: elasticsearch==7.13.4; extra == "all"
-Requires-Dist: pydantic<2; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: polars==1.19.0; extra == "all"
 Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: pyarrow<=18.0.0; extra == "all"
+Requires-Dist: opensearch-py==2.4.2; extra == "all"
+Requires-Dist: numpy<2; extra == "all"
+Requires-Dist: zenpy; extra == "all"
+Requires-Dist: slack-sdk; extra == "all"
 Provides-Extra: dev
+Requires-Dist: pydantic<2; extra == "dev"
+Requires-Dist: scipy<=1.14.1; extra == "dev"
+Requires-Dist: jinja2; extra == "dev"
+Requires-Dist: duckdb; extra == "dev"
+Requires-Dist: boto3; extra == "dev"
 Requires-Dist: tenacity; extra == "dev"
+Requires-Dist: openai; extra == "dev"
+Requires-Dist: elasticsearch==7.13.4; extra == "dev"
+Requires-Dist: pandas; extra == "dev"
 Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: polars==1.19.0; extra == "dev"
+Requires-Dist: pyarrow; extra == "dev"
+Requires-Dist: pyarrow<=18.0.0; extra == "dev"
 Requires-Dist: opensearch-py==2.4.2; extra == "dev"
+Requires-Dist: acryl-datahub[dev]; extra == "dev"
 Requires-Dist: numpy<2; extra == "dev"
-Requires-Dist: openai; extra == "dev"
-Requires-Dist: scipy<=1.14.1; extra == "dev"
-Requires-Dist: pyarrow<=18.0.0; extra == "dev"
-Requires-Dist: jinja2; extra == "dev"
-Requires-Dist: polars<=1.16.0; extra == "dev"
-Requires-Dist: pandas; extra == "dev"
 Requires-Dist: slack-sdk; extra == "dev"
-Requires-Dist: boto3; extra == "dev"
-Requires-Dist: duckdb; extra == "dev"
-Requires-Dist: acryl-datahub[dev]; extra == "dev"
-Requires-Dist: pydantic<2; extra == "dev"
-Requires-Dist: pyarrow; extra == "dev"
-Requires-Dist: elasticsearch==7.13.4; extra == "dev"
 Dynamic: provides-extra
 Dynamic: requires-dist

{acryl_datahub_cloud-0.3.8rc4 → acryl_datahub_cloud-0.3.8rc6}/setup.py

@@ -20,7 +20,7 @@ plugins = {
     | aws_common
     | {
         "opensearch-py==2.4.2",
-        "polars<=1.16.0",
+        "polars==1.19.0",
         "elasticsearch==7.13.4",
         "numpy<2",
         "scipy<=1.14.1",
@@ -69,6 +69,16 @@ setup(
                 "acryl-cs-issues = acryl_datahub_cloud.acryl_cs_issues.source:AcrylCSIssuesSource",
             ],
         },
+        "include_package_data": True,
+        "package_data": {
+            "acryl_datahub_cloud": [
+                "*.json",
+            ],
+            "acryl_datahub_cloud.datahub_metadata_sharing": [
+                "scroll_shared_entities.gql",
+                "share_entity.gql",
+            ],
+        },
     },
     extras_require={
         **{plugin: list(dependencies) for (plugin, dependencies) in plugins.items()},

{acryl_datahub_cloud-0.3.8rc4 → acryl_datahub_cloud-0.3.8rc6}/src/acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.8rc4",
+  "version": "0.3.8rc6",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

acryl_datahub_cloud-0.3.8rc6/src/acryl_datahub_cloud/datahub_metadata_sharing/scroll_shared_entities.gql

@@ -0,0 +1,204 @@
+fragment shareDetails on Share {
+  lastShareResults {
+    destination {
+      urn
+    }
+    implicitShareEntity {
+      urn
+      type
+    }
+    created {
+      time
+      actor {
+          urn
+      }
+    }
+    lastAttempt {
+      time
+      actor {
+        urn
+      }
+    }
+    lastSuccess {
+      time
+      actor {
+        urn
+      }
+    }
+    status
+    message
+    shareConfig {
+      enableUpstreamLineage
+      enableDownstreamLineage
+    }
+    statusLastUpdated
+  }
+  lastUnshareResults {
+    destination {
+      urn
+    }
+    implicitShareEntity {
+      urn
+      type
+    }
+    created {
+      time
+      actor {
+        urn
+      }
+    }
+    lastAttempt {
+      time
+      actor {
+        urn
+      }
+    }
+    lastSuccess {
+      time
+      actor {
+        urn
+      }
+    }
+    status
+    message
+    shareConfig {
+      enableUpstreamLineage
+      enableDownstreamLineage
+    }
+    statusLastUpdated
+  }
+}
+
+
+query scrollSharedEntities($scrollId: String, $count: Int!) {
+  scrollAcrossEntities(input: {
+    types: [],
+    query: "*",
+    scrollId: $scrollId,
+    count: $count,
+    orFilters: [
+        {
+            and: [
+                {
+                    field: "shareDestinations",
+                    condition: EXISTS,
+                    values: []
+                }
+            ]
+        }
+    ]
+  }) {
+    nextScrollId
+    count
+    total
+    searchResults {
+        entity {
+            urn 
+            type
+            ...on Dataset {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Container {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Dashboard {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Chart {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on DataJob {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on DataFlow {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on MLModel {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on MLModelGroup {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on MLPrimaryKey {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on MLFeature {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on MLFeatureTable {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Domain {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on DataProduct {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Tag {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on CorpUser {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on CorpGroup {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on GlossaryTerm {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on GlossaryNode {
+                share {
+                    ...shareDetails
+                }
+            }
+            ...on Notebook {
+                share {
+                    ...shareDetails
+                }
+            }
+        }
+        matchedFields {
+            name
+            value
+        }
+        insights {
+            text
+            icon
+        }
+    }
+  }
+}

acryl_datahub_cloud-0.3.8rc6/src/acryl_datahub_cloud/datahub_metadata_sharing/share_entity.gql

@@ -0,0 +1,9 @@
+mutation shareEntity($entityUrn: String!, $destinationUrn: String!, $lineageDirection: ShareLineageDirection) {
+    shareEntity(input: {
+        entityUrn: $entityUrn,
+        connectionUrn: $destinationUrn,
+        lineageDirection: $lineageDirection
+    }) {
+        succeeded
+    }
+}

{acryl_datahub_cloud-0.3.8rc4 → acryl_datahub_cloud-0.3.8rc6}/src/acryl_datahub_cloud/datahub_usage_reporting/query_builder.py

@@ -3,24 +3,31 @@ from typing import Dict
 
 class QueryBuilder:
     @staticmethod
-    def get_soft_deleted_entities_query() -> Dict:
+    def get_dataset_entities_query() -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}],
+            "_source": {
+                "includes": [
+                    "urn",
+                    "lastModifiedAt",
+                    "removed",
+                    "siblings",
+                    "typeNames",
+                ]
+            },
         }
 
     @staticmethod
     def get_query_entities_query() -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}],
+            "_source": {"includes": ["urn", "lastModifiedAt", "platform", "removed"]},
             "query": {
                 "bool": {
-                    "filter": {
-                        "bool": {
-                            "must_not": [
-                                {"term": {"source": "MANUAL"}},
-                            ]
-                        }
-                    }
+                    "filter": [
+                        {"bool": {"must_not": [{"term": {"source": "MANUAL"}}]}},
+                        {"exists": {"field": "platform"}},
+                    ]
                 }
             },
         }
@@ -29,6 +36,7 @@ class QueryBuilder:
     def get_upstreams_query() -> Dict:
         return {
             "sort": [{"destination.urn": {"order": "asc"}}],
+            "_source": {"includes": ["source.urn", "destination.urn"]},
             "query": {
                 "bool": {
                     "must": [
@@ -43,23 +51,27 @@ class QueryBuilder:
     def get_dashboard_usage_query(days: int) -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}],
+            "_source": {
+                "includes": [
+                    "timestampMillis",
+                    "systemMetadata.lastObserved",
+                    "urn",
+                    "eventGranularity",
+                    "viewsCount",
+                    "uniqueUserCount",
+                    "event.userCounts",
+                ]
+            },
             "query": {
                 "bool": {
-                    "filter": {
-                        "bool": {
-                            "must": [
-                                {
-                                    "range": {
-                                        "@timestamp": {
-                                            "gte": f"now-{days}d",
-                                            "lt": "now/d",
-                                        }
-                                    }
-                                },
-                                {"term": {"isExploded": False}},
-                            ]
-                        }
-                    }
+                    "filter": [
+                        {
+                            "range": {
+                                "@timestamp": {"gte": f"now-{days}d", "lt": "now/d"}
+                            }
+                        },
+                        {"term": {"isExploded": False}},
+                    ]
                 }
             },
         }
@@ -68,24 +80,28 @@ class QueryBuilder:
     def get_dataset_usage_query(days: int) -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}],
+            "_source": {
+                "includes": [
+                    "timestampMillis",
+                    "urn",
+                    "eventGranularity",
+                    "totalSqlQueries",
+                    "uniqueUserCount",
+                    "event.userCounts",
+                    "platform",
+                ]
+            },
             "query": {
                 "bool": {
-                    "filter": {
-                        "bool": {
-                            "must": [
-                                {
-                                    "range": {
-                                        "@timestamp": {
-                                            "gte": f"now-{days}d/d",
-                                            "lt": "now/d",
-                                        }
-                                    }
-                                },
-                                {"term": {"isExploded": False}},
-                                {"range": {"totalSqlQueries": {"gt": 0}}},
-                            ]
-                        }
-                    }
+                    "filter": [
+                        {
+                            "range": {
+                                "@timestamp": {"gte": f"now-{days}d/d", "lt": "now/d"}
+                            }
+                        },
+                        {"term": {"isExploded": False}},
+                        {"range": {"totalSqlQueries": {"gt": 0}}},
+                    ]
                 }
             },
         }
@@ -94,6 +110,11 @@ class QueryBuilder:
     def get_dataset_write_usage_raw_query(days: int) -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}, {"@timestamp": {"order": "asc"}}],
+            "_source": {
+                "includes": [
+                    "urn"  # Only field needed for platform extraction via regex
+                ]
+            },
             "query": {
                 "bool": {
                     "must": [
@@ -106,9 +127,6 @@ class QueryBuilder:
                     ]
                 }
             },
-            "_source": {
-                "includes": ["urn", "@timestamp"],
-            },
         }
 
     @staticmethod
@@ -141,23 +159,27 @@ class QueryBuilder:
     def get_query_usage_query(days: int) -> Dict:
         return {
             "sort": [{"urn": {"order": "asc"}}],
+            "_source": {
+                "includes": [
+                    "timestampMillis",
+                    "systemMetadata.lastObserved",
+                    "urn",
+                    "eventGranularity",
+                    "queryCount",
+                    "uniqueUserCount",
+                    "event.userCounts",
+                ]
+            },
             "query": {
                 "bool": {
-                    "filter": {
-                        "bool": {
-                            "must": [
-                                {
-                                    "range": {
-                                        "@timestamp": {
-                                            "gte": f"now-{days}d/d",
-                                            "lt": "now/d",
-                                        }
-                                    }
-                                },
-                                {"term": {"isExploded": False}},
-                            ]
-                        }
-                    }
+                    "filter": [
+                        {
+                            "range": {
+                                "@timestamp": {"gte": f"now-{days}d/d", "lt": "now/d"}
+                            }
+                        },
+                        {"term": {"isExploded": False}},
+                    ]
                 }
             },
         }