acryl-datahub-cloud 0.3.11rc0__py3-none-any.whl → 0.3.16.1rc0__py3-none-any.whl

acryl_datahub_cloud/datahub_restore/source.py

@@ -3,7 +3,7 @@ import time
 from functools import partial
 from typing import Any, Dict, Iterable, List, Optional
 
-from pydantic import Field, root_validator
+from pydantic import Field, model_validator
 
 from acryl_datahub_cloud.datahub_restore.do_restore import restore_indices
 from datahub.configuration.common import ConfigModel
@@ -64,7 +64,8 @@ class DataHubRestoreIndicesConfig(ConfigModel, StatefulIngestionConfigBase):
         description="Same as restore indices endpoint.",
     )
 
-    @root_validator(pre=True)
+    @model_validator(mode="before")
+    @classmethod
     def extract_assertion_info(cls, values: Dict[str, Any]) -> Dict[str, Any]:
         if values.get("urn") is None and values.get("urn_like") is None:
             raise ValueError("Either urn or urn_like must be provided.")

acryl_datahub_cloud/datahub_usage_reporting/excluded.py

@@ -0,0 +1,94 @@
+EXCLUDED_PATTERNS = [
+    "_ingestion",
+    "amplitude",
+    "analytics",
+    "anomaly",
+    "anomalo",
+    "airflow",
+    "app",
+    "api",
+    "aws",
+    "braze",
+    "bigquery",
+    "backfill",
+    "billing",
+    "bot",
+    "census",
+    "customer_io",
+    "connector",
+    "composer",
+    "compute",
+    "circleci",
+    "classifier",
+    "cron",
+    "datahub",
+    "data-engine",
+    "dbt",
+    "datadog",
+    "deploy",
+    "databricks",
+    "dataflow",
+    "dataplex",
+    "dagster",
+    "enterprise",
+    "export",
+    "etl",
+    "fivetran",
+    "function",
+    "google",
+    "gcp",
+    "gke",
+    "grafana",
+    "hex",
+    "hightouch",
+    "ingest",
+    "infra",
+    "infer",
+    "integration",
+    "iam",
+    "job",
+    "jenkins",
+    "looker",
+    "lineage",
+    "monte_carlo",
+    "netsuite",
+    "process",
+    "prefect",
+    "pipeline",
+    "query",
+    "redash",
+    "realtime",
+    "report",
+    "remote-executor",
+    "runner",
+    "sagemaker",
+    "salesforce",
+    "sigma",
+    "sandbox",
+    "snowplow",
+    "segment",
+    "sync",
+    "schedul",
+    "svc",
+    "sa_",
+    "_sa",
+    "sa-",
+    "-sa",
+    "snowflake",
+    "service",
+    "system",
+    "spark",
+    "task",
+    "test",
+    "team",
+    "talend",
+    "teleskope",
+    "train",
+    "tableau",
+    "unknown",
+    "wiz",
+    "warehouse",
+    "workload",
+    "workflow",
+    "worker",
+]

acryl_datahub_cloud/datahub_usage_reporting/query_builder.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta
 from typing import Dict
 
 
@@ -5,7 +6,7 @@ class QueryBuilder:
     @staticmethod
     def get_dataset_entities_query() -> Dict:
         return {
-            "sort": [{"urn": {"order": "asc"}}],
+            # "sort": [{"urn": {"order": "asc"}}],
             "_source": {
                 "includes": [
                     "urn",
@@ -19,15 +20,54 @@ class QueryBuilder:
         }
 
     @staticmethod
-    def get_query_entities_query() -> Dict:
+    def get_query_entities_query(days: int) -> Dict:
+        thirty_days_ago = datetime.now() - timedelta(days=days)
+        thirty_days_ago = thirty_days_ago.replace(
+            hour=0, minute=0, second=0, microsecond=0
+        )
+        epoch_ms = int(thirty_days_ago.timestamp() * 1000)
+
         return {
-            "sort": [{"urn": {"order": "asc"}}],
+            # "sort": [{"urn": {"order": "asc"}}],
             "_source": {"includes": ["urn", "lastModifiedAt", "platform", "removed"]},
             "query": {
                 "bool": {
                     "filter": [
                         {"bool": {"must_not": [{"term": {"source": "MANUAL"}}]}},
                         {"exists": {"field": "platform"}},
+                        {
+                            "bool": {
+                                "should": [
+                                    {
+                                        "bool": {
+                                            "filter": [
+                                                {"exists": {"field": "lastModifiedAt"}},
+                                                {
+                                                    "range": {
+                                                        "lastModifiedAt": {
+                                                            "gte": epoch_ms
+                                                        }
+                                                    }
+                                                },
+                                            ]
+                                        }
+                                    },
+                                    {
+                                        "bool": {
+                                            "must_not": {
+                                                "exists": {"field": "lastModifiedAt"}
+                                            },
+                                            "filter": {
+                                                "range": {
+                                                    "createdAt": {"gte": epoch_ms}
+                                                }
+                                            },
+                                        }
+                                    },
+                                ],
+                                "minimum_should_match": 1,
+                            }
+                        },
                     ]
                 }
             },
@@ -36,7 +76,7 @@ class QueryBuilder:
     @staticmethod
     def get_upstreams_query() -> Dict:
         return {
-            "sort": [{"destination.urn": {"order": "asc"}}],
+            # "sort": [{"destination.urn": {"order": "asc"}}],
             "_source": {"includes": ["source.urn", "destination.urn"]},
             "query": {
                 "bool": {
@@ -51,7 +91,7 @@ class QueryBuilder:
     @staticmethod
     def get_dashboard_usage_query(days: int) -> Dict:
         return {
-            "sort": [{"urn": {"order": "asc"}}],
+            # "sort": [{"urn": {"order": "asc"}}],
             "_source": {
                 "includes": [
                     "timestampMillis",
@@ -80,7 +120,7 @@ class QueryBuilder:
     @staticmethod
     def get_dataset_usage_query(days: int) -> Dict:
         return {
-            "sort": [{"urn": {"order": "asc"}}],
+            # "sort": [{"urn": {"order": "asc"}}],
             "_source": {
                 "includes": [
                     "timestampMillis",
@@ -110,7 +150,7 @@ class QueryBuilder:
     @staticmethod
     def get_dataset_write_usage_raw_query(days: int) -> Dict:
         return {
-            "sort": [{"urn": {"order": "asc"}}, {"@timestamp": {"order": "asc"}}],
+            # "sort": [{"urn": {"order": "asc"}}, {"@timestamp": {"order": "asc"}}],
             "_source": {
                 "includes": [
                     "urn"  # Only field needed for platform extraction via regex
@@ -159,7 +199,7 @@ class QueryBuilder:
     @staticmethod
     def get_query_usage_query(days: int) -> Dict:
         return {
-            "sort": [{"urn": {"order": "asc"}}],
+            # "sort": [{"urn": {"order": "asc"}}],
             "_source": {
                 "includes": [
                     "timestampMillis",