acryl-datahub-cloud 0.3.14rc0__py3-none-any.whl → 0.3.14rc2__py3-none-any.whl

acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.14rc0",
+  "version": "0.3.14rc2",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

acryl_datahub_cloud/datahub_usage_reporting/excluded.py

@@ -0,0 +1,94 @@
+EXCLUDED_PATTERNS = [
+    "_ingestion",
+    "amplitude",
+    "analytics",
+    "anomaly",
+    "anomalo",
+    "airflow",
+    "app",
+    "api",
+    "aws",
+    "braze",
+    "bigquery",
+    "backfill",
+    "billing",
+    "bot",
+    "census",
+    "customer_io",
+    "connector",
+    "composer",
+    "compute",
+    "circleci",
+    "classifier",
+    "cron",
+    "datahub",
+    "data-engine",
+    "dbt",
+    "datadog",
+    "deploy",
+    "databricks",
+    "dataflow",
+    "dataplex",
+    "dagster",
+    "enterprise",
+    "export",
+    "etl",
+    "fivetran",
+    "function",
+    "google",
+    "gcp",
+    "gke",
+    "grafana",
+    "hex",
+    "hightouch",
+    "ingest",
+    "infra",
+    "infer",
+    "integration",
+    "iam",
+    "job",
+    "jenkins",
+    "looker",
+    "lineage",
+    "monte_carlo",
+    "netsuite",
+    "process",
+    "prefect",
+    "pipeline",
+    "query",
+    "redash",
+    "realtime",
+    "report",
+    "remote-executor",
+    "runner",
+    "sagemaker",
+    "salesforce",
+    "sigma",
+    "sandbox",
+    "snowplow",
+    "segment",
+    "sync",
+    "schedul",
+    "svc",
+    "sa_",
+    "_sa",
+    "sa-",
+    "-sa",
+    "snowflake",
+    "service",
+    "system",
+    "spark",
+    "task",
+    "test",
+    "team",
+    "talend",
+    "teleskope",
+    "train",
+    "tableau",
+    "unknown",
+    "wiz",
+    "warehouse",
+    "workload",
+    "workflow",
+    "worker",
+]

acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py

@@ -22,6 +22,7 @@ from polars.datatypes import DataTypeClass
 from pydantic import Field
 from scipy.stats import expon
 
+from acryl_datahub_cloud.datahub_usage_reporting.excluded import EXCLUDED_PATTERNS
 from acryl_datahub_cloud.datahub_usage_reporting.query_builder import QueryBuilder
 from acryl_datahub_cloud.datahub_usage_reporting.usage_feature_patch_builder import (
     UsageFeaturePatchBuilder,
@@ -196,6 +197,11 @@ class DataHubUsageFeatureReportingSourceConfig(
         description="Flag to generate MCP patch for usage features.'",
     )
 
+    excluded_platforms: List[str] = Field(
+        EXCLUDED_PATTERNS,
+        description="List of platforms to exclude from usage statistics collection. This is done to avoid invite user functionality to be filled with service accounts.",
+    )
+
 
 def exp_cdf(series: polars.Series) -> polars.Series:
     with PerfTimer() as timer:
@@ -1573,6 +1579,15 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
             polars.col("platform_usage_pairs").fill_null(polars.lit([]))
         )
 
+    def _filter_users(self, users_lf: polars.LazyFrame) -> polars.LazyFrame:
+        filter_condition = polars.col("user").str.contains("@")
+        for pattern in self.config.excluded_platforms:
+            filter_condition = filter_condition & ~polars.col("user").str.contains(
+                pattern
+            )
+
+        return users_lf.filter(filter_condition)
+
     def _combine_user_totals(
         self,
         dataset_usage_lf: polars.LazyFrame,
@@ -1581,13 +1596,17 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
     ) -> polars.LazyFrame:
         """Combine user totals and top_datasets_map from all sources."""
         # Collect all unique users in one operation
-        all_users_lf = polars.concat(
-            [
-                dataset_usage_lf.select("user"),
-                dashboard_usage_lf.select("user"),
-                chart_usage_lf.select("user"),
-            ]
-        ).unique()
+        all_users_lf = (
+            polars.concat(
+                [
+                    dataset_usage_lf.select("user"),
+                    dashboard_usage_lf.select("user"),
+                    chart_usage_lf.select("user"),
+                ]
+            )
+            .unique()
+            .pipe(self._filter_users)
+        )
 
         return (
             all_users_lf.join(