acryl-datahub-cloud 0.3.14rc0__tar.gz → 0.3.14rc1__tar.gz

{acryl_datahub_cloud-0.3.14rc0 → acryl_datahub_cloud-0.3.14rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acryl-datahub-cloud
-Version: 0.3.14rc0
+Version: 0.3.14rc1
 Requires-Python: >=3.10
 Requires-Dist: avro-gen3==0.7.16
 Requires-Dist: acryl-datahub
@@ -10,92 +10,92 @@ Requires-Dist: types-croniter
 Requires-Dist: tzlocal
 Provides-Extra: datahub-lineage-features
 Requires-Dist: duckdb; extra == "datahub-lineage-features"
-Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
-Requires-Dist: pyarrow; extra == "datahub-lineage-features"
+Requires-Dist: pandas; extra == "datahub-lineage-features"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
 Requires-Dist: tenacity; extra == "datahub-lineage-features"
-Requires-Dist: pandas; extra == "datahub-lineage-features"
+Requires-Dist: pyarrow; extra == "datahub-lineage-features"
+Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
 Provides-Extra: datahub-reporting-forms
 Requires-Dist: duckdb; extra == "datahub-reporting-forms"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
-Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
+Requires-Dist: pandas; extra == "datahub-reporting-forms"
 Requires-Dist: boto3; extra == "datahub-reporting-forms"
+Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
 Requires-Dist: termcolor==2.5.0; extra == "datahub-reporting-forms"
-Requires-Dist: pandas; extra == "datahub-reporting-forms"
 Provides-Extra: datahub-reporting-extract-graph
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
 Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
-Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
 Provides-Extra: datahub-reporting-extract-sql
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
-Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
 Provides-Extra: datahub-usage-reporting
-Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-reporting"
-Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
+Requires-Dist: pandas; extra == "datahub-usage-reporting"
+Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
+Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
+Requires-Dist: duckdb; extra == "datahub-usage-reporting"
 Requires-Dist: pydantic<2; extra == "datahub-usage-reporting"
 Requires-Dist: termcolor==2.5.0; extra == "datahub-usage-reporting"
 Requires-Dist: polars==1.30.0; extra == "datahub-usage-reporting"
-Requires-Dist: pandas; extra == "datahub-usage-reporting"
-Requires-Dist: duckdb; extra == "datahub-usage-reporting"
 Requires-Dist: boto3; extra == "datahub-usage-reporting"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-reporting"
-Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
+Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-reporting"
+Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
 Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-reporting"
-Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
 Provides-Extra: datahub-metadata-sharing
 Requires-Dist: tenacity; extra == "datahub-metadata-sharing"
 Provides-Extra: datahub-action-request-owner
 Requires-Dist: tenacity; extra == "datahub-action-request-owner"
 Provides-Extra: acryl-cs-issues
+Requires-Dist: jinja2; extra == "acryl-cs-issues"
 Requires-Dist: zenpy; extra == "acryl-cs-issues"
 Requires-Dist: openai; extra == "acryl-cs-issues"
-Requires-Dist: jinja2; extra == "acryl-cs-issues"
 Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
 Provides-Extra: datahub-forms-notifications
 Requires-Dist: tenacity; extra == "datahub-forms-notifications"
 Provides-Extra: all
-Requires-Dist: polars==1.30.0; extra == "all"
-Requires-Dist: openai; extra == "all"
 Requires-Dist: pandas; extra == "all"
-Requires-Dist: elasticsearch==7.13.4; extra == "all"
-Requires-Dist: pyarrow<=18.0.0; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
-Requires-Dist: pydantic<2; extra == "all"
-Requires-Dist: termcolor==2.5.0; extra == "all"
-Requires-Dist: tenacity; extra == "all"
-Requires-Dist: slack-sdk; extra == "all"
 Requires-Dist: zenpy; extra == "all"
+Requires-Dist: numpy<2; extra == "all"
 Requires-Dist: jinja2; extra == "all"
-Requires-Dist: duckdb; extra == "all"
+Requires-Dist: pydantic<2; extra == "all"
+Requires-Dist: polars==1.30.0; extra == "all"
 Requires-Dist: boto3; extra == "all"
 Requires-Dist: opensearch-py==2.4.2; extra == "all"
-Requires-Dist: numpy<2; extra == "all"
+Requires-Dist: tenacity; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: pyarrow<=18.0.0; extra == "all"
+Requires-Dist: slack-sdk; extra == "all"
 Requires-Dist: scipy<=1.14.1; extra == "all"
+Requires-Dist: duckdb; extra == "all"
+Requires-Dist: termcolor==2.5.0; extra == "all"
+Requires-Dist: elasticsearch==7.13.4; extra == "all"
+Requires-Dist: openai; extra == "all"
 Provides-Extra: dev
-Requires-Dist: scipy<=1.14.1; extra == "dev"
-Requires-Dist: pyarrow<=18.0.0; extra == "dev"
-Requires-Dist: pyarrow; extra == "dev"
-Requires-Dist: pydantic<2; extra == "dev"
-Requires-Dist: termcolor==2.5.0; extra == "dev"
-Requires-Dist: tenacity; extra == "dev"
-Requires-Dist: slack-sdk; extra == "dev"
-Requires-Dist: polars==1.30.0; extra == "dev"
+Requires-Dist: pandas; extra == "dev"
 Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: numpy<2; extra == "dev"
 Requires-Dist: jinja2; extra == "dev"
-Requires-Dist: openai; extra == "dev"
-Requires-Dist: pandas; extra == "dev"
+Requires-Dist: scipy<=1.14.1; extra == "dev"
 Requires-Dist: duckdb; extra == "dev"
+Requires-Dist: pydantic<2; extra == "dev"
+Requires-Dist: termcolor==2.5.0; extra == "dev"
+Requires-Dist: acryl-datahub[dev]; extra == "dev"
 Requires-Dist: boto3; extra == "dev"
+Requires-Dist: polars==1.30.0; extra == "dev"
 Requires-Dist: opensearch-py==2.4.2; extra == "dev"
-Requires-Dist: numpy<2; extra == "dev"
+Requires-Dist: tenacity; extra == "dev"
+Requires-Dist: pyarrow; extra == "dev"
+Requires-Dist: pyarrow<=18.0.0; extra == "dev"
 Requires-Dist: elasticsearch==7.13.4; extra == "dev"
-Requires-Dist: acryl-datahub[dev]; extra == "dev"
+Requires-Dist: slack-sdk; extra == "dev"
+Requires-Dist: openai; extra == "dev"
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python

{acryl_datahub_cloud-0.3.14rc0 → acryl_datahub_cloud-0.3.14rc1}/src/acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.14rc0",
+  "version": "0.3.14rc1",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

acryl_datahub_cloud-0.3.14rc1/src/acryl_datahub_cloud/datahub_usage_reporting/excluded.py

@@ -0,0 +1,94 @@
+EXCLUDED_PATTERNS = [
+    "_ingestion",
+    "amplitude",
+    "analytics",
+    "anomaly",
+    "anomalo",
+    "airflow",
+    "app",
+    "api",
+    "aws",
+    "braze",
+    "bigquery",
+    "backfill",
+    "billing",
+    "bot",
+    "census",
+    "customer_io",
+    "connector",
+    "composer",
+    "compute",
+    "circleci",
+    "classifier",
+    "cron",
+    "datahub",
+    "data-engine",
+    "dbt",
+    "datadog",
+    "deploy",
+    "databricks",
+    "dataflow",
+    "dataplex",
+    "dagster",
+    "enterprise",
+    "export",
+    "etl",
+    "fivetran",
+    "function",
+    "google",
+    "gcp",
+    "gke",
+    "grafana",
+    "hex",
+    "hightouch",
+    "ingest",
+    "infra",
+    "infer",
+    "integration",
+    "iam",
+    "job",
+    "jenkins",
+    "looker",
+    "lineage",
+    "monte_carlo",
+    "netsuite",
+    "process",
+    "prefect",
+    "pipeline",
+    "query",
+    "redash",
+    "realtime",
+    "report",
+    "remote-executor",
+    "runner",
+    "sagemaker",
+    "salesforce",
+    "sigma",
+    "sandbox",
+    "snowplow",
+    "segment",
+    "sync",
+    "schedul",
+    "svc",
+    "sa_",
+    "_sa",
+    "sa-",
+    "-sa",
+    "snowflake",
+    "service",
+    "system",
+    "spark",
+    "task",
+    "test",
+    "team",
+    "talend",
+    "teleskope",
+    "train",
+    "tableau",
+    "unknown",
+    "wiz",
+    "warehouse",
+    "workload",
+    "workflow",
+    "worker",
+]

{acryl_datahub_cloud-0.3.14rc0 → acryl_datahub_cloud-0.3.14rc1}/src/acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py

@@ -22,6 +22,7 @@ from polars.datatypes import DataTypeClass
 from pydantic import Field
 from scipy.stats import expon
 
+from acryl_datahub_cloud.datahub_usage_reporting.excluded import EXCLUDED_PATTERNS
 from acryl_datahub_cloud.datahub_usage_reporting.query_builder import QueryBuilder
 from acryl_datahub_cloud.datahub_usage_reporting.usage_feature_patch_builder import (
     UsageFeaturePatchBuilder,
@@ -196,6 +197,11 @@ class DataHubUsageFeatureReportingSourceConfig(
         description="Flag to generate MCP patch for usage features.'",
     )
 
+    excluded_platforms: List[str] = Field(
+        EXCLUDED_PATTERNS,
+        description="List of platforms to exclude from usage statistics collection. This is done to avoid invite user functionality to be filled with service accounts.",
+    )
+
 
 def exp_cdf(series: polars.Series) -> polars.Series:
     with PerfTimer() as timer:
@@ -1573,6 +1579,15 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
             polars.col("platform_usage_pairs").fill_null(polars.lit([]))
         )
 
+    def _filter_users(self, users_lf: polars.LazyFrame) -> polars.LazyFrame:
+        filter_condition = polars.col("user").str.contains("@")
+        for pattern in self.config.excluded_platforms:
+            filter_condition = filter_condition & ~polars.col("user").str.contains(
+                pattern
+            )
+
+        return users_lf.filter(filter_condition)
+
     def _combine_user_totals(
         self,
         dataset_usage_lf: polars.LazyFrame,
@@ -1581,13 +1596,17 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
     ) -> polars.LazyFrame:
         """Combine user totals and top_datasets_map from all sources."""
         # Collect all unique users in one operation
-        all_users_lf = polars.concat(
-            [
-                dataset_usage_lf.select("user"),
-                dashboard_usage_lf.select("user"),
-                chart_usage_lf.select("user"),
-            ]
-        ).unique()
+        all_users_lf = (
+            polars.concat(
+                [
+                    dataset_usage_lf.select("user"),
+                    dashboard_usage_lf.select("user"),
+                    chart_usage_lf.select("user"),
+                ]
+            )
+            .unique()
+            .pipe(self._filter_users)
+        )
 
         return (
             all_users_lf.join(

{acryl_datahub_cloud-0.3.14rc0 → acryl_datahub_cloud-0.3.14rc1}/src/acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/event/notification/settings/__init__.py

@@ -10,10 +10,12 @@
 from .......schema_classes import EmailNotificationSettingsClass
 from .......schema_classes import NotificationSettingsClass
 from .......schema_classes import SlackNotificationSettingsClass
+from .......schema_classes import TeamsNotificationSettingsClass
 
 
 EmailNotificationSettings = EmailNotificationSettingsClass
 NotificationSettings = NotificationSettingsClass
 SlackNotificationSettings = SlackNotificationSettingsClass
+TeamsNotificationSettings = TeamsNotificationSettingsClass
 
 # fmt: on

acryl_datahub_cloud-0.3.14rc1/src/acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/settings/asset/__init__.py

@@ -0,0 +1,19 @@
+# mypy: ignore-errors
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# pylint: skip-file
+# fmt: off
+# isort: skip_file
+from ......schema_classes import AssetSettingsClass
+from ......schema_classes import AssetSummarySettingsClass
+from ......schema_classes import AssetSummarySettingsTemplateClass
+
+
+AssetSettings = AssetSettingsClass
+AssetSummarySettings = AssetSummarySettingsClass
+AssetSummarySettingsTemplate = AssetSummarySettingsTemplateClass
+
+# fmt: on

{acryl_datahub_cloud-0.3.14rc0 → acryl_datahub_cloud-0.3.14rc1}/src/acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py

@@ -22,6 +22,9 @@ from ......schema_classes import HelpLinkClass
 from ......schema_classes import OidcSettingsClass
 from ......schema_classes import SlackIntegrationSettingsClass
 from ......schema_classes import SsoSettingsClass
+from ......schema_classes import TeamsChannelClass
+from ......schema_classes import TeamsIntegrationSettingsClass
+from ......schema_classes import TeamsUserClass
 
 
 ApplicationsSettings = ApplicationsSettingsClass
@@ -39,5 +42,8 @@ HelpLink = HelpLinkClass
 OidcSettings = OidcSettingsClass
 SlackIntegrationSettings = SlackIntegrationSettingsClass
 SsoSettings = SsoSettingsClass
+TeamsChannel = TeamsChannelClass
+TeamsIntegrationSettings = TeamsIntegrationSettingsClass
+TeamsUser = TeamsUserClass
 
 # fmt: on

{acryl_datahub_cloud-0.3.14rc0 → acryl_datahub_cloud-0.3.14rc1}/src/acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/template/__init__.py

@@ -7,19 +7,25 @@
 # pylint: skip-file
 # fmt: off
 # isort: skip_file
+from .....schema_classes import DataHubPageTemplateAssetSummaryClass
 from .....schema_classes import DataHubPageTemplatePropertiesClass
 from .....schema_classes import DataHubPageTemplateRowClass
 from .....schema_classes import DataHubPageTemplateSurfaceClass
 from .....schema_classes import DataHubPageTemplateVisibilityClass
 from .....schema_classes import PageTemplateScopeClass
 from .....schema_classes import PageTemplateSurfaceTypeClass
+from .....schema_classes import SummaryElementClass
+from .....schema_classes import SummaryElementTypeClass
 
 
+DataHubPageTemplateAssetSummary = DataHubPageTemplateAssetSummaryClass
 DataHubPageTemplateProperties = DataHubPageTemplatePropertiesClass
 DataHubPageTemplateRow = DataHubPageTemplateRowClass
 DataHubPageTemplateSurface = DataHubPageTemplateSurfaceClass
 DataHubPageTemplateVisibility = DataHubPageTemplateVisibilityClass
 PageTemplateScope = PageTemplateScopeClass
 PageTemplateSurfaceType = PageTemplateSurfaceTypeClass
+SummaryElement = SummaryElementClass
+SummaryElementType = SummaryElementTypeClass
 
 # fmt: on