acryl-datahub-cloud 0.3.11.1rc8__tar.gz → 0.3.12rc3__tar.gz

{acryl_datahub_cloud-0.3.11.1rc8 → acryl_datahub_cloud-0.3.12rc3}/PKG-INFO

@@ -1,47 +1,50 @@
 Metadata-Version: 2.4
 Name: acryl-datahub-cloud
-Version: 0.3.11.1rc8
+Version: 0.3.12rc3
 Requires-Dist: avro-gen3==0.7.16
 Requires-Dist: acryl-datahub
+Requires-Dist: croniter
+Requires-Dist: pytz
+Requires-Dist: types-croniter
 Provides-Extra: datahub-lineage-features
-Requires-Dist: pyarrow; extra == "datahub-lineage-features"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
 Requires-Dist: duckdb; extra == "datahub-lineage-features"
+Requires-Dist: pyarrow; extra == "datahub-lineage-features"
 Requires-Dist: pandas; extra == "datahub-lineage-features"
 Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
 Provides-Extra: datahub-reporting-forms
-Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
 Requires-Dist: termcolor==2.5.0; extra == "datahub-reporting-forms"
 Requires-Dist: duckdb; extra == "datahub-reporting-forms"
+Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
 Requires-Dist: pandas; extra == "datahub-reporting-forms"
-Requires-Dist: boto3; extra == "datahub-reporting-forms"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
+Requires-Dist: boto3; extra == "datahub-reporting-forms"
 Provides-Extra: datahub-reporting-extract-graph
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
-Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
 Provides-Extra: datahub-reporting-extract-sql
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
-Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
 Provides-Extra: datahub-usage-reporting
-Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-reporting"
-Requires-Dist: pandas; extra == "datahub-usage-reporting"
-Requires-Dist: boto3; extra == "datahub-usage-reporting"
-Requires-Dist: pydantic<2; extra == "datahub-usage-reporting"
-Requires-Dist: termcolor==2.5.0; extra == "datahub-usage-reporting"
 Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-reporting"
-Requires-Dist: polars==1.23.0; extra == "datahub-usage-reporting"
-Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
+Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
 Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-reporting"
+Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
+Requires-Dist: termcolor==2.5.0; extra == "datahub-usage-reporting"
+Requires-Dist: polars==1.30.0; extra == "datahub-usage-reporting"
+Requires-Dist: boto3; extra == "datahub-usage-reporting"
 Requires-Dist: duckdb; extra == "datahub-usage-reporting"
-Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
+Requires-Dist: pandas; extra == "datahub-usage-reporting"
+Requires-Dist: pydantic<2; extra == "datahub-usage-reporting"
+Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-reporting"
 Provides-Extra: datahub-metadata-sharing
 Requires-Dist: tenacity; extra == "datahub-metadata-sharing"
 Provides-Extra: datahub-action-request-owner
@@ -51,42 +54,44 @@ Requires-Dist: jinja2; extra == "acryl-cs-issues"
 Requires-Dist: openai; extra == "acryl-cs-issues"
 Requires-Dist: zenpy; extra == "acryl-cs-issues"
 Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
+Provides-Extra: datahub-forms-notifications
+Requires-Dist: tenacity; extra == "datahub-forms-notifications"
 Provides-Extra: all
-Requires-Dist: openai; extra == "all"
-Requires-Dist: opensearch-py==2.4.2; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: pydantic<2; extra == "all"
-Requires-Dist: boto3; extra == "all"
-Requires-Dist: jinja2; extra == "all"
-Requires-Dist: zenpy; extra == "all"
+Requires-Dist: numpy<2; extra == "all"
+Requires-Dist: pyarrow<=18.0.0; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: tenacity; extra == "all"
+Requires-Dist: zenpy; extra == "all"
+Requires-Dist: duckdb; extra == "all"
+Requires-Dist: slack-sdk; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: opensearch-py==2.4.2; extra == "all"
 Requires-Dist: scipy<=1.14.1; extra == "all"
-Requires-Dist: termcolor==2.5.0; extra == "all"
 Requires-Dist: elasticsearch==7.13.4; extra == "all"
-Requires-Dist: polars==1.23.0; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
-Requires-Dist: pyarrow<=18.0.0; extra == "all"
-Requires-Dist: slack-sdk; extra == "all"
-Requires-Dist: duckdb; extra == "all"
-Requires-Dist: numpy<2; extra == "all"
+Requires-Dist: jinja2; extra == "all"
+Requires-Dist: termcolor==2.5.0; extra == "all"
+Requires-Dist: polars==1.30.0; extra == "all"
+Requires-Dist: boto3; extra == "all"
+Requires-Dist: pydantic<2; extra == "all"
+Requires-Dist: openai; extra == "all"
 Provides-Extra: dev
-Requires-Dist: scipy<=1.14.1; extra == "dev"
-Requires-Dist: openai; extra == "dev"
-Requires-Dist: opensearch-py==2.4.2; extra == "dev"
-Requires-Dist: pandas; extra == "dev"
-Requires-Dist: boto3; extra == "dev"
-Requires-Dist: pydantic<2; extra == "dev"
-Requires-Dist: termcolor==2.5.0; extra == "dev"
+Requires-Dist: acryl-datahub[dev]; extra == "dev"
 Requires-Dist: elasticsearch==7.13.4; extra == "dev"
-Requires-Dist: polars==1.23.0; extra == "dev"
-Requires-Dist: jinja2; extra == "dev"
-Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: numpy<2; extra == "dev"
+Requires-Dist: pyarrow<=18.0.0; extra == "dev"
 Requires-Dist: pyarrow; extra == "dev"
+Requires-Dist: jinja2; extra == "dev"
+Requires-Dist: termcolor==2.5.0; extra == "dev"
 Requires-Dist: tenacity; extra == "dev"
-Requires-Dist: pyarrow<=18.0.0; extra == "dev"
-Requires-Dist: acryl-datahub[dev]; extra == "dev"
+Requires-Dist: boto3; extra == "dev"
+Requires-Dist: polars==1.30.0; extra == "dev"
+Requires-Dist: zenpy; extra == "dev"
 Requires-Dist: duckdb; extra == "dev"
-Requires-Dist: numpy<2; extra == "dev"
 Requires-Dist: slack-sdk; extra == "dev"
+Requires-Dist: pandas; extra == "dev"
+Requires-Dist: pydantic<2; extra == "dev"
+Requires-Dist: openai; extra == "dev"
+Requires-Dist: scipy<=1.14.1; extra == "dev"
+Requires-Dist: opensearch-py==2.4.2; extra == "dev"
 Dynamic: provides-extra
 Dynamic: requires-dist

{acryl_datahub_cloud-0.3.11.1rc8 → acryl_datahub_cloud-0.3.12rc3}/pyproject.toml

@@ -2,6 +2,7 @@
 build-backend = "setuptools.build_meta"
 requires = ["setuptools >= 71.1", "wheel"]
 
+
 [tool.ruff.lint.isort]
 known-first-party = ["datahub"]
 combine-as-imports = true

{acryl_datahub_cloud-0.3.11.1rc8 → acryl_datahub_cloud-0.3.12rc3}/setup.py

@@ -6,6 +6,14 @@ from setuptools import setup
 _codegen_config_file = pathlib.Path("./src/acryl_datahub_cloud/_codegen_config.json")
 _codegen_config: dict = json.loads(_codegen_config_file.read_text())
 
+# Note: We are using the croniter library for cron parsing which is different from executor, which uses apscheduler, so there is a risk of mismatch here.
+# croniter is now maintained at: https://github.com/pallets-eco/croniter
+base_requirements = [
+    "croniter",
+    "pytz",
+    "types-croniter",
+]
+
 # Adding pydantic<2 since we use pydantic models to map to pyarrow models and that is only compatible in pydantic v1
 stats_common = {"pandas", "pyarrow", "duckdb", "pydantic<2"}
 aws_common = {"boto3"}
@@ -13,7 +21,7 @@ open_search_common = {"opensearch-py==2.4.2"}
 
 plugins = {
     "datahub-lineage-features": stats_common | open_search_common,
-    "datahub-reporting-forms": stats_common 
+    "datahub-reporting-forms": stats_common
     | aws_common
     | {
         "termcolor==2.5.0",
@@ -24,7 +32,7 @@ plugins = {
     | aws_common
     | {
         "opensearch-py==2.4.2",
-        "polars==1.23.0",
+        "polars==1.30.0",
         "elasticsearch==7.13.4",
         "numpy<2",
         "scipy<=1.14.1",
@@ -34,6 +42,7 @@ plugins = {
     "datahub-metadata-sharing": {"tenacity"},
     "datahub-action-request-owner": {"tenacity"},
     "acryl-cs-issues": {"zenpy", "openai", "jinja2", "slack-sdk"},
+    "datahub-forms-notifications": {"tenacity"},
 }
 
 dev_requirements = {
@@ -50,6 +59,7 @@ dev_requirements = {
             "datahub-usage-reporting",
             "datahub-metadata-sharing",
             "acryl-cs-issues",
+            "datahub-forms-notifications",
         ]
         for dependency in plugins[plugin]
     ),
@@ -60,6 +70,7 @@ setup(
         **_codegen_config,
         "install_requires": [
             *_codegen_config["install_requires"],
+            *base_requirements,
         ],
         "entry_points": {
             **_codegen_config["entry_points"],
@@ -76,6 +87,7 @@ setup(
                 "acryl-cs-issues = acryl_datahub_cloud.acryl_cs_issues.source:AcrylCSIssuesSource",
                 "datahub-restore = acryl_datahub_cloud.datahub_restore.source:DataHubRestoreSource",
                 "datahub-action-request-owner = acryl_datahub_cloud.action_request.action_request_owner_source:ActionRequestOwnerSource",
+                "datahub-forms-notifications = acryl_datahub_cloud.datahub_forms_notifications.forms_notifications_source:DataHubFormsNotificationsSource",
             ],
         },
         "include_package_data": True,
@@ -84,12 +96,16 @@ setup(
                 "*.json",
                 "metadata/*.avsc",
                 "metadata/schemas/*.avsc",
-
             ],
             "acryl_datahub_cloud.datahub_metadata_sharing": [
                 "scroll_shared_entities.gql",
                 "share_entity.gql",
             ],
+            "acryl_datahub_cloud.datahub_forms_notifications": [
+                "get_search_results_total.gql",
+                "scroll_forms_for_notification.gql",
+                "send_form_notification_request.gql",
+            ],
         },
     },
     extras_require={
@@ -98,5 +114,5 @@ setup(
             set().union(*[requirements for _plugin, requirements in plugins.items()])
         ),
         "dev": list(dev_requirements),
-    }
-)
+    },
+)

{acryl_datahub_cloud-0.3.11.1rc8 → acryl_datahub_cloud-0.3.12rc3}/src/acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.11.1rc8",
+  "version": "0.3.12rc3",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

{acryl_datahub_cloud-0.3.11.1rc8 → acryl_datahub_cloud-0.3.12rc3}/src/acryl_datahub_cloud/action_request/action_request_owner_source.py

@@ -1,6 +1,14 @@
 import logging
+import time
 from typing import Dict, Iterable, List, Optional
 
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
 from datahub.configuration import ConfigModel
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
@@ -21,7 +29,7 @@ logger = logging.getLogger(__name__)
 
 
 class ActionRequestOwnerSourceConfig(ConfigModel):
-    batch_size: int = 20
+    batch_size: int = 200
 
 
 class ActionRequestOwnerSourceReport(SourceReport):
@@ -30,7 +38,7 @@ class ActionRequestOwnerSourceReport(SourceReport):
     correct_assignees_not_found = 0
     correct_proposal_owners = 0
     incorrect_proposal_owners = 0
-    missing_entity_owners = 0
+    missing_entity = 0
     action_request_info_not_found = 0
 
 
@@ -73,6 +81,14 @@ class ActionRequestOwnerSource(Source):
         self.report = ActionRequestOwnerSourceReport()
         self.graph = ctx.require_graph("Proposal Owner source")
         self.event_not_produced_warn = False
+        self.last_print_time = time.time()
+
+    def _print_report(self) -> None:
+        time_taken = round(time.time() - self.last_print_time, 1)
+        # Print report every 2 minutes
+        if time_taken > 120:
+            self.last_print_time = time.time()
+            logger.info(f"\n{self.report.as_string()}")
 
     def _process_action_request(
         self, action_request: Dict
@@ -82,8 +98,12 @@ class ActionRequestOwnerSource(Source):
         action_type = action_request.get("type")
         action_request_entity = action_request.get("entity")
         if action_request_entity is None:
-            logger.error(f"Action request entity not found for {action_request_urn}")
-            self.report.missing_entity_owners += 1
+            self.report.failure(
+                title="Action request entity not found",
+                message="Action request entity not found",
+                context=str(action_request_urn),
+            )
+            self.report.missing_entity += 1
             return None
         resource_urn = action_request_entity.get("urn")
         sub_resource = action_request.get("subResource")
@@ -134,8 +154,10 @@ class ActionRequestOwnerSource(Source):
         )
         if action_request_info is None:
             self.report.action_request_info_not_found += 1
-            logger.error(
-                f"Action request info not found for action request {action_request_urn}"
+            self.report.failure(
+                title="Action request info not found for action request",
+                message="Action request info not found for action request",
+                context=str(action_request_urn),
             )
             return None
         action_request_info.assignedUsers = correct_users
@@ -145,6 +167,12 @@ class ActionRequestOwnerSource(Source):
             entityUrn=action_request_urn, aspect=action_request_info
         )
 
+    @retry(
+        retry=retry_if_exception_type(ConnectionError),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=4, max=10),
+        reraise=True,
+    )
     def _get_action_requests(self, start: int) -> List:
         list_action_requests = self.graph.execute_graphql(
             query=ACTION_REQUESTS,
@@ -166,10 +194,12 @@ class ActionRequestOwnerSource(Source):
     def get_workunits(self) -> Iterable[MetadataWorkUnit]:
         start = 0
         while True:
+            logger.info(f"Fetching action requests starting from {start}")
             action_requests = self._get_action_requests(start)
             if len(action_requests) == 0:
                 break
             for action_request in action_requests:
+                self._print_report()
                 result = self._process_action_request(action_request)
                 if result is not None:
                     yield result.as_workunit()