acryl-datahub-cloud 0.3.15rc4__tar.gz → 0.3.16.1rc5__tar.gz

{acryl_datahub_cloud-0.3.15rc4 → acryl_datahub_cloud-0.3.16.1rc5}/PKG-INFO

@@ -1,98 +1,98 @@
 Metadata-Version: 2.4
 Name: acryl-datahub-cloud
-Version: 0.3.15rc4
+Version: 0.3.16.1rc5
 Requires-Python: >=3.10
 Requires-Dist: avro-gen3==0.7.16
 Requires-Dist: acryl-datahub
-Requires-Dist: acryl-datahub==1.3.0
+Requires-Dist: acryl-datahub~=1.3.1.5
 Requires-Dist: croniter
 Requires-Dist: pytz
 Requires-Dist: types-croniter
 Requires-Dist: tzlocal
 Provides-Extra: datahub-lineage-features
-Requires-Dist: tenacity; extra == "datahub-lineage-features"
 Requires-Dist: duckdb; extra == "datahub-lineage-features"
-Requires-Dist: pyarrow; extra == "datahub-lineage-features"
 Requires-Dist: pandas; extra == "datahub-lineage-features"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
+Requires-Dist: tenacity; extra == "datahub-lineage-features"
+Requires-Dist: pyarrow; extra == "datahub-lineage-features"
 Provides-Extra: datahub-reporting-forms
 Requires-Dist: duckdb; extra == "datahub-reporting-forms"
+Requires-Dist: termcolor==2.5.0; extra == "datahub-reporting-forms"
 Requires-Dist: boto3; extra == "datahub-reporting-forms"
-Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
 Requires-Dist: pandas; extra == "datahub-reporting-forms"
-Requires-Dist: termcolor==2.5.0; extra == "datahub-reporting-forms"
+Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
 Provides-Extra: datahub-reporting-extract-graph
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
 Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
 Provides-Extra: datahub-reporting-extract-sql
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
-Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
 Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
 Provides-Extra: datahub-usage-reporting
-Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-reporting"
-Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
-Requires-Dist: boto3; extra == "datahub-usage-reporting"
-Requires-Dist: polars==1.34.0; extra == "datahub-usage-reporting"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-reporting"
 Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
 Requires-Dist: duckdb; extra == "datahub-usage-reporting"
-Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
 Requires-Dist: pandas; extra == "datahub-usage-reporting"
-Requires-Dist: termcolor==2.5.0; extra == "datahub-usage-reporting"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-reporting"
 Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-reporting"
+Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-reporting"
+Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
+Requires-Dist: termcolor==2.5.0; extra == "datahub-usage-reporting"
+Requires-Dist: boto3; extra == "datahub-usage-reporting"
+Requires-Dist: polars==1.34.0; extra == "datahub-usage-reporting"
+Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
 Provides-Extra: datahub-metadata-sharing
 Requires-Dist: tenacity; extra == "datahub-metadata-sharing"
 Provides-Extra: datahub-action-request-owner
 Requires-Dist: tenacity; extra == "datahub-action-request-owner"
 Provides-Extra: acryl-cs-issues
-Requires-Dist: openai; extra == "acryl-cs-issues"
-Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
 Requires-Dist: zenpy; extra == "acryl-cs-issues"
 Requires-Dist: jinja2; extra == "acryl-cs-issues"
+Requires-Dist: openai; extra == "acryl-cs-issues"
+Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
 Provides-Extra: datahub-forms-notifications
 Requires-Dist: tenacity; extra == "datahub-forms-notifications"
 Provides-Extra: all
+Requires-Dist: numpy<2; extra == "all"
+Requires-Dist: duckdb; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: slack-sdk; extra == "all"
+Requires-Dist: opensearch-py==2.4.2; extra == "all"
+Requires-Dist: elasticsearch==7.13.4; extra == "all"
 Requires-Dist: pyarrow<=18.0.0; extra == "all"
 Requires-Dist: tenacity; extra == "all"
-Requires-Dist: scipy<=1.14.1; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: termcolor==2.5.0; extra == "all"
 Requires-Dist: boto3; extra == "all"
 Requires-Dist: polars==1.34.0; extra == "all"
 Requires-Dist: openai; extra == "all"
-Requires-Dist: opensearch-py==2.4.2; extra == "all"
-Requires-Dist: jinja2; extra == "all"
-Requires-Dist: numpy<2; extra == "all"
-Requires-Dist: slack-sdk; extra == "all"
-Requires-Dist: duckdb; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: termcolor==2.5.0; extra == "all"
 Requires-Dist: zenpy; extra == "all"
-Requires-Dist: elasticsearch==7.13.4; extra == "all"
+Requires-Dist: scipy<=1.14.1; extra == "all"
+Requires-Dist: jinja2; extra == "all"
 Provides-Extra: dev
-Requires-Dist: pandas-stubs; extra == "dev"
-Requires-Dist: boto3; extra == "dev"
-Requires-Dist: openai; extra == "dev"
-Requires-Dist: opensearch-py==2.4.2; extra == "dev"
-Requires-Dist: pyarrow; extra == "dev"
-Requires-Dist: elasticsearch==7.13.4; extra == "dev"
-Requires-Dist: scipy<=1.14.1; extra == "dev"
-Requires-Dist: pyarrow-stubs; extra == "dev"
-Requires-Dist: jinja2; extra == "dev"
-Requires-Dist: numpy<2; extra == "dev"
-Requires-Dist: tenacity; extra == "dev"
 Requires-Dist: duckdb; extra == "dev"
-Requires-Dist: pyarrow<=18.0.0; extra == "dev"
-Requires-Dist: scipy-stubs; extra == "dev"
+Requires-Dist: tenacity; extra == "dev"
 Requires-Dist: polars==1.34.0; extra == "dev"
 Requires-Dist: slack-sdk; extra == "dev"
-Requires-Dist: pandas; extra == "dev"
-Requires-Dist: termcolor==2.5.0; extra == "dev"
-Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: numpy<2; extra == "dev"
+Requires-Dist: opensearch-py==2.4.2; extra == "dev"
 Requires-Dist: acryl-datahub[dev]; extra == "dev"
+Requires-Dist: pyarrow<=18.0.0; extra == "dev"
+Requires-Dist: openai; extra == "dev"
+Requires-Dist: boto3; extra == "dev"
+Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: jinja2; extra == "dev"
+Requires-Dist: pandas-stubs; extra == "dev"
+Requires-Dist: pyarrow-stubs; extra == "dev"
+Requires-Dist: termcolor==2.5.0; extra == "dev"
+Requires-Dist: scipy-stubs; extra == "dev"
+Requires-Dist: pandas; extra == "dev"
+Requires-Dist: elasticsearch==7.13.4; extra == "dev"
+Requires-Dist: pyarrow; extra == "dev"
+Requires-Dist: scipy<=1.14.1; extra == "dev"
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python

{acryl_datahub_cloud-0.3.15rc4 → acryl_datahub_cloud-0.3.16.1rc5}/setup.py

@@ -8,7 +8,7 @@ _codegen_config: dict = json.loads(_codegen_config_file.read_text())
 
 acryl_datahub = [
     # 1.3.0: Pydantic v2 was introduced in 1.2.0.10 and later RCs fixed in OSS some missing code from /metadata-ingestion SaaS
-    "acryl-datahub==1.3.0"
+    "acryl-datahub~=1.3.1.5"
 ]
 
 # Note: We are using the croniter library for cron parsing which is different from executor, which uses apscheduler, so there is a risk of mismatch here.

{acryl_datahub_cloud-0.3.15rc4 → acryl_datahub_cloud-0.3.16.1rc5}/src/acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.15rc4",
+  "version": "0.3.16.1rc5",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

{acryl_datahub_cloud-0.3.15rc4 → acryl_datahub_cloud-0.3.16.1rc5}/src/acryl_datahub_cloud/acryl_cs_issues/models.py

@@ -4,7 +4,7 @@ from enum import Enum
 from typing import Any, Iterable, List, Optional
 
 import pydantic
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, field_validator
 
 
 class DataHubProductGlossary:
@@ -308,7 +308,8 @@ class Project(BaseModel):
         platform: str
         id: str
 
-        @validator("id", pre=True)
+        @field_validator("id", mode="before")
+        @classmethod
         def id_must_be_coerced_to_string(cls, v: Any) -> str:
             if not v:
                 raise ValueError("id must be present")
@@ -328,7 +329,8 @@ class Issue(BaseModel):
         platform: str
         id: str
 
-        @validator("id", pre=True)
+        @field_validator("id", mode="before")
+        @classmethod
         def id_must_be_coerced_to_string(cls, v: Any) -> str:
             if not v:
                 raise ValueError("id must be present")

{acryl_datahub_cloud-0.3.15rc4 → acryl_datahub_cloud-0.3.16.1rc5}/src/acryl_datahub_cloud/action_request/action_request_owner_source.py

@@ -98,7 +98,7 @@ class ActionRequestOwnerSource(Source):
         action_type = action_request.get("type")
         action_request_entity = action_request.get("entity")
         if action_request_entity is None:
-            self.report.failure(
+            self.report.warning(
                 title="Action request entity not found",
                 message="Action request entity not found",
                 context=str(action_request_urn),

{acryl_datahub_cloud-0.3.15rc4 → acryl_datahub_cloud-0.3.16.1rc5}/src/acryl_datahub_cloud/datahub_forms_notifications/forms_notifications_source.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import time
 from dataclasses import dataclass
@@ -18,6 +17,7 @@ from acryl_datahub_cloud.datahub_forms_notifications.query import (
     GRAPHQL_SCROLL_FORMS_FOR_NOTIFICATIONS,
     GRAPHQL_SEND_FORM_NOTIFICATION_REQUEST,
 )
+from acryl_datahub_cloud.graphql_utils import parse_extra_properties_for_model
 from acryl_datahub_cloud.notifications.notification_recipient_builder import (
     NotificationRecipientBuilder,
 )
@@ -399,9 +399,9 @@ class DataHubFormsNotificationsSource(Source):
         )
         for result in results:
             extra_properties = result["extraProperties"]
-            extra_properties_map = {
-                x["name"]: json.loads(x["value"]) for x in extra_properties
-            }
+            extra_properties_map = parse_extra_properties_for_model(
+                extra_properties, DataHubDatasetSearchRow
+            )
             search_row = DataHubDatasetSearchRow(**extra_properties_map)
             for owner in search_row.owners:
                 if owner.startswith(USER_URN_PREFIX):

{acryl_datahub_cloud-0.3.15rc4 → acryl_datahub_cloud-0.3.16.1rc5}/src/acryl_datahub_cloud/datahub_reporting/datahub_dataset.py

@@ -12,7 +12,7 @@ import duckdb
 import pandas
 import pyarrow as pa
 import pyarrow.parquet as pq
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, field_validator
 
 from acryl_datahub_cloud.elasticsearch.graph_service import BaseModelRow, SchemaField
 from datahub.configuration.common import ConfigModel
@@ -87,7 +87,8 @@ class FileStoreBackedDatasetConfig(ConfigModel):
 
     datahub_platform: str = "acryl"
 
-    @validator("snapshot_partitioning_strategy")
+    @field_validator("snapshot_partitioning_strategy")
+    @classmethod
     def validate_partitioning_strategy(cls, v):
         if v not in PartitioningStrategy._value2member_map_:
             raise ValueError(f"Unsupported partitioning strategy: {v}")
@@ -195,7 +196,7 @@ class DataHubBasedS3Dataset:
             )
         if isinstance(row, (BaseModel, BaseModelRow)):
             # for anything extending BaseModel, we want to use the dict representation
-            write_row: Dict[str, Any] = row.dict()
+            write_row: Dict[str, Any] = row.model_dump()
         elif isinstance(row, dict):
             write_row = row
         else:

{acryl_datahub_cloud-0.3.15rc4 → acryl_datahub_cloud-0.3.16.1rc5}/src/acryl_datahub_cloud/datahub_reporting/datahub_form_reporting.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from datetime import date, datetime, timezone
 from enum import Enum
@@ -8,6 +7,7 @@ import pandas as pd
 from pydantic import BaseModel, field_validator
 
 from acryl_datahub_cloud.elasticsearch.graph_service import BaseModelRow
+from acryl_datahub_cloud.graphql_utils import parse_extra_properties_for_model
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.graph.filters import RawSearchFilter
@@ -159,7 +159,7 @@ class DataHubFormReportingData(FormData):
         on_form_scanned: Callable[[str], Any],
     ) -> pd.DataFrame:
         return pd.DataFrame(
-            x.dict()
+            x.model_dump()
             for x in self.get_data(
                 on_asset_scanned=on_asset_scanned, on_form_scanned=on_form_scanned
             )
@@ -336,10 +336,9 @@ class DataHubFormReportingData(FormData):
             if row_index % 100 == 0:
                 logger.info(f"Scanned {row_index} assets")
             extra_properties = row["extraProperties"]
-
-            extra_properties_map = {
-                x["name"]: json.loads(x["value"]) for x in extra_properties
-            }
+            extra_properties_map = parse_extra_properties_for_model(
+                extra_properties, self.DataHubDatasetSearchRow
+            )
             search_row = self.DataHubDatasetSearchRow(**extra_properties_map)
             if on_asset_scanned:
                 on_asset_scanned(search_row.urn)

{acryl_datahub_cloud-0.3.15rc4 → acryl_datahub_cloud-0.3.16.1rc5}/src/acryl_datahub_cloud/datahub_reporting/extract_graph.py

@@ -6,7 +6,7 @@ from typing import List, Optional
 
 import boto3
 from opensearchpy import OpenSearch
-from pydantic import validator
+from pydantic import field_validator
 
 from acryl_datahub_cloud.datahub_reporting.datahub_dataset import (
     DataHubBasedS3Dataset,
@@ -43,8 +43,9 @@ class DataHubReportingExtractGraphSourceConfig(ConfigModel):
     query_timeout: int = 30
     extract_batch_size: int = 2000
 
-    @validator("extract_graph_store", pre=True, always=True)
-    def set_default_extract_soft_delete_flag(cls, v, values):
+    @field_validator("extract_graph_store", mode="before")
+    @classmethod
+    def set_default_extract_soft_delete_flag(cls, v):
         if v is not None:
             if "dataset_registration_spec" not in v:
                 v["dataset_registration_spec"] = DatasetRegistrationSpec(