acryl-datahub-cloud 0.3.14.1rc4__py3-none-any.whl → 0.3.15rc0__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/GlobalSettingsInfo.avsc

@@ -460,6 +460,78 @@
       "default": null,
       "doc": "SSO integrations between DataHub and identity providers"
     },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "OAuthSettings",
+          "namespace": "com.linkedin.pegasus2avro.settings.global",
+          "fields": [
+            {
+              "type": {
+                "type": "array",
+                "items": {
+                  "type": "record",
+                  "name": "OAuthProvider",
+                  "namespace": "com.linkedin.pegasus2avro.settings.global",
+                  "fields": [
+                    {
+                      "type": "boolean",
+                      "name": "enabled",
+                      "doc": "Whether this OAuth provider is enabled."
+                    },
+                    {
+                      "type": "string",
+                      "name": "name",
+                      "doc": "The name of this OAuth provider. This is used for display purposes only."
+                    },
+                    {
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "jwksUri",
+                      "default": null,
+                      "doc": "The URI of the JSON Web Key Set (JWKS) endpoint for this OAuth provider."
+                    },
+                    {
+                      "type": "string",
+                      "name": "issuer",
+                      "doc": "The expected issuer (iss) claim in the JWTs issued by this OAuth provider."
+                    },
+                    {
+                      "type": "string",
+                      "name": "audience",
+                      "doc": "The expected audience (aud) claim in the JWTs issued by this OAuth provider."
+                    },
+                    {
+                      "type": "string",
+                      "name": "algorithm",
+                      "default": "RS256",
+                      "doc": "The JWT signing algorithm required for this provider.\nPrevents algorithm confusion attacks. Common values: RS256, RS384, RS512, PS256, ES256"
+                    },
+                    {
+                      "type": "string",
+                      "name": "userIdClaim",
+                      "default": "sub",
+                      "doc": "The JWT claim to use as the user identifier for this provider.\nDifferent providers use different claims (sub, email, preferred_username, etc.)"
+                    }
+                  ],
+                  "doc": "An OAuth Provider. This provides information required to validate inbound\nrequests with OAuth 2.0 bearer tokens."
+                }
+              },
+              "name": "providers",
+              "doc": "Trusted OAuth Providers"
+            }
+          ],
+          "doc": "Trust oauth providers to use for authentication."
+        }
+      ],
+      "name": "oauth",
+      "default": null,
+      "doc": "Settings related to the oauth authentication provider"
+    },
     {
       "type": [
         "null",

acryl_datahub_cloud/metadata/schemas/LineageFeatures.avsc

@@ -25,51 +25,76 @@
       "doc": "Cached number of entities related by lineage, downstream."
     },
     {
-      "type": {
-        "type": "record",
-        "name": "AuditStamp",
-        "namespace": "com.linkedin.pegasus2avro.common",
-        "fields": [
-          {
-            "type": "long",
-            "name": "time",
-            "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
-          },
-          {
-            "java": {
-              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      "Searchable": {
+        "addToFilters": true,
+        "fieldName": "hasAssetLevelLineageFeature",
+        "fieldType": "BOOLEAN",
+        "filterNameOverride": "Has Asset-Level Lineage"
+      },
+      "type": [
+        "null",
+        "boolean"
+      ],
+      "name": "hasAssetLevelLineage",
+      "default": null,
+      "doc": "Whether upstreamCount > 0 OR downstreamCount > 0\nstored to make filtering easier"
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "lineageFeaturesComputedAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "AuditStamp",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": "long",
+              "name": "time",
+              "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
             },
-            "type": "string",
-            "name": "actor",
-            "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
-            "Urn": "Urn"
-          },
-          {
-            "java": {
-              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": "string",
+              "name": "actor",
+              "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+              "Urn": "Urn"
             },
-            "type": [
-              "null",
-              "string"
-            ],
-            "name": "impersonator",
-            "default": null,
-            "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
-            "Urn": "Urn"
-          },
-          {
-            "type": [
-              "null",
-              "string"
-            ],
-            "name": "message",
-            "default": null,
-            "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
-          }
-        ],
-        "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
-      },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "impersonator",
+              "default": null,
+              "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+              "Urn": "Urn"
+            },
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "message",
+              "default": null,
+              "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+            }
+          ],
+          "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+        }
+      ],
       "name": "computedAt",
+      "default": null,
       "doc": "Record of when and how lineage features were computed."
     }
   ]

acryl_datahub_cloud/metadata/schemas/LogicalParent.avsc

@@ -140,5 +140,6 @@
       "name": "parent",
       "default": null
     }
-  ]
+  ],
+  "doc": "Relates a physical asset to a logical model."
 }

acryl_datahub_cloud/metadata/schemas/MetadataChangeEvent.avsc

@@ -2743,7 +2743,7 @@
                         },
                         "type": "string",
                         "name": "pictureLink",
-                        "default": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/default_avatar.png",
+                        "default": "assets/platforms/default_avatar.png",
                         "doc": "A URL which points to a picture which user wants to set as a profile photo"
                       },
                       {

acryl_datahub_cloud/metadata/schemas/MonitorInfo.avsc

@@ -171,6 +171,12 @@
           "namespace": "com.linkedin.pegasus2avro.monitor",
           "fields": [
             {
+              "Searchable": {
+                "/*/assertion": {
+                  "fieldName": "assertionUrn",
+                  "fieldType": "URN"
+                }
+              },
               "type": {
                 "type": "array",
                 "items": {
@@ -628,6 +634,7 @@
                                               },
                                               {
                                                 "Searchable": {
+                                                  "addToFilters": true,
                                                   "fieldName": "assertionType",
                                                   "fieldType": "KEYWORD"
                                                 },
@@ -990,9 +997,6 @@
                                                           ],
                                                           "name": "Asserts"
                                                         },
-                                                        "Searchable": {
-                                                          "fieldType": "URN"
-                                                        },
                                                         "java": {
                                                           "class": "com.linkedin.pegasus2avro.common.urn.Urn"
                                                         },
@@ -1383,9 +1387,6 @@
                                                           ],
                                                           "name": "Asserts"
                                                         },
-                                                        "Searchable": {
-                                                          "fieldType": "URN"
-                                                        },
                                                         "java": {
                                                           "class": "com.linkedin.pegasus2avro.common.urn.Urn"
                                                         },
@@ -1675,9 +1676,6 @@
                                                           ],
                                                           "name": "Asserts"
                                                         },
-                                                        "Searchable": {
-                                                          "fieldType": "URN"
-                                                        },
                                                         "java": {
                                                           "class": "com.linkedin.pegasus2avro.common.urn.Urn"
                                                         },
@@ -1757,9 +1755,6 @@
                                                           ],
                                                           "name": "Asserts"
                                                         },
-                                                        "Searchable": {
-                                                          "fieldType": "URN"
-                                                        },
                                                         "java": {
                                                           "class": "com.linkedin.pegasus2avro.common.urn.Urn"
                                                         },
@@ -2000,9 +1995,6 @@
                                                           ],
                                                           "name": "Asserts"
                                                         },
-                                                        "Searchable": {
-                                                          "fieldType": "URN"
-                                                        },
                                                         "java": {
                                                           "class": "com.linkedin.pegasus2avro.common.urn.Urn"
                                                         },
@@ -3285,6 +3277,23 @@
                                                 "name": "note",
                                                 "default": null,
                                                 "doc": "An optional note to give technical owners more context about the assertion, and how to troubleshoot it.\nThe UI will render this in markdown format."
+                                              },
+                                              {
+                                                "Searchable": {
+                                                  "fieldName": "entity",
+                                                  "fieldType": "URN"
+                                                },
+                                                "java": {
+                                                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                                                },
+                                                "type": [
+                                                  "null",
+                                                  "string"
+                                                ],
+                                                "name": "entity",
+                                                "default": null,
+                                                "doc": "The entity targeted by this assertion. Newly added field, automatically set by mutation",
+                                                "Urn": "Urn"
                                               }
                                             ],
                                             "doc": "Information about an assertion\n\nAcryl Only: Did you update AssertionWithoutAnnotations.pdl? If not, please update it."

acryl_datahub_cloud/metadata/schemas/StructuredPropertySettings.avsc

@@ -33,6 +33,15 @@
       "default": false,
       "doc": "Whether or not this asset should be displayed in the asset sidebar"
     },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "hideInAssetSummaryWhenEmpty",
+      "default": false,
+      "doc": "Whether or not this asset should be hidden in the asset sidebar (showInAssetSummary should be enabled)\nwhen its value is empty"
+    },
     {
       "Searchable": {
         "fieldType": "BOOLEAN"

acryl_datahub_cloud/sdk/assertion_input/assertion_input.py

@@ -3,13 +3,24 @@ This file contains the AssertionInput class and related classes, which are used
 validate and represent the input for creating an Assertion in DataHub.
 """
 
+import inspect
 import random
 import string
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
-from typing import Callable, Literal, Optional, Type, TypeAlias, TypeVar, Union
+from typing import (
+    Callable,
+    Collection,
+    Literal,
+    Optional,
+    Type,
+    TypeAlias,
+    TypeVar,
+    Union,
+    cast,
+)
 
 import pydantic
 import pytz
@@ -305,7 +316,9 @@ class DetectionMechanism:
             return_value = getattr(
                 DetectionMechanism, detection_mechanism_config.upper()
             )
-            if isinstance(return_value, pydantic.main.ModelMetaclass):
+            if inspect.isclass(return_value) and issubclass(
+                return_value, pydantic.BaseModel
+            ):
                 try:
                     # We try to instantiate here to let pydantic raise a helpful error
                     # about which parameters are missing
@@ -1147,10 +1160,13 @@ class _AssertionInput(ABC):
                 examples={
                     "Tags from string (tag name)": "my_tag_1",
                     "Tags from string (tag URN)": "urn:li:tag:my_tag_1",
-                    "Tags from list (mixed)": [
-                        "my_tag_1",
-                        "urn:li:tag:my_tag_2",
-                    ],
+                    "Tags from list (mixed)": cast(
+                        Collection[str],
+                        [
+                            "my_tag_1",
+                            "urn:li:tag:my_tag_2",
+                        ],
+                    ),
                 },
             )
 

acryl_datahub_cloud/sdk/assertions_client.py

@@ -67,6 +67,7 @@ from acryl_datahub_cloud.sdk.errors import SDKUsageError
 from datahub.errors import ItemNotFoundError
 from datahub.metadata import schema_classes as models
 from datahub.metadata.urns import AssertionUrn, CorpUserUrn, DatasetUrn, MonitorUrn
+from datahub.sdk.search_filters import FilterDsl
 
 if TYPE_CHECKING:
     from datahub.sdk.main_client import DataHubClient
@@ -321,6 +322,7 @@ class AssertionsClient:
                 incident_behavior=incident_behavior,
                 tags=tags,
                 created_by=updated_by,
+                enabled=enabled if enabled is not None else True,
             )
 
         # 3. Check for any issues e.g. different dataset urns
@@ -407,6 +409,7 @@ class AssertionsClient:
                 tags=tags,
                 created_by=updated_by,
                 schedule=schedule,
+                enabled=enabled if enabled is not None else True,
             )
 
         # 3. Check for any issues e.g. different dataset urns
@@ -494,6 +497,7 @@ class AssertionsClient:
                 schedule=schedule,
                 freshness_schedule_check_type=freshness_schedule_check_type,
                 lookback_window=lookback_window,
+                enabled=enabled if enabled is not None else True,
             )
 
         # 3. Check for any issues e.g. different dataset urns
@@ -584,6 +588,7 @@ class AssertionsClient:
                 schedule=schedule,
                 criteria_condition=parsed_criteria.condition,
                 criteria_parameters=parsed_criteria.parameters,
+                enabled=enabled if enabled is not None else True,
             )
 
         # 3. Check for any issues e.g. different dataset urns
@@ -686,6 +691,7 @@ class AssertionsClient:
                 tags=tags,
                 created_by=updated_by,
                 schedule=schedule,
+                enabled=enabled if enabled is not None else True,
             )
 
         # 3. Check for any issues e.g. different dataset urns
@@ -759,17 +765,37 @@ class AssertionsClient:
         except ItemNotFoundError:
             pass
 
-        # Get monitor entity
-        monitor_urn = Monitor._ensure_id(id=(dataset_urn, urn))
+        # Get monitor entity by searching for monitors where assertionUrn equals the assertion urn
+        monitor_urn: Optional[MonitorUrn] = None
         maybe_monitor_entity: Optional[Monitor] = None
         try:
-            entity = self.client.entities.get(monitor_urn)
-            if entity is not None:
-                assert isinstance(entity, Monitor)
-                maybe_monitor_entity = entity
-        except ItemNotFoundError:
+            # Search for monitor entities with assertionUrn matching the assertion urn
+            monitor_filter = FilterDsl.and_(
+                FilterDsl.entity_type("monitor"),
+                FilterDsl.custom_filter("assertionUrn", "EQUAL", [str(urn)]),
+            )
+            monitor_urns = list(self.client.search.get_urns(filter=monitor_filter))
+
+            if monitor_urns:
+                # Log if there are multiple monitors found, because this is unexpected
+                if len(monitor_urns) > 1:
+                    logger.warning(
+                        f"Multiple monitors found for assertion {urn}, which should never happen: {monitor_urns}"
+                    )
+                # Use the first matching monitor
+                monitor_urn = MonitorUrn.from_string(str(monitor_urns[0]))
+                entity = self.client.entities.get(monitor_urn)
+                if entity is not None:
+                    assert isinstance(entity, Monitor)
+                    maybe_monitor_entity = entity
+        except ItemNotFoundError as e:
+            logger.debug(f"Could not find monitor for assertion {urn}: {e}")
             pass
 
+        # If no monitor found via search, fall back to creating a new monitor with the dataset urn and assertion urn
+        if monitor_urn is None:
+            monitor_urn = Monitor._ensure_id(id=(dataset_urn, urn))
+
         return maybe_assertion_entity, monitor_urn, maybe_monitor_entity
 
     def _merge_smart_freshness_input(
@@ -2749,6 +2775,7 @@ class AssertionsClient:
                 incident_behavior=incident_behavior,
                 tags=tags,
                 created_by=updated_by,
+                enabled=enabled if enabled is not None else True,
             )
 
         # 3. Check for any issues e.g. different dataset urns
@@ -3704,6 +3731,7 @@ class AssertionsClient:
                 incident_behavior=incident_behavior,
                 tags=tags,
                 created_by=updated_by,
+                enabled=enabled if enabled is not None else True,
             )
 
         # 3. Check for any issues e.g. different dataset urns

acryl_datahub_cloud/sdk/entities/subscription.py

@@ -1,4 +1,3 @@
-import uuid
 from typing import (
     Type,
     Union,
@@ -9,6 +8,7 @@ from typing_extensions import (
     assert_never,
 )
 
+from datahub.emitter.mcp_builder import DatahubKey
 from datahub.metadata import schema_classes as models
 from datahub.metadata.urns import (
     SubscriptionUrn,
@@ -17,6 +17,24 @@ from datahub.metadata.urns import (
 from datahub.sdk.entity import Entity
 
 
+class SubscriptionKey(DatahubKey):
+    """
+    Key class for generating stable subscription identifiers.
+
+    The main goal is to have stable IDs that are deterministic based on the
+    entity and actor URNs, which helps prevent duplicate subscriptions during
+    eventual consistency scenarios when multiple subscription requests happen
+    in quick succession.
+
+    This implementation matches the behavior expected in the backend when a
+    subscription is created, ensuring consistent ID generation between the
+    Python SDK and Java backend services.
+    """
+
+    entity_urn: str
+    actor_urn: str
+
+
 class Subscription(Entity):
     """
     Subscription entity class.
@@ -37,8 +55,8 @@ class Subscription(Entity):
         self,
         # SubscriptionInfo
         info: models.SubscriptionInfoClass,
-        # Identity; it is automatically generated if not provided
-        id: Union[str, SubscriptionUrn, None] = None,
+        # Identity
+        id: Union[str, SubscriptionUrn],
     ):
         """
         Initialize the Subscription entity.
@@ -61,13 +79,11 @@ class Subscription(Entity):
         return self._urn
 
     @classmethod
-    def _ensure_id(cls, id: Union[str, SubscriptionUrn, None]) -> SubscriptionUrn:
+    def _ensure_id(cls, id: Union[str, SubscriptionUrn]) -> SubscriptionUrn:
         if isinstance(id, str):
             return SubscriptionUrn.from_string(id)
         elif isinstance(id, SubscriptionUrn):
             return id
-        elif id is None:
-            return SubscriptionUrn.from_string(f"urn:li:subscription:{uuid.uuid4()}")
         else:
             assert_never(id)
 

acryl_datahub_cloud/sdk/subscription_client.py

@@ -6,7 +6,7 @@ from typing_extensions import TypeAlias
 
 import datahub.metadata.schema_classes as models
 from acryl_datahub_cloud.sdk.entities.assertion import Assertion
-from acryl_datahub_cloud.sdk.entities.subscription import Subscription
+from acryl_datahub_cloud.sdk.entities.subscription import Subscription, SubscriptionKey
 from datahub.emitter.enum_helpers import get_enum_options
 from datahub.emitter.mce_builder import make_ts_millis
 from datahub.emitter.rest_emitter import EmitMode
@@ -120,8 +120,14 @@ class SubscriptionClient:
             skip_cache=True,
         )
         if not existing_subscriptions:
-            # new subscription
+            # new subscription - use stable ID generation
+            subscription_key = SubscriptionKey(
+                entity_urn=dataset_urn.urn(),
+                actor_urn=parsed_subscriber_urn.urn(),
+            )
+
             subscription = Subscription(
+                id=f"urn:li:subscription:{subscription_key.guid()}",
                 info=models.SubscriptionInfoClass(
                     entityUrn=dataset_urn.urn(),
                     actorUrn=parsed_subscriber_urn.urn(),