acryl-datahub 0.15.0.1rc13__py3-none-any.whl → 0.15.0.1rc15__py3-none-any.whl

datahub/specific/dataset.py

@@ -1,27 +1,27 @@
-from typing import Dict, Generic, List, Optional, Tuple, TypeVar, Union
+from typing import Generic, List, Optional, Tuple, TypeVar, Union
 
-from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
+from datahub.emitter.mcp_patch_builder import MetadataPatchProposal, PatchPath
 from datahub.metadata.com.linkedin.pegasus2avro.common import TimeStamp
 from datahub.metadata.schema_classes import (
     DatasetPropertiesClass as DatasetProperties,
     EditableDatasetPropertiesClass as EditableDatasetProperties,
     EditableSchemaMetadataClass as EditableSchemaMetadata,
     FineGrainedLineageClass as FineGrainedLineage,
-    GlobalTagsClass as GlobalTags,
     GlossaryTermAssociationClass as Term,
-    GlossaryTermsClass as GlossaryTerms,
     KafkaAuditHeaderClass,
-    OwnerClass as Owner,
-    OwnershipTypeClass,
     SchemaMetadataClass,
     SystemMetadataClass,
     TagAssociationClass as Tag,
     UpstreamClass as Upstream,
     UpstreamLineageClass as UpstreamLineage,
 )
-from datahub.specific.custom_properties import CustomPropertiesPatchHelper
-from datahub.specific.ownership import OwnershipPatchHelper
-from datahub.specific.structured_properties import StructuredPropertiesPatchHelper
+from datahub.specific.aspect_helpers.custom_properties import HasCustomPropertiesPatch
+from datahub.specific.aspect_helpers.ownership import HasOwnershipPatch
+from datahub.specific.aspect_helpers.structured_properties import (
+    HasStructuredPropertiesPatch,
+)
+from datahub.specific.aspect_helpers.tags import HasTagsPatch
+from datahub.specific.aspect_helpers.terms import HasTermsPatch
 from datahub.utilities.urns.tag_urn import TagUrn
 from datahub.utilities.urns.urn import Urn
 
@@ -48,7 +48,7 @@ class FieldPatchHelper(Generic[_Parent]):
         self._parent._add_patch(
             self.aspect_name,
             "add",
-            path=f"/{self.aspect_field}/{self.field_path}/globalTags/tags/{tag.tag}",
+            path=(self.aspect_field, self.field_path, "globalTags", "tags", tag.tag),
             value=tag,
         )
         return self
@@ -59,7 +59,7 @@ class FieldPatchHelper(Generic[_Parent]):
         self._parent._add_patch(
             self.aspect_name,
             "remove",
-            path=f"/{self.aspect_field}/{self.field_path}/globalTags/tags/{tag}",
+            path=(self.aspect_field, self.field_path, "globalTags", "tags", tag),
             value={},
         )
         return self
@@ -68,7 +68,13 @@ class FieldPatchHelper(Generic[_Parent]):
         self._parent._add_patch(
             self.aspect_name,
             "add",
-            path=f"/{self.aspect_field}/{self.field_path}/glossaryTerms/terms/{term.urn}",
+            path=(
+                self.aspect_field,
+                self.field_path,
+                "glossaryTerms",
+                "terms",
+                term.urn,
+            ),
             value=term,
         )
         return self
@@ -79,7 +85,7 @@ class FieldPatchHelper(Generic[_Parent]):
         self._parent._add_patch(
             self.aspect_name,
             "remove",
-            path=f"/{self.aspect_field}/{self.field_path}/glossaryTerms/terms/{term}",
+            path=(self.aspect_field, self.field_path, "glossaryTerms", "terms", term),
             value={},
         )
         return self
@@ -88,7 +94,14 @@ class FieldPatchHelper(Generic[_Parent]):
         return self._parent
 
 
-class DatasetPatchBuilder(MetadataPatchProposal):
+class DatasetPatchBuilder(
+    HasOwnershipPatch,
+    HasCustomPropertiesPatch,
+    HasStructuredPropertiesPatch,
+    HasTagsPatch,
+    HasTermsPatch,
+    MetadataPatchProposal,
+):
     def __init__(
         self,
         urn: str,
@@ -98,34 +111,16 @@ class DatasetPatchBuilder(MetadataPatchProposal):
         super().__init__(
             urn, system_metadata=system_metadata, audit_header=audit_header
         )
-        self.custom_properties_patch_helper = CustomPropertiesPatchHelper(
-            self, DatasetProperties.ASPECT_NAME
-        )
-        self.ownership_patch_helper = OwnershipPatchHelper(self)
-        self.structured_properties_patch_helper = StructuredPropertiesPatchHelper(self)
-
-    def add_owner(self, owner: Owner) -> "DatasetPatchBuilder":
-        self.ownership_patch_helper.add_owner(owner)
-        return self
 
-    def remove_owner(
-        self, owner: str, owner_type: Optional[OwnershipTypeClass] = None
-    ) -> "DatasetPatchBuilder":
-        """
-        param: owner_type is optional
-        """
-        self.ownership_patch_helper.remove_owner(owner, owner_type)
-        return self
-
-    def set_owners(self, owners: List[Owner]) -> "DatasetPatchBuilder":
-        self.ownership_patch_helper.set_owners(owners)
-        return self
+    @classmethod
+    def _custom_properties_location(cls) -> Tuple[str, PatchPath]:
+        return DatasetProperties.ASPECT_NAME, ("customProperties",)
 
     def add_upstream_lineage(self, upstream: Upstream) -> "DatasetPatchBuilder":
         self._add_patch(
             UpstreamLineage.ASPECT_NAME,
             "add",
-            path=f"/upstreams/{self.quote(upstream.dataset)}",
+            path=("upstreams", upstream.dataset),
             value=upstream,
         )
         return self
@@ -136,14 +131,14 @@ class DatasetPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             UpstreamLineage.ASPECT_NAME,
             "remove",
-            path=f"/upstreams/{dataset}",
+            path=("upstreams", dataset),
             value={},
         )
         return self
 
     def set_upstream_lineages(self, upstreams: List[Upstream]) -> "DatasetPatchBuilder":
         self._add_patch(
-            UpstreamLineage.ASPECT_NAME, "add", path="/upstreams", value=upstreams
+            UpstreamLineage.ASPECT_NAME, "add", path=("upstreams",), value=upstreams
         )
         return self
 
@@ -159,7 +154,7 @@ class DatasetPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 UpstreamLineage.ASPECT_NAME,
                 "add",
-                path=DatasetPatchBuilder.quote_fine_grained_path(
+                path=self._build_fine_grained_path(
                     transform_op, downstream_urn, query_id, upstream_urn
                 ),
                 value={"confidenceScore": fine_grained_lineage.confidenceScore},
@@ -179,12 +174,15 @@ class DatasetPatchBuilder(MetadataPatchProposal):
         return transform_op, downstream_urn, query_id
 
     @classmethod
-    def quote_fine_grained_path(
+    def _build_fine_grained_path(
         cls, transform_op: str, downstream_urn: str, query_id: str, upstream_urn: str
-    ) -> str:
+    ) -> PatchPath:
         return (
-            f"/fineGrainedLineages/{cls.quote(transform_op)}/"
-            f"{cls.quote(downstream_urn)}/{cls.quote(query_id)}/{cls.quote(upstream_urn)}"
+            "fineGrainedLineages",
+            transform_op,
+            downstream_urn,
+            query_id,
+            upstream_urn,
         )
 
     def remove_fine_grained_upstream_lineage(
@@ -199,7 +197,7 @@ class DatasetPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 UpstreamLineage.ASPECT_NAME,
                 "remove",
-                path=DatasetPatchBuilder.quote_fine_grained_path(
+                path=self._build_fine_grained_path(
                     transform_op, downstream_urn, query_id, upstream_urn
                 ),
                 value={},
@@ -212,37 +210,11 @@ class DatasetPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             UpstreamLineage.ASPECT_NAME,
             "add",
-            path="/fineGrainedLineages",
+            path=("fineGrainedLineages",),
             value=fine_grained_lineages,
         )
         return self
 
-    def add_tag(self, tag: Tag) -> "DatasetPatchBuilder":
-        self._add_patch(
-            GlobalTags.ASPECT_NAME, "add", path=f"/tags/{tag.tag}", value=tag
-        )
-        return self
-
-    def remove_tag(self, tag: Union[str, Urn]) -> "DatasetPatchBuilder":
-        if isinstance(tag, str) and not tag.startswith("urn:li:tag:"):
-            tag = TagUrn.create_from_id(tag)
-        self._add_patch(GlobalTags.ASPECT_NAME, "remove", path=f"/tags/{tag}", value={})
-        return self
-
-    def add_term(self, term: Term) -> "DatasetPatchBuilder":
-        self._add_patch(
-            GlossaryTerms.ASPECT_NAME, "add", path=f"/terms/{term.urn}", value=term
-        )
-        return self
-
-    def remove_term(self, term: Union[str, Urn]) -> "DatasetPatchBuilder":
-        if isinstance(term, str) and not term.startswith("urn:li:glossaryTerm:"):
-            term = "urn:li:glossaryTerm:" + term
-        self._add_patch(
-            GlossaryTerms.ASPECT_NAME, "remove", path=f"/terms/{term}", value={}
-        )
-        return self
-
     def for_field(
         self, field_path: str, editable: bool = True
     ) -> FieldPatchHelper["DatasetPatchBuilder"]:
@@ -269,38 +241,11 @@ class DatasetPatchBuilder(MetadataPatchProposal):
                     else EditableDatasetProperties.ASPECT_NAME
                 ),
                 "add",
-                path="/description",
+                path=("description",),
                 value=description,
             )
         return self
 
-    def set_custom_properties(
-        self, custom_properties: Dict[str, str]
-    ) -> "DatasetPatchBuilder":
-        self._add_patch(
-            DatasetProperties.ASPECT_NAME,
-            "add",
-            path="/customProperties",
-            value=custom_properties,
-        )
-        return self
-
-    def add_custom_property(self, key: str, value: str) -> "DatasetPatchBuilder":
-        self.custom_properties_patch_helper.add_property(key, value)
-        return self
-
-    def add_custom_properties(
-        self, custom_properties: Optional[Dict[str, str]] = None
-    ) -> "DatasetPatchBuilder":
-        if custom_properties is not None:
-            for key, value in custom_properties.items():
-                self.custom_properties_patch_helper.add_property(key, value)
-        return self
-
-    def remove_custom_property(self, key: str) -> "DatasetPatchBuilder":
-        self.custom_properties_patch_helper.remove_property(key)
-        return self
-
     def set_display_name(
         self, display_name: Optional[str] = None
     ) -> "DatasetPatchBuilder":
@@ -308,7 +253,7 @@ class DatasetPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 DatasetProperties.ASPECT_NAME,
                 "add",
-                path="/name",
+                path=("name",),
                 value=display_name,
             )
         return self
@@ -320,7 +265,7 @@ class DatasetPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 DatasetProperties.ASPECT_NAME,
                 "add",
-                path="/qualifiedName",
+                path=("qualifiedName",),
                 value=qualified_name,
             )
         return self
@@ -332,7 +277,7 @@ class DatasetPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 DatasetProperties.ASPECT_NAME,
                 "add",
-                path="/created",
+                path=("created",),
                 value=timestamp,
             )
         return self
@@ -344,37 +289,7 @@ class DatasetPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 DatasetProperties.ASPECT_NAME,
                 "add",
-                path="/lastModified",
+                path=("lastModified",),
                 value=timestamp,
             )
         return self
-
-    def set_structured_property(
-        self, property_name: str, value: Union[str, float, List[Union[str, float]]]
-    ) -> "DatasetPatchBuilder":
-        """
-        This is a helper method to set a structured property.
-        @param property_name: the name of the property (either bare or urn form)
-        @param value: the value of the property (for multi-valued properties, this can be a list)
-        """
-        self.structured_properties_patch_helper.set_property(property_name, value)
-        return self
-
-    def add_structured_property(
-        self, property_name: str, value: Union[str, float]
-    ) -> "DatasetPatchBuilder":
-        """
-        This is a helper method to add a structured property.
-        @param property_name: the name of the property (either bare or urn form)
-        @param value: the value of the property (for multi-valued properties, this value will be appended to the list)
-        """
-        self.structured_properties_patch_helper.add_property(property_name, value)
-        return self
-
-    def remove_structured_property(self, property_name: str) -> "DatasetPatchBuilder":
-        """
-        This is a helper method to remove a structured property.
-        @param property_name: the name of the property (either bare or urn form)
-        """
-        self.structured_properties_patch_helper.remove_property(property_name)
-        return self

datahub/specific/form.py

@@ -5,15 +5,13 @@ from datahub.metadata.schema_classes import (
     FormInfoClass as FormInfo,
     FormPromptClass,
     KafkaAuditHeaderClass,
-    OwnerClass as Owner,
-    OwnershipTypeClass,
     SystemMetadataClass,
 )
-from datahub.specific.ownership import OwnershipPatchHelper
+from datahub.specific.aspect_helpers.ownership import HasOwnershipPatch
 from datahub.utilities.urns.urn import Urn
 
 
-class FormPatchBuilder(MetadataPatchProposal):
+class FormPatchBuilder(HasOwnershipPatch, MetadataPatchProposal):
     def __init__(
         self,
         urn: str,
@@ -23,31 +21,13 @@ class FormPatchBuilder(MetadataPatchProposal):
         super().__init__(
             urn, system_metadata=system_metadata, audit_header=audit_header
         )
-        self.ownership_patch_helper = OwnershipPatchHelper(self)
-
-    def add_owner(self, owner: Owner) -> "FormPatchBuilder":
-        self.ownership_patch_helper.add_owner(owner)
-        return self
-
-    def remove_owner(
-        self, owner: str, owner_type: Optional[OwnershipTypeClass] = None
-    ) -> "FormPatchBuilder":
-        """
-        param: owner_type is optional
-        """
-        self.ownership_patch_helper.remove_owner(owner, owner_type)
-        return self
-
-    def set_owners(self, owners: List[Owner]) -> "FormPatchBuilder":
-        self.ownership_patch_helper.set_owners(owners)
-        return self
 
     def set_name(self, name: Optional[str] = None) -> "FormPatchBuilder":
         if name is not None:
             self._add_patch(
                 FormInfo.ASPECT_NAME,
                 "add",
-                path="/name",
+                path=("name",),
                 value=name,
             )
         return self
@@ -57,7 +37,7 @@ class FormPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 FormInfo.ASPECT_NAME,
                 "add",
-                path="/description",
+                path=("description",),
                 value=description,
             )
         return self
@@ -67,7 +47,7 @@ class FormPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 FormInfo.ASPECT_NAME,
                 "add",
-                path="/type",
+                path=("type",),
                 value=type,
             )
         return self
@@ -76,7 +56,7 @@ class FormPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             FormInfo.ASPECT_NAME,
             "add",
-            path=f"/prompts/{self.quote(prompt.id)}",
+            path=("prompts", prompt.id),
             value=prompt,
         )
         return self
@@ -90,7 +70,7 @@ class FormPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             FormInfo.ASPECT_NAME,
             "remove",
-            path=f"/prompts/{self.quote(prompt_id)}",
+            path=("prompts", prompt_id),
             value=prompt_id,
         )
         return self
@@ -104,7 +84,7 @@ class FormPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             FormInfo.ASPECT_NAME,
             "add",
-            path="/actors/owners",
+            path=("actors", "owners"),
             value=is_ownership,
         )
         return self
@@ -113,7 +93,7 @@ class FormPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             FormInfo.ASPECT_NAME,
             "add",
-            path=f"/actors/users/{self.quote(str(user_urn))}",
+            path=("actors", "users", user_urn),
             value=user_urn,
         )
         return self
@@ -122,7 +102,7 @@ class FormPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             FormInfo.ASPECT_NAME,
             "remove",
-            path=f"/actors/users/{self.quote(str(user_urn))}",
+            path=("actors", "users", user_urn),
             value=user_urn,
         )
         return self
@@ -131,7 +111,7 @@ class FormPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             FormInfo.ASPECT_NAME,
             "add",
-            path=f"/actors/groups/{self.quote(str(group_urn))}",
+            path=("actors", "groups", group_urn),
             value=group_urn,
         )
         return self
@@ -140,7 +120,7 @@ class FormPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             FormInfo.ASPECT_NAME,
             "remove",
-            path=f"/actors/groups/{self.quote(str(group_urn))}",
+            path=("actors", "groups", group_urn),
             value=group_urn,
         )
         return self

datahub/specific/structured_property.py

@@ -29,7 +29,7 @@ class StructuredPropertyPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             StructuredPropertyDefinition.ASPECT_NAME,
             "add",
-            path="/qualifiedName",
+            path=("qualifiedName",),
             value=qualified_name,
         )
         return self
@@ -41,7 +41,7 @@ class StructuredPropertyPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 StructuredPropertyDefinition.ASPECT_NAME,
                 "add",
-                path="/displayName",
+                path=("displayName",),
                 value=display_name,
             )
         return self
@@ -53,7 +53,7 @@ class StructuredPropertyPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             StructuredPropertyDefinition.ASPECT_NAME,
             "add",
-            path="/valueType",
+            path=("valueType",),
             value=value_type,
         )
         return self
@@ -66,7 +66,7 @@ class StructuredPropertyPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 StructuredPropertyDefinition.ASPECT_NAME,
                 "add",
-                path="/typeQualifier",
+                path=("typeQualifier",),
                 value=type_qualifier,
             )
         return self
@@ -78,7 +78,7 @@ class StructuredPropertyPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             StructuredPropertyDefinition.ASPECT_NAME,
             "add",
-            path=f"/allowedValues/{str(allowed_value.get('value'))}",
+            path=("allowedValues", str(allowed_value.get("value"))),
             value=allowed_value,
         )
         return self
@@ -87,7 +87,7 @@ class StructuredPropertyPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             StructuredPropertyDefinition.ASPECT_NAME,
             "add",
-            path="/cardinality",
+            path=("cardinality",),
             value=cardinality,
         )
         return self
@@ -98,7 +98,7 @@ class StructuredPropertyPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             StructuredPropertyDefinition.ASPECT_NAME,
             "add",
-            path=f"/entityTypes/{self.quote(str(entity_type))}",
+            path=("entityTypes", str(entity_type)),
             value=entity_type,
         )
         return self
@@ -110,7 +110,7 @@ class StructuredPropertyPatchBuilder(MetadataPatchProposal):
             self._add_patch(
                 StructuredPropertyDefinition.ASPECT_NAME,
                 "add",
-                path="/description",
+                path=("description",),
                 value=description,
             )
         return self
@@ -119,7 +119,7 @@ class StructuredPropertyPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             StructuredPropertyDefinition.ASPECT_NAME,
             "add",
-            path="/immutable",
+            path=("immutable",),
             value=immutable,
         )
         return self

datahub/sql_parsing/sql_parsing_aggregator.py

@@ -198,7 +198,7 @@ class TableSwap:
 
 @dataclasses.dataclass
 class PreparsedQuery:
-    # If not provided, we will generate one using the fast fingerprint generator.
+    # If not provided, we will generate one using the fingerprint generator.
     query_id: Optional[QueryId]
 
     query_text: str
@@ -622,7 +622,6 @@ class SqlParsingAggregator(Closeable):
             query_fingerprint = get_query_fingerprint(
                 known_query_lineage.query_text,
                 platform=self.platform.platform_name,
-                fast=True,
             )
         formatted_query = self._maybe_format_query(known_query_lineage.query_text)
 
@@ -848,7 +847,6 @@ class SqlParsingAggregator(Closeable):
             query_fingerprint = get_query_fingerprint(
                 parsed.query_text,
                 platform=self.platform.platform_name,
-                fast=True,
             )
 
         # Format the query.

datahub/specific/custom_properties.py

@@ -1,37 +0,0 @@
-from typing import Generic, TypeVar
-
-from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
-
-_Parent = TypeVar("_Parent", bound=MetadataPatchProposal)
-
-
-class CustomPropertiesPatchHelper(Generic[_Parent]):
-    def __init__(
-        self,
-        parent: _Parent,
-        aspect_name: str,
-    ) -> None:
-        self.aspect_name = aspect_name
-        self._parent = parent
-        self.aspect_field = "customProperties"
-
-    def parent(self) -> _Parent:
-        return self._parent
-
-    def add_property(self, key: str, value: str) -> "CustomPropertiesPatchHelper":
-        self._parent._add_patch(
-            self.aspect_name,
-            "add",
-            path=f"/{self.aspect_field}/{key}",
-            value=value,
-        )
-        return self
-
-    def remove_property(self, key: str) -> "CustomPropertiesPatchHelper":
-        self._parent._add_patch(
-            self.aspect_name,
-            "remove",
-            path=f"/{self.aspect_field}/{key}",
-            value={},
-        )
-        return self

datahub/specific/ownership.py

@@ -1,48 +0,0 @@
-from typing import Generic, List, Optional, TypeVar
-
-from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
-from datahub.metadata.schema_classes import (
-    OwnerClass,
-    OwnershipClass,
-    OwnershipTypeClass,
-)
-
-_Parent = TypeVar("_Parent", bound=MetadataPatchProposal)
-
-
-class OwnershipPatchHelper(Generic[_Parent]):
-    def __init__(self, parent: _Parent) -> None:
-        self._parent = parent
-        self.aspect_field = OwnershipClass.ASPECT_NAME
-
-    def parent(self) -> _Parent:
-        return self._parent
-
-    def add_owner(self, owner: OwnerClass) -> "OwnershipPatchHelper":
-        self._parent._add_patch(
-            OwnershipClass.ASPECT_NAME,
-            "add",
-            path=f"/owners/{owner.owner}/{owner.type}",
-            value=owner,
-        )
-        return self
-
-    def remove_owner(
-        self, owner: str, owner_type: Optional[OwnershipTypeClass] = None
-    ) -> "OwnershipPatchHelper":
-        """
-        param: owner_type is optional
-        """
-        self._parent._add_patch(
-            OwnershipClass.ASPECT_NAME,
-            "remove",
-            path=f"/owners/{owner}" + (f"/{owner_type}" if owner_type else ""),
-            value=owner,
-        )
-        return self
-
-    def set_owners(self, owners: List[OwnerClass]) -> "OwnershipPatchHelper":
-        self._parent._add_patch(
-            OwnershipClass.ASPECT_NAME, "add", path="/owners", value=owners
-        )
-        return self